Missing Data Handler | 缺失數據處理器

Missing Data Handler

Interactive techniques and best practices for handling missing data in Python and JavaScript

Core Concepts

Types of Missingness

Understanding MCAR, MAR, and MNAR patterns

MCAR: Missing Completely at Random MAR: Missing at Random MNAR: Missing Not at Random

Impact Assessment

Evaluating how missing data affects analysis

Missing Percentage: 15%

Strategy Selection

Decision framework for choosing techniques

Missing Data Pattern Visualizer

100 rows

Python Implementation

Deletion Strategies

import pandas as pd
import numpy as np

# Create sample dataset with missing values
data = {
    'Name': ['Alice', 'Bob', None, 'David', 'Eve'],
    'Age': [25, None, 30, 35, None],
    'Salary': [50000, 60000, None, 80000, 55000],
    'Department': ['IT', 'HR', 'IT', None, 'Finance']
}
df = pd.DataFrame(data)

print("Original Dataset:")
print(df)
print(f"\nMissing values per column:")
print(df.isnull().sum())

# Method 1: Drop rows with any missing values
df_dropna_any = df.dropna()
print(f"\nAfter dropping rows with ANY missing values:")
print(df_dropna_any)

# Method 2: Drop rows only if ALL values are missing
df_dropna_all = df.dropna(how='all')
print(f"\nAfter dropping rows with ALL missing values:")
print(df_dropna_all)

# Method 3: Drop columns with missing values
df_drop_cols = df.dropna(axis=1)
print(f"\nAfter dropping columns with missing values:")
print(df_drop_cols)

# Method 4: Drop with threshold (keep rows with at least 3 non-null values)
df_thresh = df.dropna(thresh=3)
print(f"\nAfter dropping rows with fewer than 3 non-null values:")
print(df_thresh)

Simple Imputation Methods

import pandas as pd
import numpy as np

# Create dataset with different types of missing data
np.random.seed(42)
dates = pd.date_range('2023-01-01', periods=10, freq='D')
data = {
    'Date': dates,
    'Temperature': [22.5, None, 24.1, 23.8, None, 25.2, 24.9, None, 23.4, 22.8],
    'Humidity': [45, 52, None, 48, 51, None, 47, 49, None, 46],
    'Category': ['A', 'B', None, 'A', 'B', 'A', None, 'B', 'A', None]
}
df = pd.DataFrame(data)

print("Original Dataset:")
print(df)

# Mean imputation for numerical data
df_mean = df.copy()
df_mean['Temperature'].fillna(df_mean['Temperature'].mean(), inplace=True)
df_mean['Humidity'].fillna(df_mean['Humidity'].mean(), inplace=True)
print(f"\nAfter Mean Imputation:")
print(df_mean[['Temperature', 'Humidity']])

# Median imputation
df_median = df.copy()
df_median['Temperature'].fillna(df_median['Temperature'].median(), inplace=True)
df_median['Humidity'].fillna(df_median['Humidity'].median(), inplace=True)
print(f"\nAfter Median Imputation:")
print(df_median[['Temperature', 'Humidity']])

# Mode imputation for categorical data
df_mode = df.copy()
mode_category = df_mode['Category'].mode()[0]
df_mode['Category'].fillna(mode_category, inplace=True)
print(f"\nAfter Mode Imputation for Category:")
print(df_mode['Category'])

# Forward fill for time series
df_ffill = df.copy()
df_ffill.fillna(method='ffill', inplace=True)
print(f"\nAfter Forward Fill:")
print(df_ffill)

# Backward fill
df_bfill = df.copy()
df_bfill.fillna(method='bfill', inplace=True)
print(f"\nAfter Backward Fill:")
print(df_bfill)

Advanced Imputation Techniques

import pandas as pd
import numpy as np
from scipy import interpolate

# Advanced imputation simulation
np.random.seed(42)
n_samples = 20
data = {
    'X': np.linspace(0, 10, n_samples),
    'Y': np.sin(np.linspace(0, 10, n_samples)) + np.random.normal(0, 0.1, n_samples),
    'Feature1': np.random.normal(50, 10, n_samples),
    'Feature2': np.random.normal(100, 20, n_samples)
}

# Introduce missing values
missing_indices = np.random.choice(n_samples, 6, replace=False)
for idx in missing_indices[:3]:
    data['Y'][idx] = np.nan
for idx in missing_indices[3:]:
    data['Feature1'][idx] = np.nan

df = pd.DataFrame(data)
print("Dataset with Missing Values:")
print(df.head(10))

# Method 1: Linear Interpolation
df_interp = df.copy()
df_interp['Y_interpolated'] = df_interp['Y'].interpolate(method='linear')
print(f"\nLinear Interpolation Results:")
print(df_interp[['Y', 'Y_interpolated']].head(10))

# Method 2: Polynomial Interpolation
df_poly = df.copy()
df_poly['Y_poly'] = df_poly['Y'].interpolate(method='polynomial', order=2)
print(f"\nPolynomial Interpolation Results:")
print(df_poly[['Y', 'Y_poly']].head(10))

# Method 3: KNN-style imputation simulation
def simple_knn_impute(df, column, k=3):
    """Simple KNN imputation simulation"""
    df_result = df.copy()
    missing_mask = df_result[column].isnull()
    
    for idx in df_result[missing_mask].index:
        # Find k nearest neighbors based on other features
        other_features = ['Feature2']  # Use available features
        distances = []
        
        for other_idx in df_result[~missing_mask].index:
            dist = sum((df_result.loc[idx, feat] - df_result.loc[other_idx, feat])**2 
                      for feat in other_features if pd.notna(df_result.loc[idx, feat]))
            distances.append((dist, other_idx))
        
        # Get k nearest neighbors
        distances.sort()
        nearest_k = distances[:min(k, len(distances))]
        
        # Impute with mean of k nearest neighbors
        imputed_value = np.mean([df_result.loc[neighbor_idx, column] 
                               for _, neighbor_idx in nearest_k])
        df_result.loc[idx, column] = imputed_value
    
    return df_result[column]

# Apply KNN-style imputation
df['Feature1_knn'] = simple_knn_impute(df, 'Feature1', k=3)
print(f"\nKNN-style Imputation Results:")
print(df[['Feature1', 'Feature1_knn']].head(10))

# Method 4: Multiple Imputation simulation
def multiple_imputation_simulation(df, column, n_imputations=3):
    """Simple multiple imputation simulation"""
    results = []
    
    for i in range(n_imputations):
        df_temp = df.copy()
        # Add random noise to make different imputations
        noise_factor = 0.1 * np.std(df_temp[column].dropna())
        
        mean_val = df_temp[column].mean()
        noise = np.random.normal(0, noise_factor, df_temp[column].isnull().sum())
        
        df_temp.loc[df_temp[column].isnull(), column] = mean_val + noise
        results.append(df_temp[column].values)
    
    # Average the imputations
    final_imputation = np.mean(results, axis=0)
    return final_imputation

multiple_result = multiple_imputation_simulation(df, 'Feature1')
df['Feature1_multiple'] = multiple_result
print(f"\nMultiple Imputation Results:")
print(df[['Feature1', 'Feature1_multiple']].head(10))

JavaScript Implementation

JavaScript Array Operations

// Sample dataset with missing values
const dataset = [
    { name: 'Alice', age: 25, salary: 50000, department: 'IT' },
    { name: 'Bob', age: null, salary: 60000, department: 'HR' },
    { name: null, age: 30, salary: null, department: 'IT' },
    { name: 'David', age: 35, salary: 80000, department: null },
    { name: 'Eve', age: null, salary: 55000, department: 'Finance' }
];

console.log('Original Dataset:');
console.log(dataset);

// Function to check if object has missing values
function hasMissingValues(obj, fields = []) {
    if (fields.length === 0) fields = Object.keys(obj);
    return fields.some(field => obj[field] === null || obj[field] === undefined);
}

// Method 1: Remove rows with any missing values
const completeRecords = dataset.filter(record => !hasMissingValues(record));
console.log('\nComplete records only:');
console.log(completeRecords);

// Method 2: Remove rows with missing values in specific fields
const requiredFields = ['name', 'age'];
const filteredByFields = dataset.filter(record => !hasMissingValues(record, requiredFields));
console.log('\nRecords with required fields complete:');
console.log(filteredByFields);

// Method 3: Count missing values per field
function countMissingValues(data) {
    const fields = Object.keys(data[0]);
    const counts = {};
    
    fields.forEach(field => {
        counts[field] = data.filter(record => 
            record[field] === null || record[field] === undefined
        ).length;
    });
    
    return counts;
}

const missingCounts = countMissingValues(dataset);
console.log('\nMissing values per field:');
console.log(missingCounts);

// Method 4: Remove fields with high missing percentage
function removeHighMissingFields(data, threshold = 0.5) {
    const totalRecords = data.length;
    const missingCounts = countMissingValues(data);
    const fieldsToKeep = Object.keys(missingCounts).filter(field => 
        missingCounts[field] / totalRecords < threshold
    );
    
    return data.map(record => {
        const newRecord = {};
        fieldsToKeep.forEach(field => {
            newRecord[field] = record[field];
        });
        return newRecord;
    });
}

const reducedFields = removeHighMissingFields(dataset, 0.4);
console.log('\nAfter removing high-missing fields (>40%):');
console.log(reducedFields);

JavaScript Imputation Methods

// Enhanced dataset for imputation testing
const timeSeriesData = [
    { date: '2023-01-01', temperature: 22.5, humidity: 45, category: 'A' },
    { date: '2023-01-02', temperature: null, humidity: 52, category: 'B' },
    { date: '2023-01-03', temperature: 24.1, humidity: null, category: null },
    { date: '2023-01-04', temperature: 23.8, humidity: 48, category: 'A' },
    { date: '2023-01-05', temperature: null, humidity: 51, category: 'B' },
    { date: '2023-01-06', temperature: 25.2, humidity: null, category: 'A' },
    { date: '2023-01-07', temperature: 24.9, humidity: 47, category: null },
    { date: '2023-01-08', temperature: null, humidity: 49, category: 'B' }
];

console.log('Original Time Series Data:');
console.log(timeSeriesData);

// Statistical helper functions
function mean(arr) {
    const valid = arr.filter(x => x !== null && x !== undefined);
    return valid.length > 0 ? valid.reduce((a, b) => a + b, 0) / valid.length : null;
}

function median(arr) {
    const valid = arr.filter(x => x !== null && x !== undefined).sort((a, b) => a - b);
    if (valid.length === 0) return null;
    const mid = Math.floor(valid.length / 2);
    return valid.length % 2 !== 0 ? valid[mid] : (valid[mid - 1] + valid[mid]) / 2;
}

function mode(arr) {
    const valid = arr.filter(x => x !== null && x !== undefined);
    if (valid.length === 0) return null;
    
    const frequency = {};
    valid.forEach(val => frequency[val] = (frequency[val] || 0) + 1);
    
    let maxCount = 0;
    let modes = [];
    for (const val in frequency) {
        if (frequency[val] > maxCount) {
            maxCount = frequency[val];
            modes = [val];
        } else if (frequency[val] === maxCount) {
            modes.push(val);
        }
    }
    return modes[0];
}

// Method 1: Mean Imputation
function meanImputation(data, field) {
    const values = data.map(record => record[field]);
    const meanValue = mean(values);
    
    return data.map(record => ({
        ...record,
        [field]: record[field] !== null && record[field] !== undefined ? record[field] : meanValue
    }));
}

const meanImputed = meanImputation([...timeSeriesData], 'temperature');
console.log('\nAfter Mean Imputation (Temperature):');
console.log(meanImputed.map(d => ({ date: d.date, temperature: d.temperature })));

// Method 2: Median Imputation
function medianImputation(data, field) {
    const values = data.map(record => record[field]);
    const medianValue = median(values);
    
    return data.map(record => ({
        ...record,
        [field]: record[field] !== null && record[field] !== undefined ? record[field] : medianValue
    }));
}

const medianImputed = medianImputation([...timeSeriesData], 'humidity');
console.log('\nAfter Median Imputation (Humidity):');
console.log(medianImputed.map(d => ({ date: d.date, humidity: d.humidity })));

// Method 3: Mode Imputation for categorical data
function modeImputation(data, field) {
    const values = data.map(record => record[field]);
    const modeValue = mode(values);
    
    return data.map(record => ({
        ...record,
        [field]: record[field] !== null && record[field] !== undefined ? record[field] : modeValue
    }));
}

const modeImputed = modeImputation([...timeSeriesData], 'category');
console.log('\nAfter Mode Imputation (Category):');
console.log(modeImputed.map(d => ({ date: d.date, category: d.category })));

// Method 4: Forward Fill
function forwardFill(data, field) {
    let lastValidValue = null;
    
    return data.map(record => {
        if (record[field] !== null && record[field] !== undefined) {
            lastValidValue = record[field];
            return record;
        } else {
            return {
                ...record,
                [field]: lastValidValue
            };
        }
    });
}

const forwardFilled = forwardFill([...timeSeriesData], 'temperature');
console.log('\nAfter Forward Fill (Temperature):');
console.log(forwardFilled.map(d => ({ date: d.date, temperature: d.temperature })));

// Method 5: Backward Fill
function backwardFill(data, field) {
    const result = [...data];
    let nextValidValue = null;
    
    // Iterate backwards
    for (let i = result.length - 1; i >= 0; i--) {
        if (result[i][field] !== null && result[i][field] !== undefined) {
            nextValidValue = result[i][field];
        } else if (nextValidValue !== null) {
            result[i] = {
                ...result[i],
                [field]: nextValidValue
            };
        }
    }
    
    return result;
}

const backwardFilled = backwardFill([...timeSeriesData], 'temperature');
console.log('\nAfter Backward Fill (Temperature):');
console.log(backwardFilled.map(d => ({ date: d.date, temperature: d.temperature })));

Advanced JavaScript Algorithms

// Advanced imputation algorithms in JavaScript
const advancedDataset = [
    { id: 1, feature1: 10, feature2: 20, target: 100 },
    { id: 2, feature1: 15, feature2: null, target: 150 },
    { id: 3, feature1: null, feature2: 25, target: null },
    { id: 4, feature1: 20, feature2: 30, target: 200 },
    { id: 5, feature1: 25, feature2: null, target: 250 },
    { id: 6, feature1: null, feature2: 35, target: null },
    { id: 7, feature1: 30, feature2: 40, target: 300 },
    { id: 8, feature1: 35, feature2: 45, target: 350 }
];

console.log('Advanced Dataset:');
console.log(advancedDataset);

// Method 1: Linear Interpolation
function linearInterpolation(data, field) {
    const result = [...data];
    
    for (let i = 0; i < result.length; i++) {
        if (result[i][field] === null || result[i][field] === undefined) {
            // Find previous and next valid values
            let prevIdx = i - 1;
            let nextIdx = i + 1;
            
            while (prevIdx >= 0 && (result[prevIdx][field] === null || result[prevIdx][field] === undefined)) {
                prevIdx--;
            }
            
            while (nextIdx < result.length && (result[nextIdx][field] === null || result[nextIdx][field] === undefined)) {
                nextIdx++;
            }
            
            if (prevIdx >= 0 && nextIdx < result.length) {
                // Linear interpolation
                const prevVal = result[prevIdx][field];
                const nextVal = result[nextIdx][field];
                const ratio = (i - prevIdx) / (nextIdx - prevIdx);
                result[i][field] = prevVal + ratio * (nextVal - prevVal);
            } else if (prevIdx >= 0) {
                // Use previous value
                result[i][field] = result[prevIdx][field];
            } else if (nextIdx < result.length) {
                // Use next value
                result[i][field] = result[nextIdx][field];
            }
        }
    }
    
    return result;
}

const interpolated = linearInterpolation([...advancedDataset], 'feature1');
console.log('\nAfter Linear Interpolation (feature1):');
console.log(interpolated.map(d => ({ id: d.id, feature1: d.feature1 })));

// Method 2: K-Nearest Neighbors Imputation
function knnImputation(data, field, k = 3, referenceFields = []) {
    const result = [...data];
    
    // If no reference fields specified, use all numeric fields except the target
    if (referenceFields.length === 0) {
        referenceFields = Object.keys(data[0]).filter(key => 
            key !== field && 
            key !== 'id' && 
            typeof data.find(d => d[key] !== null && d[key] !== undefined)?.[key] === 'number'
        );
    }
    
    for (let i = 0; i < result.length; i++) {
        if (result[i][field] === null || result[i][field] === undefined) {
            // Calculate distances to all other records with valid target values
            const distances = [];
            
            for (let j = 0; j < result.length; j++) {
                if (i !== j && result[j][field] !== null && result[j][field] !== undefined) {
                    let distance = 0;
                    let validFeatures = 0;
                    
                    for (const refField of referenceFields) {
                        if (result[i][refField] !== null && result[i][refField] !== undefined &&
                            result[j][refField] !== null && result[j][refField] !== undefined) {
                            distance += Math.pow(result[i][refField] - result[j][refField], 2);
                            validFeatures++;
                        }
                    }
                    
                    if (validFeatures > 0) {
                        distance = Math.sqrt(distance / validFeatures);
                        distances.push({ index: j, distance: distance });
                    }
                }
            }
            
            // Sort by distance and take k nearest
            distances.sort((a, b) => a.distance - b.distance);
            const nearest = distances.slice(0, Math.min(k, distances.length));
            
            if (nearest.length > 0) {
                // Calculate weighted average (inverse distance weighting)
                let weightedSum = 0;
                let totalWeight = 0;
                
                for (const neighbor of nearest) {
                    const weight = neighbor.distance === 0 ? 1 : 1 / (neighbor.distance + 1e-8);
                    weightedSum += result[neighbor.index][field] * weight;
                    totalWeight += weight;
                }
                
                result[i][field] = weightedSum / totalWeight;
            }
        }
    }
    
    return result;
}

const knnImputed = knnImputation([...advancedDataset], 'target', 3, ['feature1', 'feature2']);
console.log('\nAfter KNN Imputation (target):');
console.log(knnImputed.map(d => ({ id: d.id, target: d.target })));

// Method 3: Multiple Imputation
function multipleImputation(data, field, nImputations = 5) {
    const imputations = [];
    
    for (let imp = 0; imp < nImputations; imp++) {
        const result = [...data];
        
        // Add random noise to create different imputations
        const validValues = data.filter(d => d[field] !== null && d[field] !== undefined)
                               .map(d => d[field]);
        
        if (validValues.length > 0) {
            const mean = validValues.reduce((a, b) => a + b, 0) / validValues.length;
            const variance = validValues.reduce((a, b) => a + Math.pow(b - mean, 2), 0) / validValues.length;
            const stdDev = Math.sqrt(variance);
            
            for (let i = 0; i < result.length; i++) {
                if (result[i][field] === null || result[i][field] === undefined) {
                    // Generate random value from normal distribution
                    const noise = (Math.random() - 0.5) * 2 * stdDev * 0.2; // 20% of std dev
                    result[i][field] = mean + noise;
                }
            }
        }
        
        imputations.push(result);
    }
    
    // Average the imputations
    const finalResult = [...data];
    for (let i = 0; i < finalResult.length; i++) {
        if (finalResult[i][field] === null || finalResult[i][field] === undefined) {
            const values = imputations.map(imp => imp[i][field]);
            finalResult[i][field] = values.reduce((a, b) => a + b, 0) / values.length;
        }
    }
    
    return finalResult;
}

const multipleImputed = multipleImputation([...advancedDataset], 'feature2', 3);
console.log('\nAfter Multiple Imputation (feature2):');
console.log(multipleImputed.map(d => ({ id: d.id, feature2: d.feature2 })));

// Method 4: Regression-based Imputation
function regressionImputation(data, targetField, predictorFields) {
    // Simple linear regression for single predictor
    function simpleLinearRegression(x, y) {
        const n = x.length;
        const sumX = x.reduce((a, b) => a + b, 0);
        const sumY = y.reduce((a, b) => a + b, 0);
        const sumXY = x.reduce((sum, xi, i) => sum + xi * y[i], 0);
        const sumXX = x.reduce((sum, xi) => sum + xi * xi, 0);
        
        const slope = (n * sumXY - sumX * sumY) / (n * sumXX - sumX * sumX);
        const intercept = (sumY - slope * sumX) / n;
        
        return { slope, intercept };
    }
    
    const result = [...data];
    
    // Use first predictor field for simple regression
    const predictorField = predictorFields[0];
    
    // Get complete cases for training
    const completeData = data.filter(d => 
        d[targetField] !== null && d[targetField] !== undefined &&
        d[predictorField] !== null && d[predictorField] !== undefined
    );
    
    if (completeData.length > 1) {
        const x = completeData.map(d => d[predictorField]);
        const y = completeData.map(d => d[targetField]);
        
        const { slope, intercept } = simpleLinearRegression(x, y);
        
        // Predict missing values
        for (let i = 0; i < result.length; i++) {
            if ((result[i][targetField] === null || result[i][targetField] === undefined) &&
                result[i][predictorField] !== null && result[i][predictorField] !== undefined) {
                result[i][targetField] = slope * result[i][predictorField] + intercept;
            }
        }
    }
    
    return result;
}

const regressionImputed = regressionImputation([...advancedDataset], 'target', ['feature1']);
console.log('\nAfter Regression Imputation (target ~ feature1):');
console.log(regressionImputed.map(d => ({ id: d.id, feature1: d.feature1, target: d.target })));

Interactive Comparison Tool

Best Practices & Guidelines

Decision Framework

Quick Assessment:

Large dataset (>1000 rows) Low missing rate (<10%) Random missing pattern

Validation Checklist

✓ Analyzed missing patterns
✓ Compared distributions before/after
✓ Tested multiple methods
✓ Validated on holdout data
✓ Documented assumptions
✓ Assessed potential bias

Trade-off Analysis Matrix

Method	Speed	Accuracy	Bias Risk	Best For
Deletion	⚡⚡⚡	⭐⭐	🔴 High	MCAR, Large datasets
Mean/Median	⚡⚡⚡	⭐⭐	🟡 Medium	Simple, Quick fixes
KNN	⚡⚡	⭐⭐⭐	🟡 Medium	Mixed data types
Multiple	⚡	⭐⭐⭐⭐	🟢 Low	Critical analysis

搜尋此網誌

AI Hot Shorts

Missing Data Handler

Missing Data Handler

Core Concepts

Types of Missingness

Impact Assessment

Strategy Selection

Missing Data Pattern Visualizer

Python Implementation

Deletion Strategies

Simple Imputation Methods

Advanced Imputation Techniques

JavaScript Implementation

JavaScript Array Operations

JavaScript Imputation Methods

Advanced JavaScript Algorithms

Interactive Comparison Tool

Best Practices & Guidelines

Decision Framework

Quick Assessment:

Validation Checklist

Trade-off Analysis Matrix

留言

發佈留言

此網誌的熱門文章

Ember's Yoga Odyssey

Cross-Disciplinary Knowledge Integration

Ember's Whisper: A Journey of Fiery Hearts