传感器油耗数据去噪优化问询：IQR+移动平均后仍存骤降问题

我正在处理传感器采集的油耗数据，数据存在突变跳变/骤降类噪声，导致数据不一致。目标是识别并剔除这些异常值，保障后续分析的准确性与可靠性。

关键详情 #

• 传感器记录包含Unix时间戳、油耗值、车速及其他数据；
• 每10分钟生成40-80条数据；
• 需要一致且稳健的去噪与数据平滑方法。

已实现的代码 #

// value == Fuel Consumption
var data = FileReader.ReadCsv(path).Where(d => d.Value > 0).ToList();

var cleanedData = RemoveOutliers(data.Select(d => new DataPoint(d.Timestamp, d.Value, d.Speed)).ToList(), 1.5);
cleanedData = ApplyMovingAverage(cleanedData, 8);

List<AnomalyDetectionResult> anomalyDetectionResults = [];
foreach (var dataPoint in cleanedData)
{
    // todo
}

static List<DataPoint> RemoveOutliers(List<DataPoint> data, double iqrFactor)
{
    var values = data.Select(d => d.Value).ToList();
    values.Sort();

    double q1 = GetPercentile(values, 25);
    double q3 = GetPercentile(values, 75);
    double iqr = q3 - q1;
    double lowerBound = q1 - iqrFactor * iqr;
    double upperBound = q3 + iqrFactor * iqr;

    return data.Where(d => d.Value >= lowerBound && d.Value <= upperBound).ToList();
}

static List<DataPoint> ApplyMovingAverage(List<DataPoint> data, int windowSize)
{
    var smoothedData = new List<DataPoint>();
    for (int i = 0; i < data.Count; i++)
    {
        var window = data.Skip(Math.Max(0, i - windowSize + 1)).Take(windowSize).ToList();
        double avg = window.Average(d => d.Value);
        smoothedData.Add(new DataPoint(data[i].Timestamp, avg, data[i].Speed));
    }
    return smoothedData;
}

static double GetPercentile(List<double> sortedValues, double percentile)
{
    if (!sortedValues.Any()) return 0;

    double rank = percentile / 100.0 * (sortedValues.Count - 1);
    int lowerIndex = (int)Math.Floor(rank);
    int upperIndex = (int)Math.Ceiling(rank);

    if (lowerIndex == upperIndex) return sortedValues[lowerIndex];

    return sortedValues[lowerIndex] + (rank - lowerIndex) * (sortedValues[upperIndex] - sortedValues[lowerIndex]);
}

public class DataPoint(DateTime timestamp, double value, int speed)
{
    public DateTime Timestamp { get; set; } = timestamp;
    public double Value { get; set; } = value;
    public int Speed { get; set; } = speed;
}

数据处理效果 #

处理前数据 #

处理后仍存在骤降的情况 #

优化方案与指导 #

问题根源 #

当前全局IQR检测仅基于数据整体分布，忽略了油耗的时序依赖特性——骤降噪声属于局部时序突变，而非全局分布异常，因此无法被有效识别；后续的普通移动平均仅能平滑数据，无法修复残留的局部异常。

具体优化措施 #

1. 改用局部时序窗口的异常检测

替换全局IQR，采用滑动窗口内的统计特征识别局部突变，同时结合业务逻辑（车速与油耗的关联）：

static List<DataPoint> RemoveTemporalOutliers(List<DataPoint> data, int windowSize, double deviationThreshold)
{
    var cleaned = new List<DataPoint>();
    for (int i = 0; i < data.Count; i++)
    {
        // 构造对称滑动窗口，避免边界越界
        int start = Math.Max(0, i - windowSize / 2);
        int end = Math.Min(data.Count - 1, i + windowSize / 2);
        var window = data.GetRange(start, end - start + 1);
        
        double windowMean = window.Average(d => d.Value);
        double windowStd = Math.Sqrt(window.Average(d => Math.Pow(d.Value - windowMean, 2)));
        
        // 业务规则：车速为0时油耗应接近0，此场景不判定为异常
        if (data[i].Speed == 0 && data[i].Value < 0.1)
        {
            cleaned.Add(data[i]);
            continue;
        }
        
        // 绝对偏差超过阈值时，用窗口均值替换异常值（避免时序断裂）
        if (Math.Abs(data[i].Value - windowMean) <= deviationThreshold * windowStd)
        {
            cleaned.Add(data[i]);
        }
        else
        {
            cleaned.Add(new DataPoint(data[i].Timestamp, windowMean, data[i].Speed));
        }
    }
    return cleaned;
}

2. 替换移动平均为中位数滤波

普通移动平均易受残留异常值干扰，中位数滤波对突变噪声的鲁棒性更强：

static List<DataPoint> ApplyMedianFilter(List<DataPoint> data, int windowSize)
{
    var smoothed = new List<DataPoint>();
    for (int i = 0; i < data.Count; i++)
    {
        int start = Math.Max(0, i - windowSize / 2);
        int end = Math.Min(data.Count - 1, i + windowSize / 2);
        var windowValues = data.GetRange(start, end - start + 1)
                               .Select(d => d.Value)
                               .OrderBy(v => v)
                               .ToList();
        
        double median;
        int mid = windowValues.Count / 2;
        if (windowValues.Count % 2 == 0)
        {
            median = (windowValues[mid - 1] + windowValues[mid]) / 2.0;
        }
        else
        {
            median = windowValues[mid];
        }
        
        smoothed.Add(new DataPoint(data[i].Timestamp, median, data[i].Speed));
    }
    return smoothed;
}

3. 增加业务规则校验

利用油耗与车速的强相关性过滤异常：

static bool IsValidByBusinessRule(DataPoint current, DataPoint previous)
{
    if (previous == null) return true;
    
    // 车速从非0变为0时，油耗应快速降至怠速水平（不超过行驶时30%）
    if (current.Speed == 0 && previous.Speed > 0)
    {
        return current.Value < previous.Value * 0.3;
    }
    
    // 车速稳定波动（<5）时，油耗波动不应超过20%
    if (Math.Abs(current.Speed - previous.Speed) < 5)
    {
        return Math.Abs(current.Value - previous.Value) < previous.Value * 0.2;
    }
    
    return true;
}

4. 优化处理流程

推荐执行顺序：

1. 业务规则校验，过滤明显不符合逻辑的点；
2. 局部时序异常检测，替换突变值；
3. 中位数滤波平滑，消除残留噪声。

示例调用：

var rawData = FileReader.ReadCsv(path)
                       .Where(d => d.Value > 0)
                       .Select(d => new DataPoint(d.Timestamp, d.Value, d.Speed))
                       .ToList();

// 步骤1：业务规则过滤
var validatedData = new List<DataPoint>();
for (int i = 0; i < rawData.Count; i++)
{
    if (i == 0 || IsValidByBusinessRule(rawData[i], rawData[i-1]))
    {
        validatedData.Add(rawData[i]);
    }
}

// 步骤2：局部时序异常处理
var temporalCleaned = RemoveTemporalOutliers(validatedData, 10, 2.0); // 窗口大小10，2倍标准差阈值

// 步骤3：中位数平滑
var finalCleanedData = ApplyMedianFilter(temporalCleaned, 5);

内容的提问来源于stack exchange，提问作者DPTP