修正滚动MAD异常检测函数：动态排除已识别异常值

核心需求 #

• 基于窗口大小为3的滚动Median Absolute Deviation (MAD)算法检测异常值，计算中位数与MAD
• 已检测出的异常值排除在后续MAD计算之外
• 滚动窗口仅使用非异常值实现动态适配

原问题代码 #

detect_anomalies <- function(df, window_size, k) {
  df <- df %>% 
    arrange(Date_Label)

  cleaned_values <- c()  # Stores values that are NOT anomalies
  removed_rows <- list()  # Stores detected anomalies
  
  # Initialize new columns
  df$rolling_median <- NA
  df$abs_dev <- NA
  df$mad_value <- NA
  df$lower_bound <- NA
  df$upper_bound <- NA
  df$anomaly <- FALSE
  
  # Process each row iteratively
  for (i in 1:nrow(df)) {
    # Add the current value to cleaned_values first
    cleaned_values <- c(cleaned_values, df$Attempts[i])

    # Ensure we have enough past values
    if (length(cleaned_values) >= window_size) {
      # Extract past `window_size` values before adding the current row
      past_values <- head(cleaned_values, window_size)
      
      # Compute rolling median & MAD
      median_value <- median(past_values, na.rm = TRUE)
      abs_dev <- abs(df$Attempts[i] - median_value)
      mad_value <- median(tail(abs_dev, window_size), na.rm = TRUE)
      
      # Define anomaly thresholds
      lower_bound <- median_value - k * mad_value
      upper_bound <- median_value + k * mad_value
      
      # Check if the current row is an anomaly
      is_anomaly <- ifelse(df$Attempts[i] < lower_bound, "TRUE", "FALSE")

      # Store values in dataframe
      df$rolling_median[i] <- median_value
      df$abs_dev[i] <- abs_dev
      df$mad_value[i] <- mad_value
      df$lower_bound[i] <- lower_bound
      df$upper_bound[i] <- upper_bound
      df$anomaly[i] <- is_anomaly
    
    # Add current value to cleaned_values only if it's NOT an anomaly
      if (i <= window_size) {  
        cleaned_values <- c(cleaned_values, df$Attempts[i])  # Always add the first window_size values
        } else if (is_anomaly == "FALSE") {  
          cleaned_values <- c(cleaned_values, df$Attempts[i])  # Add only non-anomalous values
          } else {  
            removed_rows[[length(removed_rows) + 1]] <- df[i, ]  # Store anomaly separately
          }
    }
  }
  
  return(list(cleaned_df = df, anomalies = do.call(rbind, removed_rows)))
  }

# Run the anomaly detection function
result <- detect_anomalies(data, window_size = 3, k = 1)

# Cleaned dataset (excluding anomalies)
cleaned_data <- result$cleaned_df

# Anomalous rows
anomalies <- result$anomalies

示例数据集 #

数据结构与样本 #

代码生成数据集 #

library(tibble)

data <- tibble::tibble(
  Date_Label = as.Date(c("2024-04-01", "2024-05-01", "2024-06-01", "2024-07-01", 
                         "2024-08-01", "2024-09-01", "2024-10-01", "2024-11-01", 
                         "2024-12-01", "2025-01-01")),
  Attempts = c(186518, 202397, 252707, 236194, 217135, 240986, 205524, 
               160624, 142238, 193088)
)

期望输出结果 #

示例说明：11月Attempts被标记为异常（基于9-10月非异常值+当前值的中位数判断）；12月Attempts基于9-10月非异常值检测为异常，11月异常值已排除在窗口计算外。

修正后的代码 #

detect_anomalies <- function(df, window_size, k) {
  df <- df %>% 
    arrange(Date_Label) %>%
    mutate(
      rolling_median = NA_real_,
      abs_dev = NA_real_,
      mad_value = NA_real_,
      lower_bound = NA_real_,
      upper_bound = NA_real_,
      anomaly = NA_logical_
    )
  
  cleaned_values <- c()  # 存储非异常值，用于滚动窗口计算
  removed_rows <- list()  # 存储检测到的异常行
  
  # 初始化前window_size-1个值为非异常，加入cleaned_values
  for (i in 1:(window_size - 1)) {
    cleaned_values <- c(cleaned_values, df$Attempts[i])
  }
  
  # 从第window_size行开始处理
  for (i in window_size:nrow(df)) {
    # 获取当前待检测值
    current_val <- df$Attempts[i]
    
    # 取最近的window_size个非异常值作为计算窗口
    # 如果非异常值不足window_size，用全部已有的非异常值（保证计算可行）
    window_vals <- tail(cleaned_values, window_size)
    
    # 计算中位数
    median_val <- median(window_vals, na.rm = TRUE)
    
    # 计算MAD：窗口内每个值与中位数的绝对偏差的中位数
    abs_deviations <- abs(window_vals - median_val)
    mad_val <- median(abs_deviations, na.rm = TRUE)
    
    # 计算上下阈值
    lower_bound <- median_val - k * mad_val
    upper_bound <- median_val + k * mad_val
    
    # 判断是否为异常：超出上下阈值
    is_anomaly <- current_val < lower_bound || current_val > upper_bound
    
    # 填充当前行的计算结果
    df$rolling_median[i] <- median_val
    df$abs_dev[i] <- abs(current_val - median_val)
    df$mad_value[i] <- mad_val
    df$lower_bound[i] <- lower_bound
    df$upper_bound[i] <- upper_bound
    df$anomaly[i] <- is_anomaly
    
    # 更新cleaned_values或存储异常行
    if (!is_anomaly) {
      cleaned_values <- c(cleaned_values, current_val)
    } else {
      removed_rows[[length(removed_rows) + 1]] <- df[i, ]
    }
  }
  
  # 转换异常列表为数据框（若没有异常则返回空数据框）
  anomalies_df <- if (length(removed_rows) > 0) do.call(rbind, removed_rows) else df[0, ]
  
  return(list(cleaned_df = df, anomalies = anomalies_df))
}

# 运行函数
result <- detect_anomalies(data, window_size = 3, k = 1)

# 获取结果
cleaned_data <- result$cleaned_df
anomalies <- result$anomalies

代码修正说明 #

1. 窗口计算逻辑修正：先基于历史非异常值构建窗口，再检测当前值，避免待检测值提前混入计算窗口
2. MAD计算逻辑修正：正确计算窗口内所有值与中位数的绝对偏差，再取该偏差的中位数作为MAD
3. 异常判断逻辑完善：同时检查上下阈值，符合MAD异常检测的标准逻辑
4. cleaned_values管理优化：初始仅填充前window_size-1个值，后续仅将非异常值加入，避免重复添加和逻辑混乱
5. 数据类型统一：使用布尔值存储异常标记，避免字符串类型导致的判断错误
6. 边界处理：当非异常值不足window_size时，用全部已有非异常值计算，保证流程不中断

内容的提问来源于stack exchange，提问作者user2845095