90%训练集10%测试集1000次模拟时KernelKNN缺失值错误求助

问题背景 #

以90%为训练数据、10%为测试数据进行1000次模拟时，调用KernelKnn函数出现错误：the TEST_data includes missing values，原代码及错误详情如下：

#pemanggilan paket yang digunakan
library(caret)
library(KernelKnn)

#perhitungan akurasi prediksi dengan 100 kali simulasi
sim = function(B, p1, k1)
{
  hasileu = matrix(0, B, 4)
  for (i in 1:B)
  {
    #memanggil data boston
    boston = MASS::Boston

    #membagi data train dan data test
    indexes = createDataPartition(boston$medv, p = p1, list = F)
    train = boston[indexes, ]
    test = boston[-indexes, ]
    
    train_x = train[, -14]
    train_x = scale(train_x)[,]
    train_y = train[,14]
    
    test_x = test[, -14]
    test_x = scale(test[,-14])[,]
    test_y = test[,14]
    
    #model prediksi regresi KNN
    pred1 = KernelKnn(train_x, TEST_data = test_x, train_y, k = k1, method = 'euclidean', weights_function = NULL, regression = T)

    #perhitungan akurasi
    mse = mean((test_y - pred1)^2)
    mae = mean(abs(test_y - pred1))
    rmse = sqrt(mse)
    mape = mean(abs((test_y - pred1)/test_y))
    r2 = cor(test_y, pred1)^2
    hasileu[i,1] = rmse
    hasileu[i,2] = mape
    hasileu[i,3] = mae
    hasileu[i,4] = r2
  }
  return(apply(hasileu, 2, mean))
}

#penentuan rentang nilai K
hitung.variasi.k = function(B,p,K)
{
  has11 = matrix(0,K,4)
  for (i in 1:K)
  {
    has11[i,] = sim(B,p,i)
  }
  has11
}

#hasil akurasi berbagai proporsi data uji
K = 10
has11 = hitung.variasi.k(1000, 0.9,K)

错误信息：

Error in KernelKnn(train_x, TEST_data = test_x, train_y, k = k1, method = "euclidean", : 
the TEST_data includes missing values

错误原因 #

1. 标准化方式错误：原代码单独对测试集调用scale()，会基于测试集自身的均值和标准差计算标准化值。当测试集样本量极小（10%的Boston数据集仅约50个样本），可能出现某特征所有值完全相同的情况，此时标准差为0，scale()会生成NA，导致测试数据出现缺失值。
2. 数据泄露风险：即使没有生成NA，用测试集自身统计量标准化也属于数据泄露，违反机器学习训练/测试分离的原则。

解决方案 #

用训练集的均值和标准差标准化测试集，同时处理可能出现的零标准差情况，避免生成NA。

修改后的代码 #

library(caret)
library(KernelKnn)
library(MASS) # 提前加载MASS包避免循环中重复调用

sim = function(B, p1, k1)
{
  hasileu = matrix(0, B, 4)
  for (i in 1:B)
  {
    boston = Boston

    indexes = createDataPartition(boston$medv, p = p1, list = F)
    train = boston[indexes, ]
    test = boston[-indexes, ]
    
    train_x = train[, -14]
    # 计算训练集的标准化参数
    train_mean = colMeans(train_x)
    train_sd = apply(train_x, 2, sd)
    # 处理标准差为0的情况（避免除以0）
    train_sd[train_sd == 0] = 1
    # 标准化训练集
    train_x = t((t(train_x) - train_mean) / train_sd)
    train_y = train[,14]
    
    test_x = test[, -14]
    # 使用训练集的参数标准化测试集
    test_x = t((t(test_x) - train_mean) / train_sd)
    test_y = test[,14]
    
    # 可选：检查测试集是否仍有缺失值，用于调试
    if(any(is.na(test_x))) {
      warning(paste("Simulation", i, "has NA in test_x"))
      next
    }
    
    pred1 = KernelKnn(train_x, TEST_data = test_x, train_y, k = k1, method = 'euclidean', weights_function = NULL, regression = T)

    mse = mean((test_y - pred1)^2)
    mae = mean(abs(test_y - pred1))
    rmse = sqrt(mse)
    mape = mean(abs((test_y - pred1)/test_y))
    r2 = cor(test_y, pred1)^2
    hasileu[i,1] = rmse
    hasileu[i,2] = mape
    hasileu[i,3] = mae
    hasileu[i,4] = r2
  }
  return(apply(hasileu, 2, mean))
}

hitung.variasi.k = function(B,p,K)
{
  has11 = matrix(0,K,4)
  for (i in 1:K)
  {
    has11[i,] = sim(B,p,i)
  }
  has11
}

K = 10
has11 = hitung.variasi.k(1000, 0.9,K)

关键修改点 #

• 提前加载MASS包，避免循环中重复调用MASS::Boston
• 用训练集的统计量标准化测试集，遵循机器学习流程，避免数据泄露
• 处理标准差为0的情况，防止除以0生成NA
• 添加可选的缺失值检查，用于调试异常模拟

内容的提问来源于stack exchange，提问作者Lutfi Sivana Ihzaniah