如何使用caret包绘制交叉验证平均AUC曲线、计算95%CI及叠加对比

核心实现逻辑 #

通过固定公共FPR（假阳性率）阈值，对每个折-重复组合的ROC做插值得到统一长度的TPR（真阳性率）序列，再分组计算均值解决不同迭代样本量不一致的问题。

步骤1：单模型平均ROC与AUC置信区间计算 #

# 加载依赖包
library(dplyr)
library(pROC)
library(ggplot2)

# 提取所有预测结果，定义公共FPR序列（0到1取100个点保证平滑）
pred_df <- model$pred
fpr_seq <- seq(0, 1, length.out = 100)

# 按折+重复分组，计算每个迭代的ROC并插值到统一FPR阈值
roc_list <- pred_df %>%
  group_by(Resample) %>%
  group_modify(~{
    roc_obj <- roc(response = .x$obs, predictor = .x$Yes, quiet = TRUE)
    tpr_interp <- approx(1 - roc_obj$specificities, roc_obj$sensitivities, xout = fpr_seq)$y
    tibble(FPR = fpr_seq, TPR = tpr_interp)
  }) %>%
  ungroup()

# 计算每个FPR对应的平均TPR及95%置信区间
mean_roc <- roc_list %>%
  group_by(FPR) %>%
  summarise(
    mean_TPR = mean(TPR, na.rm = TRUE),
    tpr_low = quantile(TPR, 0.025, na.rm = TRUE),
    tpr_high = quantile(TPR, 0.975, na.rm = TRUE)
  )

# 计算平均AUC的95%经验置信区间
auc_vec <- model$resample$ROC
mean_auc <- mean(auc_vec)
auc_ci <- quantile(auc_vec, c(0.025, 0.975))
cat("平均AUC：", round(mean_auc,3), " 95%置信区间：", round(auc_ci[1],3), "-", round(auc_ci[2],3), "\n")

步骤2：单模型平均ROC曲线绘制 #

ggplot(mean_roc, aes(x = FPR, y = mean_TPR)) +
  geom_line(linewidth = 1, color = "#2c3e50") +
  # 可选添加TPR的95%置信区间带
  geom_ribbon(aes(ymin = tpr_low, ymax = tpr_high), alpha = 0.2, fill = "#2c3e50") +
  # 添加随机猜测参考线
  geom_abline(slope = 1, intercept = 0, linetype = "dashed", color = "gray") +
  labs(
    x = "假阳性率 (1 - 特异度)",
    y = "真阳性率 (灵敏度)",
    title = "10折10次重复交叉验证平均ROC曲线",
    subtitle = paste0("平均AUC = ", round(mean_auc,3), " (95%CI: ", round(auc_ci[1],3), "-", round(auc_ci[2],3), ")")
  ) +
  theme_bw()

步骤3：双模型ROC曲线叠加对比 #

按步骤1的流程完成第二个模型（使用其他预测变量训练的模型）的mean_roc、mean_auc、auc_ci计算后，执行以下代码即可叠加曲线：

# 给两个模型的ROC数据添加标识
mean_roc$model <- "模型1：Class1 + Class2"
mean_roc2$model <- "模型2：其他预测变量" # 替换为第二个模型的mean_roc数据
combined_roc <- rbind(mean_roc, mean_roc2)

# 绘制对比曲线
ggplot(combined_roc, aes(x = FPR, y = mean_TPR, color = model, fill = model)) +
  geom_line(linewidth = 1) +
  geom_ribbon(aes(ymin = tpr_low, ymax = tpr_high), alpha = 0.15) +
  geom_abline(slope = 1, intercept = 0, linetype = "dashed", color = "gray") +
  labs(
    x = "假阳性率 (1 - 特异度)",
    y = "真阳性率 (灵敏度)",
    title = "两个模型交叉验证平均ROC曲线对比",
    color = "模型类型",
    fill = "模型类型"
  ) +
  # 添加AUC标注
  annotate("text", x = 0.75, y = 0.25, 
           label = paste0("模型1 AUC = ", round(mean_auc,3), " (95%CI: ", round(auc_ci[1],3), "-", round(auc_ci[2],3), ")\n",
                          "模型2 AUC = ", round(mean_auc2,3), " (95%CI: ", round(auc_ci2[1],3), "-", round(auc_ci2[2],3), ")")) +
  theme_bw()

内容的提问来源于stack exchange，提问作者MAMY