如何在GridSearchCV中传递自定义迭代器参数至自定义评分函数？

相关依赖导入 #

from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import PredefinedSplit
from sklearn.base import clone
from sklearn.metrics import make_scorer
from mlxtend.evaluate import bias_variance_decomp
from sklearn.neighbors import KNeighborsRegressor
import numpy as np

原实现代码 #

自定义交叉验证迭代器（不符合sklearn规范） #

原代码尝试直接返回fold数据和克隆的估计器，但sklearn的CV组件仅接受返回(train_index, test_index)的迭代器：

knn = KNeighborsRegressor()
X_train_len = int(round(len(X_train_knn)*0.8, 0))
test_fold = np.hstack([-1 * np.ones(X_train_len), np.zeros(len(X_train_knn)-X_train_len)])
ps = PredefinedSplit(test_fold)

def custom_iterator(X_train_knn, Y_train_ar, estimator, ps=ps):
    for train_index, test_index in ps.split():
        X_train_fold, X_test_fold = X_train_knn[train_index], X_train_knn[test_index]
        y_train_fold, y_test_fold = Y_train_ar[train_index], Y_train_ar[test_index]
        estimator1 = clone(estimator)
        yield X_train_fold, X_test_fold, y_train_fold, y_test_fold, estimator1

iterator = custom_iterator(X_train_knn, Y_train_ar, estimator=knn, ps=ps )

自定义评分函数 #

原函数需要接收fold数据和估计器来计算偏差-方差分解的期望损失：

def custom_loss_func(y_true, y_pred, estimator1, X_train_fold, X_test_fold, y_train_fold, y_test_fold):
    avg_expected_loss, avg_bias, avg_var = bias_variance_decomp(
        estimator1, X_train_fold, y_train_fold, X_test_fold, y_test_fold,
        loss='mse', random_seed=0
    )
    return avg_expected_loss

score = make_scorer(custom_loss_func, greater_is_better=False).set_score_request(
    estimator1=True, X_train_fold=True, X_test_fold=True, y_train_fold=True, y_test_fold=True
)

GridSearchCV配置 #

knn = KNeighborsRegressor()
param_grid = dict(
    n_neighbors=np.arange(1,21,1),
    weights=['uniform', 'distance'],
    p=np.arange(1,4,1)
)
scoring_metrics = {
    'neg_root_mean_squared_error': 'neg_root_mean_squared_error',
    'r2': 'r2',
    'score_func': score
}

grid = GridSearchCV(
    knn, param_grid, cv=custom_iterator,
    scoring=scoring_metrics, refit=score, verbose=2, n_jobs=-1
)

核心问题：如何在每次交叉验证迭代中，将X_train_fold, X_test_fold, y_train_fold, y_test_fold, estimator1传递给自定义评分函数？已尝试用set_score_request声明参数，但不知道如何在迭代中传递，是否可以通过Metadata Routing实现？

解决方案 #

1. 遵循sklearn CV接口规范 #

sklearn的CV迭代器必须返回(train_index, test_index)，而非直接返回fold数据。我们可以直接使用PredefinedSplit作为cv参数，无需自定义迭代器。

2. 利用Metadata Routing传递元数据 #

sklearn的Metadata Routing允许评分函数请求额外的上下文数据（如当前估计器、训练/测试索引、完整训练集等）。调整评分函数，通过索引提取当前fold的数据：

修改自定义评分函数

def custom_loss_func(y_true, y_pred, estimator, X, y, train_indices, test_indices):
    # 通过索引提取当前fold的训练/测试数据
    X_train_fold = X[train_indices]
    y_train_fold = y[train_indices]
    X_test_fold = X[test_indices]
    y_test_fold = y[test_indices]
    
    # 克隆估计器用于偏差-方差分解（避免影响GridSearchCV中的拟合实例）
    cloned_estimator = clone(estimator)
    avg_expected_loss, _, _ = bias_variance_decomp(
        cloned_estimator, X_train_fold, y_train_fold, X_test_fold, y_test_fold,
        loss='mse', random_seed=0
    )
    return avg_expected_loss

# 声明评分函数需要的元数据
score = make_scorer(custom_loss_func, greater_is_better=False).set_score_request(
    estimator=True, X=True, y=True, train_indices=True, test_indices=True
)

配置并运行GridSearchCV

# 初始化PredefinedSplit
X_train_len = int(round(len(X_train_knn)*0.8, 0))
test_fold = np.hstack([-1 * np.ones(X_train_len), np.zeros(len(X_train_knn)-X_train_len)])
ps = PredefinedSplit(test_fold)

# 启用估计器的Metadata Routing
knn = KNeighborsRegressor()
knn.set_fit_request(X=True, y=True)
knn.set_score_request(X=True, y=True)

# 配置GridSearchCV
param_grid = dict(
    n_neighbors=np.arange(1,21,1),
    weights=['uniform', 'distance'],
    p=np.arange(1,4,1)
)
scoring_metrics = {
    'neg_root_mean_squared_error': 'neg_root_mean_squared_error',
    'r2': 'r2',
    'score_func': score
}

grid = GridSearchCV(
    knn, param_grid, cv=ps,
    scoring=scoring_metrics, refit='score_func', verbose=2, n_jobs=-1
)

# 执行网格搜索
grid.fit(X_train_knn, Y_train_ar)

关键说明 #

• 无需自定义迭代器：PredefinedSplit已经满足交叉验证划分需求，GridSearchCV会自动处理索引传递。
• 元数据请求：通过set_score_request声明评分函数需要的参数，GridSearchCV会自动在每次迭代中传递这些元数据。
• 克隆估计器：bias_variance_decomp需要多次训练模型，克隆当前参数的估计器可以避免干扰GridSearchCV内部的拟合流程。

内容的提问来源于stack exchange，提问作者Brian