威斯康星乳腺癌数据集特征选择遇IndexError问题求助

问题场景 #

处理Kaggle威斯康星乳腺癌数据集时，移除ID列后剩余31列（目标变量在第0列，特征列共30列，索引0-29），运行PSO+GA混合特征选择代码时，抛出错误：

IndexError: index 30 is out of bounds for axis 1 with size 30

用户代码 #

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split, StratifiedKFold
from imblearn.over_sampling import ADASYN
from sklearn.preprocessing import LabelEncoder, StandardScaler
from keras.models import Sequential
from keras.layers import Conv1D, MaxPooling1D, Flatten, Dense
from keras.callbacks import EarlyStopping, History
from sklearn.metrics import confusion_matrix, matthews_corrcoef, f1_score, recall_score
import matplotlib.pyplot as plt
from pyswarms.discrete.binary import BinaryPSO

# Load the breast cancer dataset into a Pandas DataFrame:
data = pd.read_csv("C:/ml_data/breast_cancer_data.csv")

le = LabelEncoder()
data["diagnosis"] = le.fit_transform(data["diagnosis"])

X_train, X_test, y_train, y_test = train_test_split(data.iloc[:, 1:], data.iloc[:, 0], test_size=0.2, random_state=42)

adasyn = ADASYN(random_state=42)
X_train, y_train = adasyn.fit_resample(X_train, y_train)

scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)


def create_model(input_shape):
    model = Sequential()
    model.add(Conv1D(32, kernel_size=3, activation="relu", input_shape=input_shape))
    model.add(MaxPooling1D(pool_size=2))
    model.add(Conv1D(64, kernel_size=3, activation="relu"))
    model.add(MaxPooling1D(pool_size=2))
    model.add(Flatten())
    model.add(Dense(128, activation="relu"))
    model.add(Dense(1, activation="sigmoid"))
    model.compile(optimizer="adam", loss="binary_crossentropy", metrics=["accuracy"])
    return model


def fitness_func(p, X_train, y_train):
    selected_features = np.argwhere(p == 1).flatten()
    print(selected_features.shape)
    if np.sum(p) == 0:
        return 0
    model = create_model((X_train.shape[1], 1))
    cv = StratifiedKFold(n_splits=10, shuffle=True, random_state=42)
    scores = []
    for train_index, val_index in cv.split(X_train, y_train):
        X_train_cv, X_val = X_train[train_index], X_train[val_index]
        y_train_cv, y_val = y_train[train_index], y_train[val_index]
        model.fit(
            X_train_cv[:, selected_features],
            y_train_cv,
            epochs=50,
            batch_size=32,
            verbose=0,
            callbacks=[History(), EarlyStopping(patience=5)],
        )
        scores.append(model.evaluate(X_val[:, selected_features], y_val, verbose=0)[1])
    return np.mean(scores)


def pso_optimizer(X_train, y_train):
    options = {"c1": 0.5, "c2": 0.3, "w": 0.9, "k": 2, "p": 2}
    pso = BinaryPSO(n_particles=50, dimensions=X_train.shape[1], options=options)
    pso.optimize(lambda x: -fitness_func(x, X_train, y_train), iters=50)
    return pso.pos, pso.best_cost


def ga_optimizer(X_train, y_train, pop_size=50, mutation_prob=0.1):
    population = np.random.randint(2, size=(pop_size, X_train.shape[1]))
    best_fitness = fitness_func(population[0], X_train, y_train)
    for i in range(50):
        fitness = np.apply_along_axis(fitness_func, 1, population, X_train, y_train)
        elite = population[np.argsort(fitness)[-1]]
        if fitness.max() > best_fitness:
            best_fitness = fitness.max()
        new_population = np.empty_like(population)
        new_population[0] = elite
        for j in range(1, pop_size):
            parent1 = population[np.random.choice(pop_size)]
            parent2 = population[np.random.choice(pop_size)]
            crossover_point = np.random.randint(X_train.shape[1])
            child = np.concatenate(
                (parent1[:crossover_point], parent2[crossover_point:])
            )
            mutation_mask = np.random.binomial(1, mutation_prob, size=X_train.shape[1])
            child ^= mutation_mask
            new_population[j] = child
        population = new_population
    return elite, best_fitness


def hybrid_optimizer(X_train, y_train):
    pso_pos, pso_best_cost = pso_optimizer(X_train, y_train)
    ga_pos, ga_best_cost = ga_optimizer(X_train, y_train)
    if ga_best_cost > pso_best_cost:
        return ga_pos == 1  # return selected feature mask
    else:
        return pso_pos == 1  # return selected feature mask

selected_feature_mask = hybrid_optimizer(X_train, y_train).flatten()
X_train = X_train[:, selected_feature_mask]
X_test = X_test[:, selected_feature_mask]

cv = StratifiedKFold(n_splits=10, shuffle=True, random_state=42)
scores = []
for train_index, val_index in cv.split(X_train, y_train):
    model = create_model((X_train.shape[1], 1))
    X_train_cv, X_val = X_train[train_index], X_train[val_index]
    y_train_cv, y_val = y_train[train_index], y_train[val_index]
    history = model.fit(
        X_train_cv,
        y_train_cv,
        epochs=50,
        batch_size=32,
        validation_data=(X_val, y_val),
        verbose=0,
        callbacks=[EarlyStopping(patience=5)],
    )
    X_test_selected = X_test[:, selected_feature_mask]  # select only selected features
    y_pred = model.predict_classes(X_test_selected).flatten()
    tn, fp, fn, tp = confusion_matrix(y_test, y_pred).ravel()
    scores.append(
        {
            "MCC": matthews_corrcoef(y_test, y_pred),
            "F1": f1_score(y_test, y_pred),
            "Sensitivity": recall_score(y_test, y_pred),
            "Loss": history.history["loss"][-1],
            "Val Loss": history.history["val_loss"],
            "TN": tn,
            "FP": fp,
            "FN": fn,
            "TP": tp,
        }
    )

完整报错栈 #

IndexError                                Traceback (most recent call last)
Cell In[1], line 104
    101     else:
    102         return pso_pos == 1  # return selected feature mask
---> 104 selected_feature_mask = hybrid_optimizer(X_train, y_train).flatten()
    105 X_train = X_train[:, selected_feature_mask]
    106 X_test = X_test[:, selected_feature_mask]

Cell In[1], line 97, in hybrid_optimizer(X_train, y_train)
     96 def hybrid_optimizer(X_train, y_train):
---> 97     pso_pos, pso_best_cost = pso_optimizer(X_train, y_train)
     98     ga_pos, ga_best_cost = ga_optimizer(X_train, y_train)
     99     if ga_best_cost > pso_best_cost:

Cell In[1], line 68, in pso_optimizer(X_train, y_train)
     66 options = {"c1": 0.5, "c2": 0.3, "w": 0.9, "k": 2, "p": 2}
     67 pso = BinaryPSO(n_particles=50, dimensions=X_train.shape[1], options=options)
---> 68 pso.optimize(lambda x: -fitness_func(x, X_train, y_train), iters=50)
     69 return pso.pos, pso.best_cost

File ~\.conda\envs\tf\lib\site-packages\pyswarms\discrete\binary.py:195, in BinaryPSO.optimize(self, objective_func, iters, n_processes, verbose, **kwargs)
    192 ftol_history = deque(maxlen=self.ftol_iter)
    193 for i in self.rep.pbar(iters, self.name) if verbose else range(iters):
    194     # Compute cost for current position and personal best
---> 195     self.swarm.current_cost = compute_objective_function(
    196         self.swarm, objective_func, pool, **kwargs
    197     )
    198     self.swarm.pbest_pos, self.swarm.pbest_cost = compute_pbest(
    199         self.swarm
    200     )
    201     best_cost_yet_found = np.min(self.swarm.best_cost)

File ~\.conda\envs\tf\lib\site-packages\pyswarms\backend\operators.py:239, in compute_objective_function(swarm, objective_func, pool, **kwargs)
    214 """Evaluate particles using the objective function
    215 
    216 This method evaluates each particle in the swarm according to the objective
   (...)
    236     Cost-matrix for the given swarm
    237 """
    238 if pool is None:
---> 239     return objective_func(swarm.position, **kwargs)
    240 else:
    241     results = pool.map(
    242         partial(objective_func, **kwargs),
    243         np.array_split(swarm.position, pool._processes),
    244     )

Cell In[1], line 68, in pso_optimizer.<locals>.<lambda>(x)
     66 options = {"c1": 0.5, "c2": 0.3, "w": 0.9, "k": 2, "p": 2}
     67 pso = BinaryPSO(n_particles=50, dimensions=X_train.shape[1], options=options)
---> 68 pso.optimize(lambda x: -fitness_func(x, X_train, y_train), iters=50)
     69 return pso.pos, pso.best_cost

Cell In[1], line 54, in fitness_func(p, X_train, y_train)
     51     X_train_cv, X_val = X_train[train_index], X_train[val_index]
     52     y_train_cv, y_val = y_train[train_index], y_train[val_index]
     53     model.fit(
---> 54         X_train_cv[:, selected_features],
     55         y_train_cv,
     56         epochs=50,
     57         batch_size=32,
     58         verbose=0,
     59         callbacks=[History(), EarlyStopping(patience=5)],
     60     )
     61     scores.append(model.evaluate(X_val[:, selected_features], y_val, verbose=0)[1])
     62 return np.mean(scores)

IndexError: index 30 is out of bounds for axis 1 with size 30

错误原因 #

1. PSO批量粒子处理不兼容：pyswarms的optimize方法会将整个粒子群的位置矩阵（形状(50,30)）传入目标函数，但fitness_func仅针对单个粒子（形状(30,)）设计，直接处理矩阵会导致selected_features生成包含30的非法索引。
2. 未做索引边界校验：当粒子的位置向量出现错误索引时，没有过滤超出特征维度的数值，导致切片时越界。
3. Conv1D输入格式不匹配：代码中直接传入二维特征数组，但Conv1D需要三维输入（样本数、特征数、通道数）。

修复方案 #

1. 修改PSO优化器适配批量粒子 #

将pso_optimizer中的目标函数改为批量遍历每个粒子计算适应度：

def pso_optimizer(X_train, y_train):
    options = {"c1": 0.5, "c2": 0.3, "w": 0.9, "k": 2, "p": 2}
    pso = BinaryPSO(n_particles=50, dimensions=X_train.shape[1], options=options)
    
    # 批量处理每个粒子
    def batch_fitness(particles):
        cost_list = []
        for particle in particles:
            cost_list.append(-fitness_func(particle, X_train, y_train))
        return np.array(cost_list)
    
    pso.optimize(batch_fitness, iters=50)
    return pso.pos, pso.best_cost

2. 添加特征索引边界校验并适配Conv1D输入 #

在fitness_func中过滤非法索引，同时修正模型输入格式：

def fitness_func(p, X_train, y_train):
    selected_features = np.argwhere(p == 1).flatten()
    # 过滤超出特征维度的索引
    selected_features = selected_features[selected_features < X_train.shape[1]]
    print(selected_features.shape)
    # 无合法特征时返回最低适应度
    if len(selected_features) == 0:
        return 0
    
    # 模型输入形状改为选中特征的数量
    model = create_model((len(selected_features), 1))
    cv = StratifiedKFold(n_splits=10, shuffle=True, random_state=42)
    scores = []
    for train_index, val_index in cv.split(X_train, y_train):
        X_train_cv, X_val = X_train[train_index], X_train[val_index]
        y_train_cv, y_val = y_train[train_index], y_train[val_index]
        
        # 转换为Conv1D需要的三维格式
        X_train_selected = X_train_cv[:, selected_features].reshape(-1, len(selected_features), 1)
        X_val_selected = X_val[:, selected_features].reshape(-1, len(selected_features), 1)
        
        model.fit(
            X_train_selected,
            y_train_cv,
            epochs=50,
            batch_size=32,
            verbose=0,
            callbacks=[History(), EarlyStopping(patience=5)],
        )
        scores.append(model.evaluate(X_val_selected, y_val, verbose=0)[1])
    return np.mean(scores)

内容的提问来源于stack exchange，提问作者Oluomotehinwa