请求协助优化葡萄酒质量预测ML Pipeline以降低RMSE

问题背景 #

课程任务要求训练并优化机器学习模型以实现最优预测效果，同时设有课内竞赛，模型表现不佳将导致挂科。目前已尝试多种训练方式，但RMSE分数始终不理想，请求协助优化现有机器学习Pipeline，降低RMSE并提升预测性能。

数据集信息 #

数据集描述 #

葡萄酒质量数据集

• 来源：Paulo Cortez（葡萄牙米尼奥大学吉马良斯分校）、A. Cerdeira、F. Almeida、T. Matos和J. Reis（葡萄牙波尔图绿酒地区葡萄栽培委员会(CVRVV)）@2009
• 致谢：感谢上述人员提供数据集

评估标准 #

• 核心评估指标：RMSE
• 提交格式：CSV文件需包含id和quality两列，带表头，示例如下：

id,quality
1,1

数据示例 #

train.csv样本（分隔符为分号）：

fixed acidity;volatile acidity;citric acid;residual sugar;chlorides;free sulfur dioxide;total sulfur dioxide;density;pH;sulphates;alcohol;quality;type
6.6;0.3;0.36;1.2;0.035;43.0;126.0;0.9909;3.01;0.63;11.4;6;white
7.7;0.5;0.26;1.9;0.062;9.0;31.0;0.9966;3.39;0.64;9.6;5;red
8.4;0.5;0.35;2.9;0.076;21.0;127.0;0.9976;3.23;0.63;9.2;5;red

当前使用代码 #

#!pip install optuna xgboost lightgbm catboost scikit-learn pandas numpy

import pandas as pd
import numpy as np
import optuna
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from catboost import CatBoostRegressor
import warnings

warnings.filterwarnings('ignore')

print("Load...")
train_df = pd.read_csv('train.csv', sep=';')
test_df = pd.read_csv('test.csv', sep=';')

train_df['type'] = train_df['type'].map({'white': 0, 'red': 1})
test_df['type'] = test_df['type'].map({'white': 0, 'red': 1})

X = train_df.drop(columns=['quality'])
y = train_df['quality']
test_ids = test_df['id']
X_test = test_df.drop(columns=['id'])

X = X.fillna(X.median())
X_test = X_test.fillna(X.median())

def objective(trial):
    # XGBoost
    xgb_params = {
        'n_estimators': trial.suggest_int('xgb_n_estimators', 300, 800),
        'learning_rate': trial.suggest_float('xgb_learning_rate', 0.01, 0.05, log=True),
        'max_depth': trial.suggest_int('xgb_max_depth', 4, 8),
        'subsample': trial.suggest_float('xgb_subsample', 0.6, 1.0),
        'colsample_bytree': trial.suggest_float('xgb_colsample', 0.6, 1.0),
        'reg_alpha': trial.suggest_float('xgb_alpha', 1e-3, 10.0, log=True),
        'reg_lambda': trial.suggest_float('xgb_lambda', 1e-3, 10.0, log=True),
        'random_state': 42,
        'objective': 'reg:squarederror'
    }
    
    # LightGBM
    lgb_params = {
        'n_estimators': trial.suggest_int('lgb_n_estimators', 300, 800),
        'learning_rate': trial.suggest_float('lgb_learning_rate', 0.01, 0.05, log=True),
        'max_depth': trial.suggest_int('lgb_max_depth', 4, 9),
        'num_leaves': trial.suggest_int('lgb_num_leaves', 15, 63),
        'subsample': trial.suggest_float('lgb_subsample', 0.6, 1.0),
        'colsample_bytree': trial.suggest_float('lgb_colsample', 0.6, 1.0),
        'reg_alpha': trial.suggest_float('lgb_alpha', 1e-3, 10.0, log=True),
        'reg_lambda': trial.suggest_float('lgb_lambda', 1e-3, 10.0, log=True),
        'random_state': 42,
        'verbose': -1
    }
    
    # Mix 3 models
    w_xgb = trial.suggest_float('w_xgb', 0.0, 1.0)
    w_lgb = trial.suggest_float('w_lgb', 0.0, 1.0)
    w_cat = trial.suggest_float('w_cat', 0.0, 1.0)
    
    total_w = w_xgb + w_lgb + w_cat
    if total_w == 0: 
        total_w = 1e-6
    w_xgb, w_lgb, w_cat = w_xgb / total_w, w_lgb / total_w, w_cat / total_w
    
    kf = KFold(n_splits=5, shuffle=True, random_state=42)
    oof_rmse = []
    
    for train_idx, val_idx in kf.split(X, y):
        X_tr, y_tr = X.iloc[train_idx], y.iloc[train_idx]
        X_va, y_va = X.iloc[val_idx], y.iloc[val_idx]
        
        xgb = XGBRegressor(**xgb_params)
        lgb = LGBMRegressor(**lgb_params)
        
        # Set CatBoost 
        cat = CatBoostRegressor(
            iterations=500, learning_rate=0.03, depth=6, 
            l2_leaf_reg=3, random_state=42, verbose=0
        )
        
        xgb.fit(X_tr, y_tr)
        lgb.fit(X_tr, y_tr)
        cat.fit(X_tr, y_tr)
        
        # Predic mix
        pred = (w_xgb * xgb.predict(X_va) + 
                w_lgb * lgb.predict(X_va) + 
                w_cat * cat.predict(X_va))
        
        oof_rmse.append(np.sqrt(mean_squared_error(y_va, pred)))
        
    return np.mean(oof_rmse)

print("Running...")
optuna.logging.set_verbosity(optuna.logging.WARNING) 

study = optuna.create_study(direction='minimize')
study.optimize(objective, n_trials=30, show_progress_bar=True)

print(f"\n DONE! BEST RMSE: {study.best_value:.5f}")

# Retrain
best_params = study.best_params

# Get params for each model
xgb_best = {k.replace('xgb_', ''): v for k, v in best_params.items() if k.startswith('xgb_')}
xgb_best['random_state'] = 42
xgb_best['objective'] = 'reg:squarederror'

lgb_best = {k.replace('lgb_', ''): v for k, v in best_params.items() if k.startswith('lgb_')}
lgb_best['random_state'] = 42
lgb_best['verbose'] = -1

w_xgb, w_lgb, w_cat = best_params['w_xgb'], best_params['w_lgb'], best_params['w_cat']
total_w = w_xgb + w_lgb + w_cat
w_xgb, w_lgb, w_cat = w_xgb / total_w, w_lgb / total_w, w_cat / total_w

print(f"\n  Ensemble -> XGB: {w_xgb:.2f} | LGBM: {w_lgb:.2f} | CAT: {w_cat:.2f}")
print("Make predict in Test with K-Fold Blend...")

# Blend through 5 folds
kf = KFold(n_splits=5, shuffle=True, random_state=42)
test_predictions = np.zeros(len(X_test))

for train_idx, val_idx in kf.split(X, y):
    X_tr, y_tr = X.iloc[train_idx], y.iloc[train_idx]
    
    xgb = XGBRegressor(**xgb_best)
    lgb = LGBMRegressor(**lgb_best)
    cat = CatBoostRegressor(iterations=500, learning_rate=0.03, depth=6, l2_leaf_reg=3, random_state=42, verbose=0)
    
    # Retrain n-1 folds
    xgb.fit(X_tr, y_tr)
    lgb.fit(X_tr, y_tr)
    cat.fit(X_tr, y_tr)
    
    fold_pred = (w_xgb * xgb.predict(X_test) + 
                 w_lgb * lgb.predict(X_test) + 
                 w_cat * cat.predict(X_test))
    
    test_predictions += fold_pred / kf.n_splits

min_quality, max_quality = y.min(), y.max()
test_predictions = np.clip(test_predictions, min_quality, max_quality)

submission = pd.DataFrame({
    'id': test_ids,
    'quality': test_predictions
})

submission.to_csv('submission.csv', index=False)
print("DONE!!!")

请求协助 #

需要优化上述机器学习Pipeline，降低RMSE分数，提升模型预测性能。

内容的提问来源于stack exchange，提问作者Glenn