分离自动作文评分系统训练与部署，避免重复训练耗时

我明白你的痛点——每次运行都要从头跑一遍预处理和训练，耗时太长。问题出在你之前只保存了评分结果，却没把训练好的模型、预处理用到的关键工具（比如文本清洗的vectorizer）这些能复用的对象存下来。下面是具体的解决方案，帮你实现首次训练、后续直接加载查看结果的需求：

核心思路 #

1. 拆分代码模块：把训练流程和结果查看/预测流程分开
2. 完整保存复用对象：用pickle（或joblib，适合大模型）保存所有需要复用的内容：
   • 每个作文集对应的训练好的模型（线性回归、Ridge、Lasso）
   • 预处理用的文本清洗工具（vectorizer）
   • 评分结果数据框
3. 添加判断逻辑：运行时先检查是否有已保存的模型文件，没有才执行训练，否则直接加载展示

修改后的完整代码 #

import pickle
import os
import pandas as pd
import numpy as np
from sklearn import linear_model
# 导入你项目中的自定义模块（util、Perplexity等）

def train_and_save_model():
    print("Starting training process...")
    # -------------------------- 数据预处理部分 --------------------------
    print("Fetching data...") 
    train_df = util.get_training_data('../data/training_set_rel3.tsv') 
    valid_df = util.get_validation_data('../data/valid_set.tsv') 

    print("Standardizing scores...") 
    train_df, valid_df = util.append_standardized_column(train_df, valid_df, 'score') 

    print("Calculating perplexity feature...") 
    train_df, valid_df = Perplexity().fill_perplexity_columns(train_df, valid_df) 

    print("Calculating number of sentences feature...") 
    train_df, valid_df = fill_sentence_column(train_df, valid_df) 

    print("Cleaning for spelling and word count...") 
    vectorizer_train_spelling = util.vectorizer_clean_spelling(train_df) 
    train_essays_spelling = vectorizer_train_spelling['essay'].values 
    vectorizer_valid_spelling = util.vectorizer_clean_spelling(valid_df) 
    valid_essays_spelling = vectorizer_valid_spelling['essay'].values 

    print("Calculating total words feature...") 
    train_df, valid_df = fill_total_words_column(train_df, valid_df, train_essays_spelling, valid_essays_spelling) 

    print("Calculating unique words feature...") 
    train_df, valid_df = fill_unique_words_column(train_df, valid_df, train_essays_spelling, valid_essays_spelling) 

    print("Calculating spelling feature...") 
    train_df, valid_df = fill_spelling_column(train_df, valid_df, train_essays_spelling, valid_essays_spelling) 

    print("Calculating pos tags features...") 
    train_df, valid_df = fill_pos_columns(train_df, valid_df) 

    print("Cleaning for TFIDF...") 
    vectorizer_train = util.vectorizer_clean(train_df) 
    train_essays = vectorizer_train['essay'].values 
    vectorizer_valid = util.vectorizer_clean(valid_df) 
    valid_essays = vectorizer_valid['essay'].values 

    print("Calculating TFIDF features with unigram...") 
    train_df, valid_df = fill_tfidf_column(train_df, valid_df, train_essays, valid_essays, 1) 

    # 注意：原代码中std_unique_words重复了，这里已经修正
    COLS = ['essay_set', 'spelling_correct', 'std_sentence_count', 'std_unique_words', 'std_total_words', 
            'ADJ', 'ADP', 'ADV', 'CONJ', 'DET', 'NOUN', 'NUM', 'PRT', 'PRON', 'VERB', '.', 'X', 'std_perplexity', 'std_score'] 
    train_df = train_df[COLS].join(train_df.filter(regex=("tfidf_*"))) 
    valid_df = valid_df[COLS].join(valid_df.filter(regex=("tfidf_*"))) 

    # -------------------------- 模型训练部分 --------------------------
    max_essay_set = max(train_df['essay_set'])
    # 用字典存储模型，方便按作文集/alpha索引
    linreg_models = {}
    ridge_models = {}
    lasso_models = {}
    # 评分结果存储
    linreg_scores_df = pd.DataFrame(columns=['essay_set', 'p', 'spearman']) 
    lasso_scores_df = pd.DataFrame(columns=['essay_set', 'alpha', 'p', 'spearman']) 
    ridge_scores_df = pd.DataFrame(columns=['essay_set', 'alpha', 'p', 'spearman']) 
    alphas = [x * 1.0 / 20 for x in range(20, 0, -1)] 

    for i in range(1, max_essay_set + 1): 
        print(f"\nProcessing Essay Set {i}") 
        train_x = np.asarray(train_df[train_df['essay_set'] == i].drop(['essay_set', 'std_score'], axis=1)) 
        train_std_scores = np.asarray(train_df[train_df['essay_set'] == i]['std_score'], dtype=np.float) 
        
        # 线性回归模型训练与存储
        regr = linear_model.LinearRegression(fit_intercept=False, copy_X=False)
        regr.fit(train_x, train_std_scores)
        linreg_models[i] = regr
        
        valid_x = np.asarray(valid_df[valid_df['essay_set'] == i].drop(['essay_set', 'std_score'], axis=1)) 
        valid_pred_std_scores = regr.predict(valid_x) 
        linreg_spear, p = Spearman(a=valid_df[valid_df['essay_set'] == i]["std_score"], b=valid_pred_std_scores) 
        linreg_scores_df = linreg_scores_df.append({'essay_set': i, 'p': p, 'spearman': linreg_spear}, ignore_index=True) 
        print(f"Linear Regression Spearman: {linreg_spear}") 

        # Ridge与Lasso模型训练与存储
        for a in alphas: 
            # Ridge
            ridge = linear_model.Ridge(alpha=a) 
            ridge.fit(train_x, train_std_scores)
            ridge_models[(i, a)] = ridge  # 用tuple做key，区分作文集和alpha
            valid_pred_std_scores_ridge = ridge.predict(valid_x) 
            ridge_spear, p = Spearman(a=valid_df[valid_df['essay_set'] == i]["std_score"], b=valid_pred_std_scores_ridge) 
            ridge_scores_df = ridge_scores_df.append({'essay_set': i, 'alpha': a, 'p': p, 'spearman': ridge_spear}, ignore_index=True) 
            print(f"Ridge (alpha={a}) Spearman: {ridge_spear}") 

            # Lasso
            lasso = linear_model.Lasso(alpha=a) 
            lasso.fit(train_x, train_std_scores)
            lasso_models[(i, a)] = lasso
            valid_pred_std_scores_lasso = lasso.predict(valid_x) 
            lasso_spear, p = Spearman(a=valid_df[valid_df['essay_set'] == i]["std_score"], b=valid_pred_std_scores_lasso) 
            lasso_scores_df = lasso_scores_df.append({'essay_set': i, 'alpha': a, 'p': p, 'spearman': lasso_spear}, ignore_index=True) 
            print(f"Lasso (alpha={a}) Spearman: {lasso_spear}") 

    # -------------------------- 保存所有复用对象 --------------------------
    saved_data = {
        'linreg_models': linreg_models,
        'ridge_models': ridge_models,
        'lasso_models': lasso_models,
        'linreg_scores': linreg_scores_df,
        'ridge_scores': ridge_scores_df,
        'lasso_scores': lasso_scores_df,
        # 保存预处理工具，方便后续处理新作文
        'vectorizer_train_spelling': vectorizer_train_spelling,
        'vectorizer_train': vectorizer_train,
        # 保存特征列信息，确保后续预测时特征对齐
        'feature_columns': train_df.drop(['essay_set', 'std_score'], axis=1).columns.tolist()
    }

    with open('trained_essay_model.pickle', 'wb') as f:
        pickle.dump(saved_data, f)
    print("\nTraining completed! Model saved to trained_essay_model.pickle")
    return saved_data

def load_model_and_show_results():
    print("Loading pre-trained model and results...")
    with open('trained_essay_model.pickle', 'rb') as f:
        saved_data = pickle.load(f)
    
    # 打印评分结果
    print("\n=== Linear Regression Scores ===")
    print(saved_data['linreg_scores'])
    print("\n=== Ridge Regression Scores ===")
    print(saved_data['ridge_scores'])
    print("\n=== Lasso Regression Scores ===")
    print(saved_data['lasso_scores'])
    
    # 可以在这里添加预测新作文的逻辑，比如：
    # new_essay = "Your new essay text here"
    # 用saved_data里的vectorizer做预处理，再用模型预测
    
    return saved_data

def main():
    model_file = 'trained_essay_model.pickle'
    if not os.path.exists(model_file):
        # 首次运行，执行完整训练流程
        saved_data = train_and_save_model()
    else:
        # 直接加载已训练的模型和结果
        saved_data = load_model_and_show_results()

if __name__ == "__main__":
    main()

关键细节说明 #

1. 模型存储方式：用字典存储每个作文集、每个alpha对应的模型，方便后续快速调用
2. 预处理工具保存：如果以后需要对新作文进行评分，必须用训练时的vectorizer做相同的文本清洗，否则特征会不一致
3. 大模型优化：如果你的模型文件很大，可以用joblib代替pickle（只需替换import pickle为import joblib，用joblib.dump/joblib.load即可），joblib更适合存储numpy数组和scikit-learn模型
4. 重复列修正：原代码中COLS列表里std_unique_words重复了，已经在代码中删除了重复项，避免后续数据处理出错

内容的提问来源于stack exchange，提问作者Smriti Shrestha