如何在pandas中基于提取的关键词新增Keywords列

你的原有代码已经完成了全量语料的预处理、TF-IDF词典和转换模型的训练，仅需要调整后续关键词生成逻辑，遍历所有文本批量计算即可，修改后的完整代码如下：

import re
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from nltk.stem import PorterStemmer, WordNetLemmatizer
from nltk.corpus import stopwords
stop_words = set(stopwords.words('english'))

# --- 原有预处理逻辑无需调整 ---
corpus = []
# 替换写死的10124为df长度，适配不同规模的数据集避免索引报错
for i in range(len(df)):
    # 去除标点符号
    text = re.sub('[^a-zA-Z]', ' ', df['text_data'][i])
    # 转换为小写
    text = text.lower()
    # 去除标签
    text = re.sub("</?.*?>", " <> ", text)
    # 字符串切分为单词列表
    text = text.split()
    # 词形还原
    lem = WordNetLemmatizer()
    text = [lem.lemmatize(word) for word in text if not word in stop_words]
    text = " ".join(text)
    corpus.append(text)

cv = CountVectorizer(max_df=0.8, stop_words=stop_words, max_features=10000, ngram_range=(1, 3))
X = cv.fit_transform(corpus)

tfidf_transformer = TfidfTransformer(smooth_idf=True, use_idf=True)
tfidf_transformer.fit(X)
feature_names = cv.get_feature_names()

# --- 工具函数补充topn参数，控制每个文本返回的关键词数量 ---
def sort_coo(coo_matrix):
    tuples = zip(coo_matrix.col, coo_matrix.data)
    return sorted(tuples, key=lambda x: (x[1], x[0]), reverse=True)

def extract_topn_from_vector(feature_names, sorted_items, topn=10):
    """获取排名前n的关键词和对应TF-IDF分数"""
    sorted_items = sorted_items[:topn]
    feature_vals = []
    for idx, score in sorted_items:
        feature_vals.append(feature_names[idx])
    return feature_vals

# --- 批量计算所有文本的关键词 ---
keywords_list = []
for doc in corpus:
    tf_idf_vector = tfidf_transformer.transform(cv.transform([doc]))
    sorted_items = sort_coo(tf_idf_vector.tocoo())
    # 这里调整topn参数可以修改每个文本返回的关键词数量
    keywords = extract_topn_from_vector(feature_names, sorted_items, topn=3)
    # 转换为逗号分隔的字符串，匹配目标DataFrame格式
    keywords_list.append(",".join(keywords))

# 新增Keywords列到原DataFrame
df['Keywords'] = keywords_list

# 查看结果
print(df.head())

补充说明 #

• 如果需要保留关键词的TF-IDF分数，只需要修改extract_topn_from_vector函数的返回逻辑，拼接分数到关键词字符串即可
• 可以根据业务需求调整CountVectorizer的ngram_range参数，控制返回的关键词是单字、词组还是多词短语

内容的提问来源于stack exchange，提问作者technophile_3