使用For循环导入推文CSV文件 实现SVM与NaiveBayes标签预测

Alright, let's tackle this problem head-on. You've got a folder full of CSV files with tweet data (some labeled, some not), and you want to loop through each one to run SVM and Naive Bayes for label prediction. Here's a practical, step-by-step solution using Python (the standard tool for this kind of NLP/ML task):

First, we'll pull in all the tools we need for loading data, text processing, model training, and file handling:

import os
import pandas as pd
import re
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.svm import SVC
from sklearn.naive_bayes import MultinomialNB
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import classification_report  # Optional, for model performance checks

This function will handle all the heavy lifting for one file: cleaning data, training models, predicting unlabeled tweets, and saving the results.

def process_single_csv(file_path):
    # Load the CSV file
    df = pd.read_csv(file_path)
    
    # Drop the "位置" column (ignore errors if some files don't have it)
    df = df.drop(columns=["位置"], errors="ignore")
    
    # Split into labeled (has tags) and unlabeled (missing tags) data
    labeled_data = df[df["标签"].notna()]
    unlabeled_data = df[df["标签"].isna()]
    
    # Skip if there's no labeled data to train on
    if labeled_data.empty:
        print(f"⚠️ {os.path.basename(file_path)} has no labeled data — skipping prediction.")
        return df
    
    # Clean tweet text (optional but highly recommended for better model performance)
    def clean_tweet(text):
        text = text.lower()
        text = re.sub(r"http\S+|www\S+|https\S+", "", text, flags=re.MULTILINE)  # Remove URLs
        text = re.sub(r"@\w+|#\w+", "", text)  # Remove mentions and hashtags
        text = re.sub(r"[^a-zA-Z0-9\s]", "", text)  # Remove special characters
        return text
    
    labeled_data["推文正文"] = labeled_data["推文正文"].apply(clean_tweet)
    if not unlabeled_data.empty:
        unlabeled_data["推文正文"] = unlabeled_data["推文正文"].apply(clean_tweet)
    
    # Encode text labels to numerical values (required for ML models)
    label_encoder = LabelEncoder()
    labeled_data["标签_编码"] = label_encoder.fit_transform(labeled_data["标签"])
    
    # Convert tweet text to numerical features using TF-IDF
    tfidf_vectorizer = TfidfVectorizer(stop_words="english", max_features=5000)
    X_features = tfidf_vectorizer.fit_transform(labeled_data["推文正文"])
    y_labels = labeled_data["标签_编码"]
    
    # Split labeled data into training/validation sets (to check model performance)
    X_train, X_val, y_train, y_val = train_test_split(X_features, y_labels, test_size=0.2, random_state=42)
    
    # Train SVM model
    svm_model = SVC(kernel="linear")
    svm_model.fit(X_train, y_train)
    
    # Train Naive Bayes model
    nb_model = MultinomialNB()
    nb_model.fit(X_train, y_train)
    
    # Optional: Print model performance on validation data
    print(f"\n--- Model Performance for {os.path.basename(file_path)} ---")
    print("SVM Results:")
    print(classification_report(y_val, svm_model.predict(X_val), target_names=label_encoder.classes_))
    print("Naive Bayes Results:")
    print(classification_report(y_val, nb_model.predict(X_val), target_names=label_encoder.classes_))
    
    # Predict labels for unlabeled tweets
    if not unlabeled_data.empty:
        unlabeled_features = tfidf_vectorizer.transform(unlabeled_data["推文正文"])
        unlabeled_data["标签_SVM预测"] = label_encoder.inverse_transform(svm_model.predict(unlabeled_features))
        unlabeled_data["标签_NB预测"] = label_encoder.inverse_transform(nb_model.predict(unlabeled_features))
    
    # Merge labeled and predicted data back together
    final_df = pd.concat([labeled_data.drop(columns=["标签_编码"]), unlabeled_data], ignore_index=True)
    
    # Save the processed file (adds "processed_" prefix to original filename)
    output_file = os.path.join(os.path.dirname(file_path), f"processed_{os.path.basename(file_path)}")
    final_df.to_csv(output_file, index=False, encoding="utf-8")
    print(f"✅ Processed file saved to: {output_file}")
    
    return final_df

This function will iterate over every CSV in your target folder and apply the processing function above:

def process_all_csvs(folder_path):
    # Get all CSV files in the folder
    csv_files = [f for f in os.listdir(folder_path) if f.lower().endswith(".csv")]
    
    if not csv_files:
        print("❌ No CSV files found in the specified folder.")
        return
    
    # Process each file one by one
    for file_name in csv_files:
        full_file_path = os.path.join(folder_path, file_name)
        print(f"\nProcessing file: {file_name}")
        try:
            process_single_csv(full_file_path)
        except Exception as e:
            print(f"❌ Error processing {file_name}: {str(e)}")
            continue

# Example usage — replace with your actual folder path
target_folder = "/path/to/your/csv/folder"
process_all_csvs(target_folder)

• Text Cleaning: Tweak the clean_tweet function to match your data (e.g., keep hashtags if they're relevant to your labels).
• Model Tuning: Use GridSearchCV to optimize hyperparameters (like C for SVM or alpha for Naive Bayes) and boost performance.
• Output Format: Adjust the column names or save location in the process_single_csv function to fit your needs.
• Model Selection: If you only need one prediction per tweet, you can remove either the SVM or Naive Bayes code block.

内容的提问来源于stack exchange，提问作者TheClutch01