能否用不同行数的多CSV文件构建Python CNN模型？求实现技巧

这种数据形式完全可以用来训练CNN模型，核心是先把数据处理成CNN能接受的固定维度张量，再结合适配的网络结构完成训练。以下是具体实现思路和两个框架的代码示例：

一、核心预处理技巧 #

• 统一输入尺寸：CNN要求输入维度一致，针对33行的短样本用零填充、均值填充或插值补全到70行；70行的长样本直接保留，或者根据数据特性截断到固定长度（比如取前/后33行，不过更建议统一到较长的70行）。
• 重塑数据维度：把每个CSV文件的(行数,列数)数据转成(行数,列数,1)（TensorFlow）或(1,行数,列数)（PyTorch）的单通道“伪图像”格式——相当于把表格数据当成一维序列/单通道图像来处理，适配CNN的输入要求。
• 分层划分数据集：因为只有100个样本，按6个类别分层拆分训练/验证/测试集，避免类别分布不均影响模型效果。

二、网络结构适配建议 #

• 优先选择一维CNN：你的数据是序列型（行是序列步，列是特征），一维CNN(Conv1d)比二维CNN更贴合数据特性，计算效率也更高。
• 用全局池化层做过渡：全局平均/最大池化层可以把任意长度的特征序列压缩成固定维度的向量，再接全连接层做分类，能降低预处理维度限制的影响。
• 加入正则化组件：样本量较小，务必加入Dropout和BatchNorm层防止过拟合。

三、TensorFlow/Keras实现示例 #

import tensorflow as tf
from tensorflow.keras import layers, models
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder

# 加载并预处理单个CSV文件
def process_csv(file_path, target_len=70):
    df = pd.read_csv(file_path)
    data = df.values
    # 截断或填充到目标长度
    if data.shape[0] > target_len:
        data = data[:target_len, :]
    else:
        pad_len = target_len - data.shape[0]
        data = np.pad(data, ((0, pad_len), (0, 0)), mode='constant')
    # 转成(70,141,1)的单通道格式
    return data[..., np.newaxis]

# 替换成你的文件路径和对应类别标签
file_paths = ["file1.csv", "file2.csv", ...]
labels = ["classA", "classB", ...]

# 编码类别标签
le = LabelEncoder()
encoded_labels = le.fit_transform(labels)

# 加载所有数据
X = np.array([process_csv(fp) for fp in file_paths])
y = encoded_labels

# 分层划分数据集
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, stratify=y)

# 构建一维CNN模型
model = models.Sequential([
    layers.Conv1D(32, kernel_size=3, activation='relu', input_shape=(70, 141)),
    layers.MaxPooling1D(pool_size=2),
    layers.Conv1D(64, kernel_size=3, activation='relu'),
    layers.GlobalAveragePooling1D(),
    layers.Dense(64, activation='relu'),
    layers.Dropout(0.5),
    layers.Dense(6, activation='softmax')
])

model.compile(optimizer='adam',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

# 启动训练
history = model.fit(X_train, y_train,
                    epochs=50,
                    batch_size=8,
                    validation_data=(X_val, y_val))

四、PyTorch实现示例 #

import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder

# 自定义数据集类
class CSVDataset(Dataset):
    def __init__(self, file_paths, labels, target_len=70):
        self.file_paths = file_paths
        self.labels = labels
        self.target_len = target_len

    def __len__(self):
        return len(self.file_paths)

    def __getitem__(self, idx):
        df = pd.read_csv(self.file_paths[idx])
        data = df.values.astype(np.float32)
        # 截断或填充到目标长度
        if data.shape[0] > self.target_len:
            data = data[:self.target_len, :]
        else:
            pad_len = self.target_len - data.shape[0]
            data = np.pad(data, ((0, pad_len), (0, 0)), mode='constant')
        # 转成(1,70,141)格式适配PyTorch的输入要求
        data = torch.tensor(data).unsqueeze(0)
        label = torch.tensor(self.labels[idx], dtype=torch.long)
        return data, label

# 替换成你的文件路径和类别标签
file_paths = ["file1.csv", "file2.csv", ...]
labels = ["classA", "classB", ...]

# 编码标签
le = LabelEncoder()
encoded_labels = le.fit_transform(labels)

# 分层划分数据集
train_paths, val_paths, train_labels, val_labels = train_test_split(file_paths, encoded_labels, test_size=0.2, stratify=encoded_labels)

# 创建DataLoader
train_dataset = CSVDataset(train_paths, train_labels)
val_dataset = CSVDataset(val_paths, val_labels)

train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=8)

# 构建一维CNN模型
class CNNClassifier(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv1d(in_channels=141, out_channels=32, kernel_size=3)
        self.pool = nn.MaxPool1d(2)
        self.conv2 = nn.Conv1d(32, 64, kernel_size=3)
        self.global_pool = nn.AdaptiveAvgPool1d(1)
        self.fc1 = nn.Linear(64, 64)
        self.dropout = nn.Dropout(0.5)
        self.fc2 = nn.Linear(64, 6)

    def forward(self, x):
        # 调整维度适配Conv1D输入：(batch,1,70,141) -> (batch,141,70)
        x = x.squeeze(1).transpose(1, 2)
        x = torch.relu(self.conv1(x))
        x = self.pool(x)
        x = torch.relu(self.conv2(x))
        x = self.global_pool(x).squeeze(-1)
        x = torch.relu(self.fc1(x))
        x = self.dropout(x)
        x = self.fc2(x)
        return x

# 训练配置
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = CNNClassifier().to(device)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)

# 训练循环
for epoch in range(50):
    model.train()
    train_loss = 0.0
    for data, labels in train_loader:
        data, labels = data.to(device), labels.to(device)
        optimizer.zero_grad()
        outputs = model(data)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()
        train_loss += loss.item() * data.size(0)
    
    # 验证阶段
    model.eval()
    val_loss = 0.0
    correct = 0
    total = 0
    with torch.no_grad():
        for data, labels in val_loader:
            data, labels = data.to(device), labels.to(device)
            outputs = model(data)
            loss = criterion(outputs, labels)
            val_loss += loss.item() * data.size(0)
            _, predicted = torch.max(outputs.data, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()
    
    # 打印 epoch 结果
    print(f'Epoch {epoch+1}: Train Loss: {train_loss/len(train_loader.dataset):.4f}, Val Loss: {val_loss/len(val_loader.dataset):.4f}, Val Acc: {correct/total:.4f}')

内容的提问来源于stack exchange，提问作者gvsa