如何改进自然语言推理任务中准确率低迷的RNN模型

我是NLP新手，在完成大学课程的自然语言推理（NLI）任务时，基于WNLI数据集实现RNN（实际用GRU）模型，但准确率始终卡在0.5633左右，调整batch size、网络层数、训练轮数、隐藏层尺寸、嵌入维度及学习率后效果没有改善甚至更差。以下是问题排查与修复方案：

原始代码核心问题 #

• 词汇表变量不匹配：定义了word2index_combined但预处理函数中使用未定义的word2index，导致词汇表为空，所有单词映射为0，模型无法学习有效语义。
• 数据集参数缺失：data类初始化需要padding参数，但实例化时未传入，引发参数错误。
• 隐藏状态初始化错误：每次forward随机初始化h_0且未同步设备，GRU默认会自动初始化，手动随机初始化破坏训练稳定性。
• 词汇表构建逻辑错误：三次调用预处理函数重复构建词汇表，测试/验证集未复用训练集词汇，导致OOV问题。
• DataLoader参数错误：指定collate_fn但类未实现该方法，引发属性错误。
• GRU输出处理错误：双向GRU仅取最后一层的单方向输出，丢失一半语义信息。
• 预处理过度：移除所有停用词和标点，丢失否定词、连接词等推理关键信息。

修正后的完整代码 #

import pandas as pd
import torch
import torch.nn as nn
from torch.optim import Adam
from tqdm import tqdm
from torch.utils.data import DataLoader, Dataset
from sklearn.metrics import accuracy_score

from nltk import word_tokenize
from nltk.corpus import stopwords
import string 

# 保留否定类停用词，仅移除标点
s_words = set(stopwords.words('english')) - {"not", "no", "never", "none"}
puncts = string.punctuation

word2index = {}  # 统一词汇表变量

def data_preprocessing(fname, is_train=False):
    data = pd.read_csv(fname, sep='\t')
    sentences_1 = data['sentence1'].tolist()
    sentences_1 = [word_tokenize(s) for s in sentences_1]
    sentences_1 = [[w.lower() for w in sent if w.lower() not in puncts] for sent in sentences_1] 
    sentences_2 = data['sentence2'].tolist()
    sentences_2 = [word_tokenize(s) for s in sentences_2]
    sentences_2 = [[w.lower() for w in sent if w.lower() not in puncts] for sent in sentences_2] 
    labels = data['label'].astype(int).tolist()  
    
    # 仅训练集构建词汇表
    if is_train:
        k = 1
        for sent1, sent2 in zip(sentences_1, sentences_2):
            for w in sent1 + sent2:
                if w not in word2index:
                    word2index[w] = k
                    k += 1       
    return sentences_1, sentences_2, labels

class NLIDataset(Dataset):
    def __init__(self, sent_1, sent_2, labels, padding_len):
        self.padding_len = padding_len
        self.sent_1 = self._pad(sent_1)
        self.sent_2 = self._pad(sent_2)        
        self.labels = labels
        
    def __len__(self):
        return len(self.sent_1)
    
    def __getitem__(self, idx):
        s1 = [word2index.get(word, 0) for word in self.sent_1[idx]]
        s2 = [word2index.get(word, 0) for word in self.sent_2[idx]]
        return torch.tensor(s1), torch.tensor(s2), torch.tensor(self.labels[idx])
    
    def _pad(self, sentences):
        padded_sents = []
        for sent in sentences:
            if len(sent) < self.padding_len:
                padded_sent = sent + [0] * (self.padding_len - len(sent))
            else:
                padded_sent = sent[:self.padding_len]
            padded_sents.append(padded_sent)
        return padded_sents

class BiGRUNLI(nn.Module):
    def __init__(self, vocab_size, hidden_size, emb_dim, num_layers):
        super().__init__()
        self.vocab_size = vocab_size + 1  # 包含padding的0
        self.hidden_size = hidden_size
        self.emb_dim = emb_dim
        self.num_layers = num_layers
        
        self.emb = nn.Embedding(self.vocab_size, self.emb_dim, padding_idx=0)
        self.gru = nn.GRU(self.emb_dim, self.hidden_size, num_layers=self.num_layers, 
                          batch_first=True, bidirectional=True)
        self.fc1 = nn.Linear(hidden_size * 4, hidden_size)
        self.relu = nn.ReLU()
        self.dropout = nn.Dropout(0.3)
        self.fc2 = nn.Linear(hidden_size, 2)
        
    def forward(self, inp_seq1, inp_seq2):
        inp1 = self.emb(inp_seq1)
        inp2 = self.emb(inp_seq2)
        
        # GRU自动初始化隐藏状态
        _, last_hidden1 = self.gru(inp1)
        _, last_hidden2 = self.gru(inp2)
        
        # 拼接双向GRU最后一层的前后向输出
        sent1_rep = torch.cat((last_hidden1[-2], last_hidden1[-1]), dim=1)
        sent2_rep = torch.cat((last_hidden2[-2], last_hidden2[-1]), dim=1)
        
        combined = torch.cat((sent1_rep, sent2_rep), dim=1)
        out = self.fc1(combined)
        out = self.relu(out)
        out = self.dropout(out)
        out = self.fc2(out)
        return out

def validate(model, dev_loader, device):
    label_pred = []
    label_original = []
    model.eval()
    with torch.no_grad():
        for X1, X2, y in dev_loader:
            X1, X2, y = X1.to(device), X2.to(device), y.to(device)
            out = model(X1, X2)
            label = torch.argmax(out, dim=1)
            label_pred.extend(label.cpu().numpy().tolist())
            label_original.extend(y.cpu().numpy().tolist())
    return accuracy_score(label_original, label_pred)   

def train(model, train_loader, dev_loader, device):
    val_accuracy = 0
    criterion = nn.CrossEntropyLoss()
    optimizer = Adam(model.parameters(), lr=1e-4)
    epochs = 20
    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'max', patience=2, factor=0.5)
    
    model.train()
    for e in range(epochs):
        total_loss = 0.0
        for X1, X2, y in tqdm(train_loader):
            X1, X2, y = X1.to(device), X2.to(device), y.to(device)
            optimizer.zero_grad()
            out = model(X1, X2)
            loss = criterion(out, y)
            loss.backward()
            optimizer.step()
            total_loss += loss.item()
        
        print(f"Epoch {e+1} | Train Loss: {total_loss/len(train_loader):.4f}")
        acc = validate(model, dev_loader, device)
        print(f"Dev Accuracy: {acc:.4f}")
        scheduler.step(acc)
        
        if acc > val_accuracy:
            torch.save(model.state_dict(), 'best_bigru_nli.pt')
            val_accuracy = acc
        model.train()     

if __name__ == "__main__":
    # 加载数据，仅训练集构建词汇表
    train_s1, train_s2, train_labels = data_preprocessing('WNLI/train.tsv', is_train=True)
    dev_s1, dev_s2, dev_labels = data_preprocessing('WNLI/dev.tsv')
    test_s1, test_s2, test_labels = data_preprocessing('WNLI/test.tsv')
    
    # 计算合理padding长度
    max_len = max([len(s) for s in train_s1 + train_s2])
    padding_len = min(max_len, 64)
    
    # 初始化数据集
    train_data = NLIDataset(train_s1, train_s2, train_labels, padding_len)
    dev_data = NLIDataset(dev_s1, dev_s2, dev_labels, padding_len)
    test_data = NLIDataset(test_s1, test_s2, test_labels, padding_len)
    
    # 构建DataLoader
    train_loader = DataLoader(train_data, batch_size=32, shuffle=True, drop_last=True)
    dev_loader = DataLoader(dev_data, batch_size=32, shuffle=False)
    test_loader = DataLoader(test_data, batch_size=32, shuffle=False)
    
    # 设备配置
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    vocab_size = len(word2index)
    hidden_size = 256
    emb_dim = 300
    num_layers = 2
    
    model = BiGRUNLI(vocab_size, hidden_size, emb_dim, num_layers).to(device)
    train(model, train_loader, dev_loader, device)

额外优化建议 #

• 使用预训练词嵌入：替换随机初始化的Embedding为GloVe或Word2Vec，提升模型初始性能。
• 添加交互机制：在GRU输出后加入注意力层，让模型关注两个句子的关联部分。
• 检查类别平衡：若WNLI标签分布不平衡，使用加权交叉熵损失优化。

内容的提问来源于stack exchange，提问作者Felix