字符级深度LSTM训练后输出乱码，求故障排查方案

受Karpathy关于RNN的博客启发，我用PyTorch实现了一个深度LSTM并在莎士比亚语料上训练。CPU上训练近300轮（约3小时）后，损失已趋于平稳，但模型输出仍为乱码，采样示例如下：

Sampling with input: V
VoODRtOvInl:
ede n'cu cna:dnc
tsehFfisHy:cor.lsshal,d La:LCNmse o aLuIs,OLESav
wl ieomiintuebdh ieg.ro CAaadldm ULcSoi.,yaO
lteoorrEua huagu.o SIapShy:tTo,ilcaG doII NcIr:luP oAnaddsgwSoucc:gWa Whotcd
yklvebf yhoROcaSSatakMkTn tiLLyl swbytas dht uMt

目前怀疑问题根源：

• 采样方法错误
• 数据处理存在问题
• 训练时长不足

class DeepLSTM(nn.Module):
    def __init__(self, input_size, hidden_size, num_layers, sequence_size, output_size):
        super(DeepLSTM, self).__init__()
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.num_layers = num_layers
        self.sequence_size = sequence_size
        self.output_size = output_size

        self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)
        self.fc = nn.Linear(hidden_size, output_size)

    def forward(self, inputs):
        batch_size = inputs.size(0)
        
        # store the output from the fully-connected layer (batch_size x output_size x sequence_size)
        logits = torch.empty((batch_size, self.output_size, self.sequence_size))
        
        h = torch.zeros((self.num_layers, batch_size, self.hidden_size))
        c = torch.zeros((self.num_layers, batch_size, self.hidden_size))

        output, (h, c) = self.lstm(x, (h, c))
        for seq in range(self.sequence_size):
            logits[:, :, seq] = self.fc(output[:, seq, :])
        
        return logits
    
    def sample(self, x, num_samples, index_to_char):
        chars = [index_to_char[torch.argmax(x).item()]]
        
        # Initialize hidden states and cell states
        h = torch.zeros((self.num_layers, self.hidden_size))
        c = torch.zeros((self.num_layers, self.hidden_size))

        for _ in range(num_samples):
            output, (h, c) = self.lstm(x, (h, c))
            logits = self.fc(torch.flatten(output))
            probabilities = torch.sigmoid(logits)
            idx = torch.multinomial(probabilities, num_samples=1)
            chars.append(index_to_char[idx.item()])
            x = nn.functional.one_hot(idx, self.output_size).to(torch.float32)
            
        return ''.join(chars)

def get_vocabulary_and_mappings (chars):
    vocabulary = list(set(chars))
    vocabulary.sort()
    print("Vocabulary size: ", len(vocabulary))
    
    index_to_char = {}
    char_to_index = {}
    
    for idx, char in enumerate(vocabulary):
        index_to_char[idx] = char
        char_to_index[char] = idx
    
    return vocabulary, index_to_char, char_to_index

def get_batch(chars, char_to_index, V, sequence_size, batch_size):
    x = torch.empty((batch_size, sequence_size, V))
    y = torch.empty((batch_size, V, sequence_size))
    
    chars = torch.tensor([char_to_index[char] for char in chars])

    batch_id = 0
    sequence_id = 0
    num_chars = len(chars)
    num_sequences = (num_chars-1) // sequence_size
    
    for i in range(num_sequences):
        start_idx = i * sequence_size
        end_idx = start_idx + sequence_size
        
        chars_in = chars[start_idx : end_idx]
        chars_out = chars[start_idx + 1 : end_idx + 1]
        
        x[batch_id] = nn.functional.one_hot(chars_in, V)
        y[batch_id] = nn.functional.one_hot(chars_out, V).T
        
        batch_id += 1
        
        if batch_id == batch_size:
            batch_id = 0
            yield x, y
    
    last_batch_size = num_sequences % batch_size
    if last_batch_size != 0:
        yield x[:last_batch_size], y[:last_batch_size]

chars = [char for char in corpus if char.isalpha() or char in string.punctuation or char =='\n' or char == ' ']
vocabulary, index_to_char, char_to_index = get_vocabulary_and_mappings(chars)
V = len(vocabulary)

input_size = V
hidden_size = 128
num_layers = 3
sequence_size = 25
output_size = V
batch_size = 64

epochs = 1000
print_percentage = 20
sample_percentage = 25
num_sample = 250

model = DeepLSTM(input_size, hidden_size, num_layers, sequence_size, output_size)

criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.01)

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device)

for epoch in range(1, epochs + 1):
    loss = None
    
    for x, y_true in get_batch(chars, char_to_index, V, sequence_size, batch_size):
        x = x.to(device)
        y_true = y_true.to(device)
        
        # forward
        logits = model(x)
        loss = criterion(logits, y_true)
        
        # backward
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        
    # print loss
    print(f"Epoch: {epoch} -- loss: {loss:.6f}")
        
    # sample
    if epoch % sample_percentage == 0:
        char_x = torch.randint(V, (1,))
        sample_x = nn.functional.one_hot(char_x, V).to(torch.float32)
        print("Sampling with input: ", index_to_char[char_x.item()])
        sample_y = model.sample(sample_x, 250, index_to_char)

        print(sample_y)

1. 采样方法核心错误 #

• 激活函数误用：多分类任务需用softmax而非sigmoid，sigmoid仅适用于二分类，softmax能保证概率和为1，确保multinomial采样逻辑合理。
• 输入维度不匹配：LSTM输入需符合(batch_size, seq_len, input_size)（batch_first=True时），当前采样的x缺少序列维度，需调整为(1,1,V)。
• 设备不匹配：若用GPU训练，采样时初始化的隐藏状态仍在CPU，会导致报错，需将隐藏状态移至模型所在设备。

修改后的sample方法：

def sample(self, x, num_samples, index_to_char, device):
    chars = [index_to_char[torch.argmax(x).item()]]
    # 调整输入维度适配batch_first=True的LSTM
    x = x.unsqueeze(0).unsqueeze(0).to(device)
    
    # 初始化隐藏状态并移至对应设备
    h = torch.zeros((self.num_layers, 1, self.hidden_size)).to(device)
    c = torch.zeros((self.num_layers, 1, self.hidden_size)).to(device)

    for _ in range(num_samples):
        output, (h, c) = self.lstm(x, (h, c))
        # 取LSTM最后一步输出传入全连接层
        logits = self.fc(output[:, -1, :])
        probabilities = torch.softmax(logits, dim=1)
        idx = torch.multinomial(probabilities, num_samples=1)
        chars.append(index_to_char[idx.item()])
        # 构造下一个输入，调整维度
        x = nn.functional.one_hot(idx, self.output_size).to(torch.float32).unsqueeze(0).to(device)
            
    return ''.join(chars)

2. 模型forward方法错误 #

• 变量名错误：方法参数为inputs，但调用LSTM时用了未定义的x，需改为inputs。
• 设备不匹配：初始化的隐藏状态和logits张量默认在CPU，若模型在GPU上会报错，需与输入张量同设备。

修改后的forward方法：

def forward(self, inputs):
    batch_size = inputs.size(0)
    device = inputs.device
    
    # 在对应设备上初始化logits
    logits = torch.empty((batch_size, self.output_size, self.sequence_size), device=device)
    
    # 初始化隐藏状态并移至对应设备
    h = torch.zeros((self.num_layers, batch_size, self.hidden_size), device=device)
    c = torch.zeros((self.num_layers, batch_size, self.hidden_size), device=device)

    output, (h, c) = self.lstm(inputs, (h, c))
    for seq in range(self.sequence_size):
        logits[:, :, seq] = self.fc(output[:, seq, :])
    
    return logits

3. 数据处理问题 #

• 标签维度不匹配：CrossEntropyLoss要求目标为类别索引（形状(batch_size, sequence_size)），而非one-hot编码（当前(batch_size, V, sequence_size)），否则损失计算逻辑错误。
• 内存浪费：预先创建空张量易导致未初始化数据问题，建议动态构建batch。

修改后的get_batch方法：

def get_batch(chars, char_to_index, V, sequence_size, batch_size):
    chars = torch.tensor([char_to_index[char] for char in chars])
    num_chars = len(chars)
    num_sequences = (num_chars - 1) // sequence_size
    
    batch_x = []
    batch_y = []
    
    for i in range(num_sequences):
        start_idx = i * sequence_size
        end_idx = start_idx + sequence_size
        
        chars_in = chars[start_idx:end_idx]
        chars_out = chars[start_idx+1:end_idx+1]
        
        batch_x.append(nn.functional.one_hot(chars_in, V).float())
        batch_y.append(chars_out)
        
        if len(batch_x) == batch_size:
            yield torch.stack(batch_x), torch.stack(batch_y)
            batch_x = []
            batch_y = []
    
    if batch_x:
        yield torch.stack(batch_x), torch.stack(batch_y)

同时调整训练循环中的损失计算：

# forward
logits = model(x)
# logits形状为(N, C, seq_len)，目标为(N, seq_len)，直接传入即可
loss = criterion(logits, y_true)

4. 超参数优化 #

• 学习率过高：Adam优化器用lr=0.01易导致训练不稳定，建议改为0.001或更小。
• 序列长度较短：25的序列长度难以捕捉莎士比亚文本的长依赖，可尝试增大至50或100。
• 训练轮数：CPU上300轮可能不足，建议改用GPU加速或继续训练。

总结 #

先修复代码中的变量名、设备匹配、激活函数、标签维度等硬bug，再调整超参数，模型输出质量会明显提升。

内容的提问来源于stack exchange，提问作者ndan