基于PyTorch的Transformer聊天机器人始终输出‘I Don't Know’求助

我基于Transformer架构搭建聊天机器人，调整过超参数（d_model、dropout等）、模型结构、词汇量（23k→5k）、数据集样本量后，模型仍异常——无论输入什么，输出始终是‘I Don't Know’。以下是相关代码与数据集信息，求技术帮助。

模型代码 #

class Transformer(nn.Module):
    def __init__(self, vocab,
                 d_model=512,
                 n_head=8,
                 num_encoder_layers=6,
                 num_decoder_layers=6,
                 dim_feedforward=2048,
                 dropout=0.1,
                 max_len=15) -> None:
        """Instantiating Transformer class
        Args:
            config (Config): model config, the instance of data_utils.utils.Config
            vocab (Vocabulary): the instance of data_utils.vocab_tokenizer.Vocabulary
        """
        super(Transformer, self).__init__()
        self.vocab = vocab
        d_model = d_model #512
        n_head = n_head #8
        num_encoder_layers = num_encoder_layers #6
        num_decoder_layers = num_decoder_layers #6
        dim_feedforward = dim_feedforward #2048
        dropout = dropout #0.1

        self.input_embedding = Embeddings(vocab, d_model, max_len)

        self.transfomrer = torch.nn.Transformer(d_model=d_model,
                                                nhead=n_head,
                                                num_encoder_layers=num_encoder_layers,
                                                num_decoder_layers=num_decoder_layers,
                                                dim_feedforward=dim_feedforward,
                                                dropout=dropout,
                                                batch_first=True)

        self.proj_vocab_layer = nn.Linear(in_features=d_model,
                                          out_features=len(vocab))

        # https://pytorch.org/docs/stable/_modules/torch/nn/modules/module.html#Module.apply
        self.apply(self._initailze)

    def forward(self, enc_input: torch.Tensor, dec_input: torch.Tensor) -> torch.Tensor:

        x_enc_embed = self.input_embedding(enc_input.long())
        x_dec_embed = self.input_embedding(dec_input.long())

        # Masking
        # tensor([[False, False, False,  True,  ...,  True]])
        src_key_padding_mask = enc_input == self.vocab["<pad>"]
        tgt_key_padding_mask = dec_input == self.vocab["<pad>"]

        memory_key_padding_mask = src_key_padding_mask
        tgt_mask = self.transfomrer.generate_square_subsequent_mask(dec_input.size(1))

        # transformer ref: https://pytorch.org/docs/stable/nn.html#torch.nn.Transformer
        src_key_padding_mask = src_key_padding_mask.type(torch.float)
        tgt_key_padding_mask = tgt_key_padding_mask.type(torch.float)
        memory_key_padding_mask = memory_key_padding_mask.type(torch.float)
        tgt_mask = tgt_mask.type(torch.float).to(device)

        feature = self.transfomrer(src = x_enc_embed,
                                   tgt = x_dec_embed,
                                   src_key_padding_mask = src_key_padding_mask,
                                   tgt_key_padding_mask = tgt_key_padding_mask,
                                   memory_key_padding_mask=memory_key_padding_mask,
                                   tgt_mask = tgt_mask)

        logits = self.proj_vocab_layer(feature)
       
        return logits

    def _initailze(self, layer):
        if isinstance(layer, (nn.Linear)):
            nn.init.kaiming_uniform_(layer.weight)


class Embeddings(nn.Module):
    def __init__(self, vocab, embed_size, max_len):
        super(Embeddings, self).__init__()
        self.token_embedding = TokenEmbedding(vocab_size=len(vocab),
                                              embed_size=embed_size,
                                              pad_id=vocab["<pad>"])

        self.pos_embedding = PositionalEmbedding(d_model=embed_size,
                                                 max_len=max_len+2)

    def forward(self, x):
        token_embed = self.token_embedding(x)
        pos_embed = self.pos_embedding(x)

        # print(x.shape, token_embed.shape, pos_embed.shape)

        return token_embed + pos_embed

class PositionalEmbedding(nn.Module):
    
    def __init__(self, d_model, max_len=512):
        super().__init__()

        # Compute the positional encodings once in log space.
        pe = torch.zeros(max_len, d_model).float()
        pe.require_grad = False

        position = torch.arange(0, max_len).float().unsqueeze(1)
        div_term = (torch.arange(0, d_model, 2).float() * -(math.log(10000.0) / d_model)).exp()

        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)

        pe = pe.unsqueeze(0)
        self.register_buffer('pe', pe)

    def forward(self, x):
        return self.pe[:, :x.size(1)]

训练循环代码 #

d_model = 512  # also tried 32, 65, 128
n_head = 8
num_encoder_layers = 6
num_decoder_layers = num_encoder_layers
dim_feedforward = 2048
dropout = 0.1 # tried 0.5 as well

epochs = 10

transformer = Transformer(word_map,
                 d_model=d_model,
                 n_head=n_head,
                 num_encoder_layers=num_encoder_layers,
                 num_decoder_layers=num_decoder_layers,
                 dim_feedforward=dim_feedforward,
                 dropout=dropout,
                 max_len=15).to(device)

adam_optimizer = torch.optim.Adam(transformer.parameters(), lr=0.00, betas=(0.9, 0.98), eps=1e-9)
transformer_optimizer = AdamWarmup(model_size=512, warmup_steps = 4000, optimizer = adam_optimizer)
criterion = LossWithLS(len(word_map), 0.1)

for epoch in range(epochs):

    transformer.train()
    sum_loss = 0
    count = 0

    for i, (enc_inp, dec_inp, dec_out) in enumerate(train_loader):

        samples = enc_inp.shape[-1]

        # Move to device
        enc_inp = enc_inp.to(device)
        dec_inp, dec_out = dec_inp.to(device), dec_out.to(device)

        # Get the transformer outputs
        out = transformer(enc_inp, dec_inp)

        # Compute the loss
        # print(out.shape, reply_target.shape)
        loss = criterion(out, dec_out)

        # Backprop
        transformer_optimizer.optimizer.zero_grad()
        loss.backward()
        transformer_optimizer.step()

        sum_loss += loss.item() * samples
        # print(loss.item(),  samples)
        count += samples

        if i % (batch_size * 5) == 0:
            # print(loss.item(),  samples)
            print("Epoch [{}][{}/{}]\tLoss: {:.3f}".format(epoch, i, len(train_loader), sum_loss/count))
            getResults(transformer)

    state = {'epoch': epoch, 'transformer': transformer, 'transformer_optimizer': transformer_optimizer}
    # torch.save(state, 'checkpoint_' + str(epoch) + '.pth.tar')

数据集信息 #

• 采用Cornell电影对话数据集，路径：

corpus_movie_conv = '/content/drive/My Drive/AI/cornell movie-dialogs corpus/movie_conversations.txt'
corpus_movie_lines = '/content/drive/My Drive/AI/cornell movie-dialogs corpus/movie_lines.txt'

• 已完成预处理：添加PAD、BOS、EOS标识，尝试过将词汇量从23k调整至5k，也调整过数据集样本量，均无改善。

核心排查与修复建议 #

• 修复学习率：训练代码中Adam优化器的lr=0.00是致命错误，模型完全无法更新参数。将其改为合理值，比如1e-4，或根据warmup策略设置初始学习率。
• 修正掩码格式：PyTorch官方Transformer的src_key_padding_mask、tgt_key_padding_mask等参数要求传入布尔张量（True表示该位置需要被mask），代码中转为float类型会导致掩码失效，删除.type(torch.float)转换。
• 修复初始化方法拼写：_initailze拼写错误（应为_initialize），导致线性层初始化逻辑未执行，修正后确保参数正确初始化。
• 检查生成逻辑：查看getResults函数的解码过程，确认是否存在强制输出‘I Don't Know’的逻辑，或该token在词汇表中是否被错误赋予高概率。
• 验证损失函数：确认LossWithLS的标签平滑实现是否合理，避免因平滑过度导致模型倾向于输出高频无意义token。
• 校验数据加载：检查train_loader输出的enc_inp、dec_inp、dec_out是否符合格式要求，比如dec_inp是否以BOS开头，dec_out是否以EOS结尾，对话样本是否正确配对。

内容的提问来源于stack exchange，提问作者GKV