如何替换transformers的tokenize()与pad_sequence()函数？解决导入错误

问题重现 #

你导入代码如下：

import torch, csv, transformers, random
import torch.nn as nn
from torch.utils.data import Dataset
import torch.optim as optim
import pandas as pd
from transformers import GPT2Tokenizer, GPT2LMHeadModel, tokenize, pad_squences

触发错误：

ImportError                               Traceback (most recent call last)
<ipython-input-35-e04c63220105> in <module>
      4 import torch.optim as optim
      5 import pandas as pd
----> 6 from transformers import GPT2Tokenizer, GPT2LMHeadModel, tokenize, pad_squences

ImportError: cannot import name 'tokenize' from 'transformers' (/usr/local/lib/python3.8/dist-packages/transformers/__init__.py)

你的Dataset类中尝试使用tokenize和pad_sequences的代码：

class RephraseDataset(Dataset):
    def __init__(self, data, tokenizer):
        self.data = data
        self.tokenizer = tokenizer

    def __len__(self):
        return len(self.data)

    def __getitem__(self, index):
        query, rephrases = self.data[index]
        tokenized_query = tokenizer.encode(query, add_special_tokens=True)
        # tokenized_query = tokenize(self.tokenizer, query)
        padded_query = tokenized_query + [tokenizer.pad_token_id] * (max_length - len(tokenized_query))
        # padded_query = pad_sequences(self.tokenizer, r, max_length=128)
        tokenized_rephrases = [tokenize(self.tokenizer, r) for r in rephrases]
        padded_rephrases = [pad_sequences(self.tokenizer, r, max_length=128) for r in tokenized_rephrases]
        return padded_query, padded_rephrases

# Create the dataset
dataset = RephraseDataset(data, tokenizer)

# Create a dataloader
dataloader = torch.utils.data.DataLoader(
    dataset,
    batch_size=32,
    shuffle=True,
)

错误原因 #

1. 不存在的顶层API：transformers库从未提供过可直接导入的tokenize函数，分词操作必须通过初始化后的GPT2Tokenizer实例完成。
2. 拼写错误：你写的pad_squences是拼写错误，正确名称是pad_sequences，且它不属于transformers的顶层导入项，而是属于torch.nn.utils.rnn模块，或是tokenizer实例的内置方法。

解决方法（无需回退版本） #

步骤1：修正导入语句 #

移除错误的tokenize和pad_squences，保留正确导入，若需torch的padding工具可单独导入：

import torch, csv, transformers, random
import torch.nn as nn
from torch.utils.data import Dataset
import torch.optim as optim
import pandas as pd
from transformers import GPT2Tokenizer, GPT2LMHeadModel
from torch.nn.utils.rnn import pad_sequence

步骤2：修改Dataset类的分词与Padding逻辑 #

使用tokenizer实例的方法完成分词，同时规范padding操作（注意GPT2默认无pad_token，需手动设置）：

class RephraseDataset(Dataset):
    def __init__(self, data, tokenizer, max_length=128):
        self.data = data
        self.tokenizer = tokenizer
        self.max_length = max_length
        # 为GPT2 tokenizer设置pad_token（默认无，复用eos_token）
        if self.tokenizer.pad_token is None:
            self.tokenizer.pad_token = self.tokenizer.eos_token

    def __len__(self):
        return len(self.data)

    def __getitem__(self, index):
        query, rephrases = self.data[index]
        
        # 处理query的分词与padding
        tokenized_query = self.tokenizer.encode(
            query,
            add_special_tokens=True,
            truncation=True,
            max_length=self.max_length,
            return_tensors="pt"
        ).squeeze()  # 去掉多余维度
        # 补全padding到指定长度
        pad_len = self.max_length - len(tokenized_query)
        padded_query = torch.cat([tokenized_query, torch.tensor([self.tokenizer.pad_token_id]*pad_len)]) if pad_len >0 else tokenized_query
        
        # 处理rephrases的分词与padding
        padded_rephrases = []
        for r in rephrases:
            tok_r = self.tokenizer.encode(
                r,
                add_special_tokens=True,
                truncation=True,
                max_length=self.max_length,
                return_tensors="pt"
            ).squeeze()
            pad_len_r = self.max_length - len(tok_r)
            pad_r = torch.cat([tok_r, torch.tensor([self.tokenizer.pad_token_id]*pad_len_r)]) if pad_len_r>0 else tok_r
            padded_rephrases.append(pad_r)
        
        # 转换为tensor格式方便批量处理
        padded_rephrases = torch.stack(padded_rephrases)
        return padded_query, padded_rephrases

补充说明 #

如果想进一步简化代码，可使用transformers提供的DataCollatorForLanguageModeling自动处理批量数据的padding，无需手动在Dataset中补全padding逻辑。

内容的提问来源于stack exchange，提问作者mchd