Python基于BERT的文本分类代码运行时进度条卡在0%问题求解

核心错误原因 #

• 数据集读取逻辑错误：BERTDataset类的__getitem__方法调用分词器时传入了全量文本集合，而非单条样本文本，导致每次取样本都要处理数万条数据，直接卡住进度条。
• 模型前向传播变量名错误：前向传播输出变量命名为x，但后续计算损失和指标时误用了未定义的outputs变量，会触发运行时异常。
• 分词参数不匹配：bert-base-uncased为全小写预训练模型，do_lower_case设为False会导致分词效果偏差。

修复后的代码 #

## Import Packages
import tez
import torch
import torch.nn as nn
import transformers
from transformers import AdamW, get_linear_schedule_with_warmup
from sklearn import metrics
import pandas as pd

## Create a data loader using a class named BERTDataset 
class BERTDataset:
    def __init__(self, texts, targets, max_len = 64):
        self.texts = texts
        self.targets = targets
        self.tokenizer = transformers.BertTokenizer.from_pretrained(
            "bert-base-uncased", #model name
            do_lower_case = True # 修正：uncased模型需开启小写转换
        )
        self.max_len = max_len
        
    # length function
    def __len__(self):
        return len(self.texts)
    
    def __getitem__(self, idx):
        text = str(self.texts[idx])
        # 修正：传入单条文本text而非全量texts
        inputs = self.tokenizer.encode_plus(
        text,
        None,
        add_special_tokens = True,
        max_length = self.max_len,
        padding = "max_length",
        truncation = True
        )
        resp = {
            "ids": torch.tensor(inputs["input_ids"], dtype = torch.long),
            "mask": torch.tensor(inputs["attention_mask"], dtype = torch.long),
            "token_type_ids": torch.tensor(inputs["token_type_ids"], dtype = torch.long),
            "targets": torch.tensor(self.targets[idx], dtype = torch.float),
            
            ## for multiclass classification, convert change dtype from torch.float to torch.long
            #"targets": torch.tensor(self.targets[idx], dtype = torch.long),
        }
        return resp

## Build the model
class TextModel(tez.Model):
    def __init__(self, num_classes, num_train_steps):
        
        super().__init__()
        self.bert = transformers.BertModel.from_pretrained(
            "bert-base-uncased", return_dict = False
        )
        self.bert_drop = nn.Dropout(0.3)
        self.out = nn.Linear(768, num_classes) # num_classes is 1 or > 1 if it is binary multiclass classification respectively
        self.num_train_steps = num_train_steps
        
        
    # optimizer
    def fetch_optimizer(self):
        opt = AdamW(self.parameters(), lr = 3e-5) # 修正：BERT微调推荐学习率区间2e-5~5e-5
        return opt
    
    # scheduler
    def fetch_scheduler(self):
        sch = get_linear_schedule_with_warmup(
            self.optimizer, num_warmup_steps = 0, num_training_steps = self.num_train_steps        
        )
        return sch
    
    # loss
    def loss(self, outputs, targets):
        return nn.BCEWithLogitsLoss()(outputs, targets.view(-1, 1))
    
        ## include the next line if you have multiclass classification 
        # return nn.CrossEntropyLoss()(outputs, targets)
    
    # calculate accuracy
    def monitor_metrics(self, outputs, targets):
        outputs = torch.sigmoid(outputs).cpu().detach().numpy() >= 0.5
        
       
        targets = targets.cpu().detach().numpy()
        return {"accuracy": metrics.accuracy_score(targets, outputs)}
    
    # forward function
    def forward(self, ids, mask, token_type_ids, targets = None):
        _, x = self.bert(ids, attention_mask = mask, token_type_ids = token_type_ids)
        x = self.bert_drop(x)
        x = self.out(x)
        if targets is not None:
            # 修正：使用正确的输出变量x而非未定义的outputs
            loss = self.loss(x, targets)
            met = self.monitor_metrics(x, targets)
            return x, loss, met
        return x, 0, {} # if there is no target, return 0        


## Read Dataset
def train_model(fold):
    df = pd.read_csv("imdb_folds.csv") # read file
    df_train = df[df.kfold != fold].reset_index(drop=True)
    df_valid = df[df.kfold == fold].reset_index(drop=True)
    
    train_dataset = BERTDataset(df_train.review.values, df_train.sentiment.values)
    valid_dataset = BERTDataset(df_valid.review.values, df_valid.sentiment.values)
    
    # n_train_steps = int(len(df_train) / TRAIN_BS * EPOCHS)
    n_train_steps = int(len(df_train) / 32 * 10)
    model = TextModel(num_classes = 1, num_train_steps = n_train_steps)
    
    es = tez.callbacks.EarlyStopping(monitor = "valid_loss", patience = 3, model_path = "model.bin")
    # 无cuda设备可将device改为"cpu"
    model.fit(
        train_dataset, 
        valid_dataset = valid_dataset, 
        device = "cuda", 
        epochs = 10, 
        train_bs = 32,
        callbacks = [es],
    )
    
    
if __name__=="__main__":
    train_model(fold = 0)

补充说明 #

如果本地无CUDA环境，将model.fit中的device参数改为"cpu"即可正常运行。

内容的提问来源于stack exchange，提问作者William