保存BERT类模型时Transformers报错：method无法JSON序列化

TypeError Traceback (most recent call last)
Cell In[15], line 1
----> 1 trainer.train()
2 trainer.save_model(f"astrobert-output/ft-{model_name}-{run_name}-final")

File ~/miniconda3/envs/paper-class/lib/python3.11/site-packages/transformers/trainer.py:1624, in Trainer.train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)
1622 hf_hub_utils.enable_progress_bars()
1623 else:
-> 1624 return inner_training_loop(
1625 args=args,
1626 resume_from_checkpoint=resume_from_checkpoint,
1627 trial=trial,
1628 ignore_keys_for_eval=ignore_keys_for_eval,
1629 )

File ~/miniconda3/envs/paper-class/lib/python3.11/site-packages/transformers/trainer.py:2029, in Trainer._inner_training_loop(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)
2026 self.state.epoch = epoch + (step + 1 + steps_skipped) / steps_in_epoch
2027 self.control = self.callback_handler.on_step_end(args, self.state, self.control)
-> 2029 self._maybe_log_save_evaluate(tr_loss, grad_norm, model, trial, epoch, ignore_keys_for_eval)
2030 else:
2031 self.control = self.callback_handler.on_substep_end(args, self.state, self.control)

File ~/miniconda3/envs/paper-class/lib/python3.11/site-packages/transformers/trainer.py:2423, in Trainer._maybe_log_save_evaluate(self, tr_loss, grad_norm, model, trial, epoch, ignore_keys_for_eval)
2420 self.lr_scheduler.step(metrics[metric_to_check])
2422 if self.control.should_save:
-> 2423 self._save_checkpoint(model, trial, metrics=metrics)
2424 self.control = self.callback_handler.on_save(self.args, self.state, self.control)

File ~/miniconda3/envs/paper-class/lib/python3.11/site-packages/transformers/trainer.py:2499, in Trainer._save_checkpoint(self, model, trial, metrics)
2497 else:
2498 staging_output_dir = os.path.join(run_dir, f"tmp-{checkpoint_folder}")
-> 2499 self.save_model(staging_output_dir, _internal_call=True)
2501 if not self.args.save_only_model:
2502 # Save optimizer and scheduler
2503 self._save_optimizer_and_scheduler(staging_output_dir)

File ~/miniconda3/envs/paper-class/lib/python3.11/site-packages/transformers/trainer.py:3016, in Trainer.save_model(self, output_dir, _internal_call)
3013 self.model_wrapped.save_checkpoint(output_dir)
3015 elif self.args.should_save:
-> 3016 self._save(output_dir)
3018 # Push to the Hub when save_model is called by the user.
3019 if self.args.push_to_hub and not _internal_call:

File ~/miniconda3/envs/paper-class/lib/python3.11/site-packages/transformers/trainer.py:3094, in Trainer._save(self, output_dir, state_dict)
3089 self.model.save_pretrained(
3090 output_dir, state_dict=state_dict, safe_serialization=self.args.save_safetensors
3091 )
3093 if self.tokenizer is not None:
-> 3094 self.tokenizer.save_pretrained(output_dir)
3096 # Good practice: save your training arguments together with the trained model
3097 torch.save(self.args, os.path.join(output_dir, TRAINING_ARGS_NAME))

File ~/miniconda3/envs/paper-class/lib/python3.11/site-packages/transformers/tokenization_utils_base.py:2464, in PreTrainedTokenizerBase.save_pretrained(self, save_directory, legacy_format, filename_prefix, push_to_hub, **kwargs)
2462 print(" ")
2463 with open(tokenizer_config_file, "w", encoding="utf-8") as f:
-> 2464 out_str = json.dumps(tokenizer_config, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
2465 f.write(out_str)
2466 logger.info(f"tokenizer config file saved in {tokenizer_config_file}")

File ~/miniconda3/envs/paper-class/lib/python3.11/json/init.py:238, in dumps(obj, skipkeys, ensure_ascii, check_circular, allow_nan, cls, indent, separators, default, sort_keys, **kw)
232 if cls is None:
233 cls = JSONEncoder
234 return cls(
235 skipkeys=skipkeys, ensure_ascii=ensure_ascii,
236 check_circular=check_circular, allow_nan=allow_nan, indent=indent,
237 separators=separators, default=default, sort_keys=sort_keys,
--> 238 **kw).encode(obj)

File ~/miniconda3/envs/paper-class/lib/python3.11/json/encoder.py:202, in JSONEncoder.encode(self, o)
200 chunks = self.iterencode(o, _one_shot=True)
201 if not isinstance(chunks, (list, tuple)):
-> 202 chunks = list(chunks)
203 return ''.join(chunks)

File ~/miniconda3/envs/paper-class/lib/python3.11/json/encoder.py:432, in _make_iterencode.._iterencode(o, _current_indent_level)
430 yield from _iterencode_list(o, _current_indent_level)
431 elif isinstance(o, dict):
-> 432 yield from _iterencode_dict(o, _current_indent_level)
433 else:
434 if markers is not None:

File ~/miniconda3/envs/paper-class/lib/python3.11/json/encoder.py:406, in _make_iterencode.._iterencode_dict(dct, _current_indent_level)
404 else:
405 chunks = _iterencode(value, _current_indent_level)
-> 406 yield from chunks
407 if newline_indent is not None:
408 _current_indent_level -= 1

File ~/miniconda3/envs/paper-class/lib/python3.11/json/encoder.py:439, in _make_iterencode.._iterencode(o, _current_indent_level)
437 raise ValueError("Circular reference detected")
438 markers[markerid] = o
-> 439 o = _default(o)
440 yield from _iterencode(o, _current_indent_level)
441 if markers is not None:

File ~/miniconda3/envs/paper-class/lib/python3.11/json/encoder.py:180, in JSONEncoder.default(self, o)
161 def default(self, o):
162 """Implement this method in a subclass such that it returns
163 a serializable object for o, or calls the base implementation
164 (to raise a TypeError).
(...)
178
179 """
-> 180 raise TypeError(f'Object of type {o.class.name} '
181 f'is not JSON serializable')

TypeError: Object of type method is not JSON serializable

最小复现代码（不含数据）：
```python
model_checkpoint = "adsabs/astroBERT"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, add_special_tokens=True, do_lower_case=False, use_fast=False)
model = AutoModelForSequenceClassification.from_pretrained(model_checkpoint, problem_type="multi_label_classification", num_labels=num_labels, id2label=id2label, label2id=label2id)

trainer = Trainer(
    model=model,
    train_dataset=encoded_dataset["train"],
    eval_dataset=encoded_dataset["valid"],
    tokenizer=tokenizer)

trainer.train()
trainer.save_model(f"model")

经排查，transformers库tokenization_utils_base.py中，tokenizer_config的add_special_tokens属性为<class 'method'>类型，而非初始化时设置的布尔值True/False。使用transformers 4.38.1版本，询问错误原因及是否为库的bug。

原因与解决方案 #

原因 #

这是transformers库的属性名冲突bug，触发逻辑如下：

• add_special_tokens既是tokenizer初始化时的布尔型参数，同时也是tokenizer类的内置方法名
• 部分自定义预训练模型（如astroBERT）的原始配置文件中，错误地将add_special_tokens保留为方法引用而非参数值
• transformers 4.38.x版本在序列化tokenizer配置时未做严格类型校验，导致方法对象无法被JSON序列化，最终抛出报错

解决方案 #

提供三种可行的修复方式：

1. 手动修正tokenizer配置
   在初始化tokenizer后，强制将配置中的add_special_tokens覆盖为布尔值：

tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, add_special_tokens=True, do_lower_case=False, use_fast=False)
# 修正init_kwargs中的属性
tokenizer.init_kwargs['add_special_tokens'] = True
# 同时修正tokenizer_config中的对应属性
tokenizer.tokenizer_config['add_special_tokens'] = True

2. 跳过tokenizer保存（临时方案）
   如果无需保存tokenizer，可通过两种方式实现：

# 方式1：Trainer初始化时不传入tokenizer参数
trainer = Trainer(
    model=model,
    train_dataset=encoded_dataset["train"],
    eval_dataset=encoded_dataset["valid"])

# 方式2：调用save_model时指定仅保存模型
trainer.save_model(f"model", save_only_model=True)

3. 升级transformers版本
   该bug在transformers 4.40及以上版本中已被官方修复，直接升级库即可解决：

pip install --upgrade transformers

内容的提问来源于stack exchange，提问作者suse