如何在Huggingface Transformers中合并PEFT模型与基础模型及排障

问题背景 #

尝试将PEFT微调后的模型合并到原始基础模型，但Hugging Face仅输出.safetensors格式的微调权重，合并操作失败。需要明确正确的合并方法，确认是否支持.safetensors格式，以及如何转换为.bin格式。

合并脚本与报错分析 #

合并脚本 #

# merge base + LoRa models and save the model

from peft import AutoPeftModelForCausalLM
from transformers import AutoTokenizer
import sys
import torch

device_map = {"": 0}
lora_dir = "/root/autodl-tmp/tuned_model"
base_model_name = "LLM4Binary/llm4decompile-1.3b-v1.5"
tokenizer = AutoTokenizer.from_pretrained(base_model_name, trust_remote_code=True)
model = AutoPeftModelForCausalLM.from_pretrained(lora_dir, device_map=device_map, torch_dtype=torch.bfloat16)
print(model)

model = model.merge_and_unload()

output_dir = "./output/merged_model"
model.save_pretrained(output_dir)

报错信息 #

Traceback (most recent call last):
  File "/root/miniconda3/envs/llm4decompile/lib/python3.9/site-packages/peft/peft_model.py", line 824, in __getattr__
    return super().__getattr__(name)  # defer to nn.Module's logic
  File "/root/miniconda3/envs/llm4decompile/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1695, in __getattr__
    raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")
AttributeError: 'PeftModelForCausalLM' object has no attribute 'merge_and_unload'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/root/autodl-tmp/merge.py", line 15, in <module>
    model = model.merge_and_unload()
  File "/root/miniconda3/envs/llm4decompile/lib/python3.9/site-packages/peft/peft_model.py", line 828, in __getattr__
    return getattr(self.base_model, name)
  File "/root/miniconda3/envs/llm4decompile/lib/python3.9/site-packages/torch/nn/modules/module.py", line 1695, in __getattr__
    raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")
AttributeError: 'LlamaForCausalLM' object has no attribute 'merge_and_unload'

报错原因 #

1. PEFT方法不支持合并：当前使用的是Prompt Tuning这类无法直接合并到基础模型的适配器，merge_and_unload仅支持LoRA等特定参数高效微调方法。
2. 模型加载逻辑错误：AutoPeftModelForCausalLM加载的适配器类型与合并方法不匹配，导致无法调用对应接口。

训练脚本与报错分析 #

训练脚本 #

from transformers import *
from peft import *
import torch
from datasets import load_dataset
import os
from torch.utils.data import DataLoader
from transformers import default_data_collator, get_linear_schedule_with_warmup
from tqdm import tqdm
from datasets import load_dataset
from tensorboard import * 

device = "cuda"
tokenizer_name_or_path = "LLM4Binary/llm4decompile-1.3b-v1.5"
model_name_or_path = "LLM4Binary/llm4decompile-1.3b-v1.5"
dataset_name = "asm2c"
text_column = "asm text"
label_column = "text_label"
max_length = 64
lr = 3e-2
num_epochs = 50
batch_size = 8

from datasets import load_dataset

dataset = load_dataset("json", data_files="./traindata.jsonl")
dataset = dataset["train"].train_test_split(0.2)


tokenizer = AutoTokenizer.from_pretrained("LLM4Binary/llm4decompile-1.3b-v1.5")

def preprocess_function(examples):
    inputs = examples["input"]
    outputs = examples["output"]

    # 合并input和output列
    merged_texts = [f"{input} {output_text}" for input, output_text in zip(inputs, outputs)]
    
    model_inputs = tokenizer(merged_texts, truncation=True, padding="max_length", max_length=512)
    model_inputs["labels"] = model_inputs["input_ids"].copy()  # 设置labels
    return model_inputs

processed_datasets = dataset.map(
    preprocess_function,
    batched=True,
    num_proc=1,
    remove_columns=dataset["train"].column_names,
    load_from_cache_file=False,
    desc="Running tokenizer on dataset",
)

train_dataset = processed_datasets["train"]
eval_dataset = processed_datasets["test"]

peft_config = PromptTuningConfig(
    task_type=TaskType.CAUSAL_LM,
    prompt_tuning_init=PromptTuningInit.TEXT,
    num_virtual_tokens=8,
    prompt_tuning_init_text="What's the souce code of this asm?",
    tokenizer_name_or_path=model_name_or_path,
)
checkpoint_name = f"{dataset_name}_{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}_v1.pt".replace(
    "/", "_"
)

# creating model
model = AutoModelForCausalLM.from_pretrained("LLM4Binary/llm4decompile-1.3b-v1.5") #, load_in_8bit=True, torch_dtype=torch.float16, device_map="auto")
#model = prepare_model_for_kbit_training(model)

peft_model = get_peft_model(model, peft_config)


training_args = TrainingArguments(
    output_dir="./results4",             # 保存模型的目录
    evaluation_strategy="epoch",         # 每个 epoch 进行评估
    save_strategy="epoch",               # 每个 epoch 结束时保存模型              
    learning_rate=2e-5,
    per_device_train_batch_size=4,      # 训练时的batch_size
    per_device_eval_batch_size=8,      # 验证时的batch_size
    logging_steps=10,                    # log 打印的频率
    num_train_epochs=3,
    weight_decay=0.01,
    load_best_model_at_end=False
)


trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset=eval_dataset,
    #data_collator=DataCollatorForSeq2Seq(tokenizer=tokenizer, padding=True)
)

trainer.train()
'''
trainer.evaluate(eval_dataset)

# 训练结束后手动保存模型
trainer.save_model(output_dir="./tuned_model")  # 保存最终的模型到指定的目录
tokenizer.save_pretrained(save_directory="./tuned_tokenizer")  # 保存tokenizer
'''
lora_adapter = "./lora_adapter"
peft_model.save_pretrained(lora_adapter, save_adapter=True, save_config=True)

model_to_merge = PeftModel.from_pretrained(AutoModelForCausalLM.from_pretrained(model_name_or_path).to("cuda"), lora_adapter)

merged_model = model_to_merge.merge_and_unload()
merged_model.save_pretrained("./merged_model")

报错信息 #

/root/miniconda3/envs/llm4decompile/lib/python3.10/site-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead
  warnings.warn(
PyTorch: setting up devices
The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).
Traceback (most recent call last):
  File "/root/autodl-tmp/train_pt.py", line 85, in <module>
    trainer = Trainer(
  File "/root/miniconda3/envs/llm4decompile/lib/python3.10/site-packages/transformers/utils/deprecation.py", line 165, in wrapped_func
    return func(*args, **kwargs)
  File "/root/miniconda3/envs/llm4decompile/lib/python3.10/site-packages/transformers/trainer.py", line 553, in __init__
    raise ValueError(
ValueError: You cannot perform fine-tuning on purely quantized models. Please attach trainable adapters on top of the quantized model to correctly perform fine-tuning. Please see: https://huggingface.co/docs/transformers/peft for more details

报错原因 #

1. 训练模型传入错误：Trainer中传入的是原始基础模型而非PEFT包装后的模型，导致直接对量化后的基础模型进行全量微调，违反了PEFT参数高效微调的逻辑。
2. Prompt Tuning合并逻辑错误：Prompt Tuning适配器无法通过merge_and_unload合并到基础模型，代码中强行调用该方法必然失败。

解决方案 #

一、正确的LoRA模型合并流程 #

若需合并模型，需将微调方法改为LoRA（仅LoRA等少数PEFT方法支持合并），步骤如下：

1. 修改训练脚本使用LoRA
   将PEFT配置替换为LoRAConfig，并确保Trainer传入PEFT包装后的模型：

   # 替换原PromptTuningConfig为LoRAConfig
   peft_config = LoraConfig(
       task_type=TaskType.CAUSAL_LM,
       r=8,
       lora_alpha=32,
       target_modules=["q_proj", "v_proj"],  # 根据模型结构调整目标模块
       lora_dropout=0.05,
       bias="none"
   )
   
   # Trainer中传入peft_model而非原始model
   trainer = Trainer(
       model=peft_model,
       args=training_args,
       train_dataset=train_dataset,
       eval_dataset=eval_dataset,
   )
   

2. 合并LoRA与基础模型
   使用以下脚本完成合并：

   from peft import PeftModel
   from transformers import AutoModelForCausalLM, AutoTokenizer
   import torch
   
   base_model_name = "LLM4Binary/llm4decompile-1.3b-v1.5"
   lora_dir = "./lora_adapter"  # 训练后保存的LoRA适配器目录
   
   # 加载基础模型与LoRA适配器
   base_model = AutoModelForCausalLM.from_pretrained(
       base_model_name,
       torch_dtype=torch.bfloat16,
       device_map={"": 0}
   )
   peft_model = PeftModel.from_pretrained(base_model, lora_dir)
   
   # 合并模型
   merged_model = peft_model.merge_and_unload()
   
   # 保存合并后的模型
   merged_model.save_pretrained("./merged_model")
   tokenizer = AutoTokenizer.from_pretrained(base_model_name)
   tokenizer.save_pretrained("./merged_model")
   

二、.safetensors格式支持与转换 #

1. 格式支持说明
   Hugging Face和PEFT完全兼容.safetensors格式，from_pretrained方法会自动识别并加载该格式权重，无需转换即可正常合并。

2. 转换为.bin格式
   若需要将.safetensors转换为.bin格式，可使用两种方式：

   • 单文件转换：

     import torch
     from safetensors.torch import load_file
     
     # 加载safetensors文件并保存为bin格式
     tensors = load_file("adapter_model.safetensors")
     torch.save(tensors, "adapter_model.bin")
     

   • 保存模型时指定格式：

     # 保存模型时禁用safe_serialization，输出.bin格式
     merged_model.save_pretrained("./merged_model", safe_serialization=False)
     

三、Prompt Tuning的替代方案 #

若坚持使用Prompt Tuning，无法合并到基础模型，只能通过PEFT加载适配器进行推理：

from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer

base_model_name = "LLM4Binary/llm4decompile-1.3b-v1.5"
prompt_tuning_dir = "./lora_adapter"

# 加载基础模型与Prompt Tuning适配器
base_model = AutoModelForCausalLM.from_pretrained(base_model_name)
model = PeftModel.from_pretrained(base_model, prompt_tuning_dir)
tokenizer = AutoTokenizer.from_pretrained(base_model_name)

# 推理示例
inputs = tokenizer("What's the souce code of this asm? ...", return_tensors="pt").to("cuda")
outputs = model.generate(**inputs, max_new_tokens=100)
print(tokenizer.decode(outputs[0], skip_special_tokens=True))

内容的提问来源于stack exchange，提问作者P1c3s007