Falcon-7B-Instruct 4bit+LoRA微调后输出乱码，求排查建议

AI 小北

2026-7-18

问题描述

硬件：11G VRAM的2080 Ti显卡
技术栈：4bit量化+LoRA，基础模型为Falcon-7B-Instruct，数据集为OpenAssistant指令集（timdettmers/openassistant-guanaco）
训练情况：

训练25000轮：损失从1.8降至0.7，耗时53小时，模型对"How are you?"这类简单问题输出乱码
训练300轮：损失从1.8降至1.5，仍输出乱码
训练40轮：损失从1.8降至1.7，依旧输出乱码

训练代码：

import torch, einops
from datasets import load_dataset
from peft import LoraConfig
from transformers import (
    AutoModelForCausalLM,
    AutoTokenizer,
    BitsAndBytesConfig,
    AutoTokenizer,
    TrainingArguments
)
from peft.tuners.lora import LoraLayer

from trl import SFTTrainer


def create_and_prepare_model():
    compute_dtype = getattr(torch, "float16")

    bnb_config = BitsAndBytesConfig(
        load_in_4bit=True,
        bnb_4bit_quant_type="nf4",
        bnb_4bit_compute_dtype=compute_dtype,
        bnb_4bit_use_double_quant=True,
    )

    model = AutoModelForCausalLM.from_pretrained(
        "tiiuae/falcon-7b-instruct", quantization_config=bnb_config, device_map={"": 0}, trust_remote_code=True
    )

    peft_config = LoraConfig(
        lora_alpha=16,
        lora_dropout=0.1,
        r=64,
        bias="none",
        task_type="CAUSAL_LM",
        target_modules=[
            "query_key_value"
        ],
    )

    tokenizer = AutoTokenizer.from_pretrained("tiiuae/falcon-7b-instruct", trust_remote_code=True)
    tokenizer.pad_token = tokenizer.eos_token

    return model, peft_config, tokenizer


training_arguments = TrainingArguments(
    output_dir="./results_falcon-7b-instruct-new",
    per_device_train_batch_size=1,
    gradient_accumulation_steps=10,
    optim="paged_adamw_32bit",
    save_steps=5,
    logging_steps=10,
    learning_rate=2e-4,
    fp16=True,
    max_grad_norm=0.3,
    max_steps=20,
    warmup_ratio=0.03,
    # group_by_length=True,
    lr_scheduler_type="constant",
)

model, peft_config, tokenizer = create_and_prepare_model()
model.config.use_cache = False
dataset = load_dataset("timdettmers/openassistant-guanaco", split="train")

trainer = SFTTrainer(
    model=model,
    train_dataset=dataset,
    peft_config=peft_config,
    dataset_text_field="text",
    max_seq_length=512,
    tokenizer=tokenizer,
    args=training_arguments,
    packing=True,
)

trainer.train()
trainer.save_model("falcon-instruct-7b-4bit-openassist-latest-new")
model.config.to_json_file("falcon-instruct-7b-4bit-openassist-latest-new/config.json")

排查建议

对齐推理与训练的prompt格式：OpenAssistant数据集的样本格式为"### Human: ... ### Assistant: ..."，推理时必须使用相同格式触发模型生成，比如输入"### Human: How are you? ### Assistant:"，而非直接输入问题。Falcon-7B-Instruct本身也有固定格式，训练与推理格式不一致会导致输出混乱。
关闭packing参数：代码中packing=True会将多个样本拼接成单序列，破坏短指令任务的文本边界，改成packing=False后重新训练验证。
调整学习率：当前2e-4的学习率对LoRA微调过高，建议降至1e-4或5e-5，过高学习率会导致模型参数震荡，破坏原有生成能力。
验证Lora目标模块：打印model.named_parameters()确认Falcon-7B的注意力模块名称是否为query_key_value，部分版本模型可能使用q_proj/v_proj等名称，目标模块错误会导致LoRA权重未生效。
规范推理时的模型加载：必须通过PeftModel.from_pretrained加载基础模型+LoRA权重，而非直接加载基础模型，示例代码：

from peft import PeftModel
base_model = AutoModelForCausalLM.from_pretrained("tiiuae/falcon-7b-instruct", quantization_config=bnb_config, device_map={"":0}, trust_remote_code=True)
model = PeftModel.from_pretrained(base_model, "falcon-instruct-7b-4bit-openassist-latest-new")

检查tokenizer配置：添加tokenizer.padding_side = "right"，确保padding在文本右侧，避免影响生成逻辑；推理时设置pad_token_id=tokenizer.eos_token_id，防止生成pad token导致乱码。

同类开源代码参考

以下是调整后的完整微调+推理示例代码：

import torch
from datasets import load_dataset
from peft import LoraConfig, PeftModel
from transformers import (
    AutoModelForCausalLM,
    AutoTokenizer,
    BitsAndBytesConfig,
    TrainingArguments,
    pipeline
)
from trl import SFTTrainer

def create_and_prepare_model():
    compute_dtype = torch.float16

    bnb_config = BitsAndBytesConfig(
        load_in_4bit=True,
        bnb_4bit_quant_type="nf4",
        bnb_4bit_compute_dtype=compute_dtype,
        bnb_4bit_use_double_quant=True,
    )

    model = AutoModelForCausalLM.from_pretrained(
        "tiiuae/falcon-7b-instruct", 
        quantization_config=bnb_config, 
        device_map={"": 0}, 
        trust_remote_code=True
    )

    # 确认并设置目标模块
    target_modules = ["query_key_value"]
    # 可选：打印模块名验证
    # for name, _ in model.named_parameters():
    #     if "query" in name or "key" in name or "value" in name:
    #         print(name)

    peft_config = LoraConfig(
        lora_alpha=16,
        lora_dropout=0.05,
        r=32,
        bias="none",
        task_type="CAUSAL_LM",
        target_modules=target_modules,
    )

    tokenizer = AutoTokenizer.from_pretrained("tiiuae/falcon-7b-instruct", trust_remote_code=True)
    tokenizer.pad_token = tokenizer.eos_token
    tokenizer.padding_side = "right"

    return model, peft_config, tokenizer

# 调整训练参数
training_arguments = TrainingArguments(
    output_dir="./results_falcon-7b-instruct",
    per_device_train_batch_size=1,
    gradient_accumulation_steps=8,
    optim="paged_adamw_32bit",
    save_steps=100,
    logging_steps=10,
    learning_rate=1e-4,
    fp16=True,
    max_grad_norm=0.3,
    max_steps=500,
    warmup_ratio=0.03,
    group_by_length=True,
    lr_scheduler_type="cosine",
    report_to="none"
)

model, peft_config, tokenizer = create_and_prepare_model()
model.config.use_cache = False

dataset = load_dataset("timdettmers/openassistant-guanaco", split="train")

# 关闭packing，使用标准样本格式
trainer = SFTTrainer(
    model=model,
    train_dataset=dataset,
    peft_config=peft_config,
    dataset_text_field="text",
    max_seq_length=512,
    tokenizer=tokenizer,
    args=training_arguments,
    packing=False,
)

trainer.train()
trainer.save_model("falcon-instruct-7b-4bit-openassist")

# 推理验证
base_model = AutoModelForCausalLM.from_pretrained(
    "tiiuae/falcon-7b-instruct", 
    quantization_config=bnb_config, 
    device_map={"":0}, 
    trust_remote_code=True
)
fine_tuned_model = PeftModel.from_pretrained(base_model, "falcon-instruct-7b-4bit-openassist")

generator = pipeline(
    "text-generation",
    model=fine_tuned_model,
    tokenizer=tokenizer,
    device_map={"":0}
)

prompt = "### Human: How are you? ### Assistant:"
output = generator(
    prompt,
    max_new_tokens=100,
    temperature=0.7,
    top_p=0.9,
    pad_token_id=tokenizer.eos_token_id
)
print(output[0]['generated_text'])

内容的提问来源于stack exchange，提问作者codemugal