如何将Whisper基础模型与LoRA训练的适配器模型结合用于推理？

我用LoRA训练了Whisper模型，但遇到个问题：原始训练模型目录大小为2.7G，而LoRA训练后的模型目录仅57M，只保存了附加权重信息，未包含原始权重。想请教如何将现有Whisper模型与LoRA训练的模型结合进行推理？

我的原始代码如下：

import numpy as np
import torch
from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline
from peft import PeftModel, PeftConfig

class whisper:
    # model_str
    # 1. large - "openai/whisper-large-v3"
    # 2. medium - "openai/whisper-medium"
    # 3. small - "openai/whisper-small"
    def __init__(self, baseModelPath):
        device = "cuda:0" if torch.cuda.is_available() else "cpu"
        torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
        
        
        model = AutoModelForSpeechSeq2Seq.from_pretrained(baseModelPath, torch_dtype=torch_dtype, low_cpu_mem_usage=True, use_safetensors=True)
        model.to(device)

        processor = AutoProcessor.from_pretrained(baseModelPath)

        self.pipe = pipeline(
        "automatic-speech-recognition",
        model=model,
        tokenizer=processor.tokenizer,
        feature_extractor=processor.feature_extractor,
        max_new_tokens=128,
        chunk_length_s=30,
        batch_size=16,
        return_timestamps=True,
        torch_dtype=torch_dtype,
        device=device,
        )

        
    # korean
    def getText(self, audioPath, language='<|ko|>'):
        sentence = self.pipe(audioPath, generate_kwargs={"task":"transcribe", "language":language})
        return sentence['text']

原始模型目录文件： #

-rw-r--r--  1 root root   34K Mar  4 17:49 added_tokens.json
-rw-r--r--  1 root root  1.4K Mar  5 09:48 config.json
-rw-r--r--  1 root root  3.0K Mar  5 09:48 generation_config.json
-rw-r--r--  1 root root  483K Mar  4 17:49 merges.txt
-rw-r--r--  1 root root  923M Mar  5 09:48 model.safetensors
-rw-r--r--  1 root root   52K Mar  4 17:49 normalizer.json
-rw-r--r--  1 root root  1.8G Mar  5 09:49 optimizer.pt
-rw-r--r--  1 root root   339 Mar  5 09:48 preprocessor_config.json
-rw-r--r--  1 root root   14K Mar  5 09:49 rng_state.pth
drwxr-xr-x  4 root root  4.0K Mar  4 17:49 runs
-rw-r--r--  1 root root  1.1K Mar  5 09:49 scheduler.pt
-rw-r--r--  1 root root  2.2K Mar  4 17:49 special_tokens_map.json
-rw-r--r--  1 root root  277K Mar  4 17:49 tokenizer_config.json
-rw-r--r--  1 root root   60K Mar  5 09:49 trainer_state.json
-rw-r--r--  1 root root  4.9K Mar  5 09:48 training_args.bin
-rw-r--r--  1 root root 1013K Mar  4 17:49 vocab.json

LoRA训练后模型目录文件： #

drwxr-xr-x  3 root root  4.0K Mar 21 06:50 .
drwxr-xr-x 11 root root  4.0K Mar 21 13:16 ..
-rw-r--r--  1 root root  5.0K Mar 21 06:13 README.md
-rw-r--r--  1 root root   789 Mar 21 06:13 adapter_config.json
drwxr-xr-x  2 root root  4.0K Mar 21 06:13 adapter_model
-rw-r--r--  1 root root   14M Mar 21 06:13 adapter_model.safetensors
-rw-r--r--  1 root root   34K Mar 20 12:55 added_tokens.json
-rw-r--r--  1 root root  483K Mar 20 12:55 merges.txt
-rw-r--r--  1 root root   52K Mar 20 12:55 normalizer.json
-rw-r--r--  1 root root   28M Mar 21 06:13 optimizer.pt
-rw-r--r--  1 root root   339 Mar 21 06:13 preprocessor_config.json
-rw-r--r--  1 root root   14K Mar 21 06:13 rng_state.pth
-rw-r--r--  1 root root  1.1K Mar 21 06:13 scheduler.pt
-rw-r--r--  1 root root  2.2K Mar 20 12:55 special_tokens_map.json
-rw-r--r--  1 root root  277K Mar 20 12:55 tokenizer_config.json
-rw-r--r--  1 root root   31K Mar 21 06:13 trainer_state.json
-rw-r--r--  1 root root  4.9K Mar 21 06:13 training_args.bin
-rw-r--r--  1 root root 1013K Mar 20 12:55 vocab.json

解决方案 #

直接通过peft库的PeftModel类就能把LoRA适配器加载到基础Whisper模型上，修改后的代码如下：

import numpy as np
import torch
from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline
from peft import PeftModel, PeftConfig

class whisper:
    # model_str
    # 1. large - "openai/whisper-large-v3"
    # 2. medium - "openai/whisper-medium"
    # 3. small - "openai/whisper-small"
    def __init__(self, baseModelPath, loraModelPath):
        device = "cuda:0" if torch.cuda.is_available() else "cpu"
        torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
        
        # 加载基础Whisper模型
        model = AutoModelForSpeechSeq2Seq.from_pretrained(
            baseModelPath, 
            torch_dtype=torch_dtype, 
            low_cpu_mem_usage=True, 
            use_safetensors=True
        )
        # 加载LoRA适配器并合并到基础模型
        model = PeftModel.from_pretrained(model, loraModelPath)
        # 可选：固化合并后的模型，后续可单独保存无需再加载基础模型和LoRA
        # model = model.merge_and_unload()
        model.to(device)

        # 处理器可从基础模型或LoRA目录加载，两者均包含相关文件
        processor = AutoProcessor.from_pretrained(baseModelPath)

        self.pipe = pipeline(
            "automatic-speech-recognition",
            model=model,
            tokenizer=processor.tokenizer,
            feature_extractor=processor.feature_extractor,
            max_new_tokens=128,
            chunk_length_s=30,
            batch_size=16,
            return_timestamps=True,
            torch_dtype=torch_dtype,
            device=device,
        )

        
    # korean
    def getText(self, audioPath, language='<|ko|>'):
        sentence = self.pipe(audioPath, generate_kwargs={"task":"transcribe", "language":language})
        return sentence['text']

使用说明： #

• 初始化类时，同时传入基础模型路径baseModelPath和LoRA模型路径loraModelPath
• 若需保存合并后的完整模型，取消注释model = model.merge_and_unload()，再调用model.save_pretrained("merged_model_path")即可，后续推理无需再加载基础模型和LoRA
• 必须保证基础模型与LoRA训练时使用的是同一基座模型（如均为openai/whisper-large-v3），否则会出现兼容性问题

内容的提问来源于stack exchange，提问作者C yp