加载Meta-Llama-3.1-8B-Instruct遇tokenizer解析错误，求指导

一、Tokenizer加载错误修复 #

报错核心原因是直接传入单个tokenizer.model文件路径给LlamaTokenizer.from_pretrained()，且该方法已不推荐直接加载单个文件。正确做法是手动用SentencePiece加载模型后初始化Tokenizer：

from sentencepiece import SentencePieceProcessor
from transformers import LlamaTokenizer

tokenizer_path = 'Meta-Llama-3.1-8B-Instruct/tokenizer.model'
sp_processor = SentencePieceProcessor()
sp_processor.Load(tokenizer_path)

# 适配Llama 3.1的Tokenizer初始化
tokenizer = LlamaTokenizer(
    vocab_file=None,
    tokenizer_file=tokenizer_path,
    legacy=False,
    bos_token_id=128000,
    eos_token_id=128001
)

二、模型配置参数修正 #

你的intermediate_size设置为小数会触发错误，需按照params.json规则计算：

• 公式：intermediate_size = 向上取整(dim * ffn_dim_multiplier / multiple_of) * multiple_of
• 计算结果：ceil(4096*1.3/1024)*1024 = 6144

同时补充params.json中的其他关键参数：

from transformers import LlamaConfig

model_config = LlamaConfig(
    hidden_size=4096,
    num_hidden_layers=32,
    num_attention_heads=32,
    num_key_value_heads=8,  # 对应params中的n_kv_heads
    intermediate_size=6144,
    vocab_size=128256,
    norm_eps=1e-05,
    rope_theta=500000.0,
    use_scaled_rope=True,
    bos_token_id=128000,
    eos_token_id=128001
)

三、原生权重转Transformers格式 #

原生consolidated.00.pth的权重键名与Transformers的Llama模型不匹配，需手动映射转换：

import torch

# 加载原生权重
raw_weights = torch.load('Meta-Llama-3.1-8B-Instruct/consolidated.00.pth')

# 键名映射函数
def map_key(key):
    key = key.replace('tok_embeddings.', 'embed_tokens.')
    key = key.replace('norm.', 'model.norm.')
    key = key.replace('output.', 'lm_head.')
    if 'layers.' in key:
        parts = key.split('.')
        layer_idx = parts[1]
        rest = '.'.join(parts[2:])
        rest = rest.replace('attention.wq.', 'self_attn.q_proj.')
        rest = rest.replace('attention.wk.', 'self_attn.k_proj.')
        rest = rest.replace('attention.wv.', 'self_attn.v_proj.')
        rest = rest.replace('attention.wo.', 'self_attn.o_proj.')
        rest = rest.replace('feed_forward.w1.', 'mlp.gate_proj.')
        rest = rest.replace('feed_forward.w2.', 'mlp.down_proj.')
        rest = rest.replace('feed_forward.w3.', 'mlp.up_proj.')
        rest = rest.replace('attention_norm.', 'input_layernorm.')
        rest = rest.replace('ffn_norm.', 'post_attention_layernorm.')
        return f'model.layers.{layer_idx}.{rest}'
    return key

# 转换权重并加载到模型
transformers_weights = {map_key(k): v for k, v in raw_weights.items()}
model = LlamaForCausalLM(config=model_config)
model.load_state_dict(transformers_weights)
model.eval()

四、完整可运行代码 #

import torch
from sentencepiece import SentencePieceProcessor
from transformers import LlamaTokenizer, LlamaForCausalLM, LlamaConfig

model_path = 'Meta-Llama-3.1-8B-Instruct'

# 加载Tokenizer
tokenizer_path = f'{model_path}/tokenizer.model'
sp_processor = SentencePieceProcessor()
sp_processor.Load(tokenizer_path)

tokenizer = LlamaTokenizer(
    vocab_file=None,
    tokenizer_file=tokenizer_path,
    legacy=False,
    bos_token_id=128000,
    eos_token_id=128001
)

# 配置模型
model_config = LlamaConfig(
    hidden_size=4096,
    num_hidden_layers=32,
    num_attention_heads=32,
    num_key_value_heads=8,
    intermediate_size=6144,
    vocab_size=128256,
    norm_eps=1e-05,
    rope_theta=500000.0,
    use_scaled_rope=True,
    bos_token_id=128000,
    eos_token_id=128001
)

# 加载并转换权重
raw_weights = torch.load(f'{model_path}/consolidated.00.pth')

def map_key(key):
    key = key.replace('tok_embeddings.', 'embed_tokens.')
    key = key.replace('norm.', 'model.norm.')
    key = key.replace('output.', 'lm_head.')
    if 'layers.' in key:
        parts = key.split('.')
        layer_idx = parts[1]
        rest = '.'.join(parts[2:])
        rest = rest.replace('attention.wq.', 'self_attn.q_proj.')
        rest = rest.replace('attention.wk.', 'self_attn.k_proj.')
        rest = rest.replace('attention.wv.', 'self_attn.v_proj.')
        rest = rest.replace('attention.wo.', 'self_attn.o_proj.')
        rest = rest.replace('feed_forward.w1.', 'mlp.gate_proj.')
        rest = rest.replace('feed_forward.w2.', 'mlp.down_proj.')
        rest = rest.replace('feed_forward.w3.', 'mlp.up_proj.')
        rest = rest.replace('attention_norm.', 'input_layernorm.')
        rest = rest.replace('ffn_norm.', 'post_attention_layernorm.')
        return f'model.layers.{layer_idx}.{rest}'
    return key

transformers_weights = {map_key(k): v for k, v in raw_weights.items()}
model = LlamaForCausalLM(config=model_config)
model.load_state_dict(transformers_weights)
model.eval()

# 生成文本
input_text = "Hello, how are you?"
inputs = tokenizer(input_text, return_tensors='pt')

with torch.no_grad():
    outputs = model.generate(
        inputs['input_ids'],
        max_new_tokens=50,
        temperature=0.7,
        do_sample=True
    )

decoded_output = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(decoded_output)

内容的提问来源于stack exchange，提问作者Alhusain Alharthi