使用LoRA微调BLIP-2时AdamW学习率失效问题排查

我在Colab平台使用A100显卡，通过LoRA方法在RSICD数据集上微调BLIP-2（Salesforce/blip2-opt-2.7b）模型时，发现torch.optim.AdamW的学习率设置完全失效：无论将学习率设为10^55还是10(-55)，Loss的波动幅度基本一致。

为验证该问题，我设置了gamma=0.1的ExponentialLRScheduler，每轮更新学习率——理论上Loss的变化幅度应逐阶减小，但实际打印的学习率虽降至10^-50以下，Loss依然大幅波动。

以下是我的代码片段和输出示例，恳请帮忙分析：为何学习率趋近于0时Loss仍波动剧烈？

from accelerate import dispatch_model, infer_auto_device_map
from accelerate.utils import get_balanced_memory
processor = AutoProcessor.from_pretrained("Salesforce/blip2-opt-2.7b")
model = Blip2ForConditionalGeneration.from_pretrained(
  "Salesforce/blip2-opt-2.7b",
  device_map="auto",
  torch_dtype=torch.bfloat16
)
processor.num_query_tokens = model.config.num_query_tokens
image_token = AddedToken("<image>", normalized=False, special=True)
processor.tokenizer.add_tokens([image_token], special_tokens=True)
model.resize_token_embeddings(len(processor.tokenizer), pad_to_multiple_of=64)
model.config.image_token_index = len(processor.tokenizer) - 1
device = "cuda" if torch.cuda.is_available() else "cpu"
n_epochs = 10
learning_rate = 2e-5
batch_size = 16
gradient_accumulation_steps = 1
weight_decay = 0.01
logging_steps = 5
max_grad_norm = 0
seed = 42
evaluation_strategy = "steps"
lr_scheduler_type = "constant"
lora_alpha = 32
lora_dropout = 0.05
lora_dim = 8
targetData=torch.load("/content/drive/Shareddrives/TEMFOM/target_data1.pt")
print("learning rate: " + str(learning_rate))

config = LoraConfig(
    r=lora_dim,
    lora_alpha=lora_alpha,
    lora_dropout=lora_dropout,
    bias="none",
    target_modules=["q_proj", "k_proj"]
  )
model = get_peft_model(model, config)
model.print_trainable_parameters()
model.train()
optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)
scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.1, last_epoch=-1, verbose=True)

def fine_tune(model, train_dataloader, optimizer, n_epochs, model_name="fine-tuned"):
  for epoch in range(0, n_epochs):
    for idx, batch in enumerate(train_dataloader):
      input_ids = batch.pop("input_ids").to(device)
      pixel_values = batch.pop("pixel_values").to(device, torch.float16)
      outputs = model(
        input_ids=input_ids,
        pixel_values=pixel_values,
        labels=input_ids
      )
      loss = outputs.loss
      loss.backward()
      optimizer.step()
      optimizer.zero_grad()
    print("Loss:", loss.item())
    scheduler.step()
  model.save_pretrained(direct + model_name)

  return model
train_dataset = ImageCaptioningDataset(targetData, processor)

train_dataloader = DataLoader(
  train_dataset,
  shuffle=True,
  batch_size=batch_size,
  collate_fn=collate_fn
)

fine_tuned_model = fine_tune(model, train_dataloader, optimizer, n_epochs, model_name="fine-tuned")

输出示例：

Epoch: 0
Loss: 5.318110466003418
learning rate before: 2e-10
learning rate after: 2.0000000000000002e-11
Loss: 4.520220756530762
...
Loss: 3.7369227409362793

内容的提问来源于stack exchange，提问作者Paul