预训练ViT微调后验证损失上升、准确率停滞，如何调优超参数？

我正在TESLA V100 GPU上基于timm库微调预训练的vit_base_patch16_384模型，当前使用3000张训练图像分为15个类别。模型准确率停滞在46%，且训练若干轮后验证损失上升，我认为是超参数设置存在问题，请问该如何获取合适的超参数？

训练曲线 #

训练代码 #

# Batch size
bs = 32
# Number of epochs
num_epochs = 100
# Number of classes
num_classes = 15
# Number of workers
num_cpu = multiprocessing.cpu_count()
# Size of image
imsize = int(args.size)

# timm model list
print(timm.list_models('resnet*', pretrained=True))

# finetune with vit_384
if args.net == "vit_timm_pretrained":
 size = 384

# Applying transforms to the data
image_transforms = {
 'train': transforms.Compose([
 transforms.RandomResizedCrop(size=32, scale=(0.8, 1.0)),
 transforms.Resize(size),
 transforms.RandomRotation(degrees=15),
 transforms.RandomHorizontalFlip(),
 transforms.CenterCrop(size=size),
 transforms.ToTensor(),
 transforms.Normalize([0.485, 0.456, 0.406],
 [0.229, 0.224, 0.225])
 ]),
 'valid': transforms.Compose([
 transforms.Resize(size=size),
 transforms.CenterCrop(size=size),
 transforms.ToTensor(),
 transforms.Normalize([0.485, 0.456, 0.406],
 [0.229, 0.224, 0.225])
 ])
}

# Load data from folders
dataset = {
 'train': datasets.ImageFolder(root=train_directory, transform=image_transforms['train']),
 'valid': datasets.ImageFolder(root=valid_directory, transform=image_transforms['valid'])
}

# Size of train and validation data
dataset_sizes = {
 'train': len(dataset['train']),
 'valid': len(dataset['valid'])
}

# Create iterators for data loading
dataloaders = {
 'train': data.DataLoader(dataset['train'], batch_size=bs, shuffle=True,
 num_workers=num_cpu, pin_memory=True, drop_last=True),
 'valid': data.DataLoader(dataset['valid'], batch_size=bs, shuffle=True,
 num_workers=num_cpu, pin_memory=True, drop_last=True)
}

# Class names or target labels
class_names = dataset['train'].classes
print("Classes:", class_names)

# Print the train and validation data sizes
print("Training-set size:", dataset_sizes['train'],
 "\nValidation-set size:", dataset_sizes['valid'])

# Set default device as gpu, if available
device = 'cuda' if torch.cuda.is_available() else 'cpu'

# Pre-trained vit_base_patch16_384, vit_large_patch32_384, vit_huge_patch14_224
if args.net == "vit_timm_pretrained":
 model_ft = timm.create_model("vit_base_patch16_384", pretrained=True)
 model_ft.head = nn.Linear(model_ft.head.in_features, num_classes)

# Utilize multiple GPUS
if 'cuda' in device:
 print(device)
 print("using data parallel")
 model_ft = torch.nn.DataParallel(model_ft) # make parallel
 cudnn.benchmark = True

# Loss function
criterion = nn.CrossEntropyLoss()

# Optimizer 
optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)

# Learning rate decay
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)

# Model training routine 
print("\nTraining:-\n")

model_ft.cuda()


def train_model(model, criterion, optimizer, scheduler, num_epochs=30):
 since = time.time()

 best_model_wts = copy.deepcopy(model.state_dict())
 best_acc = 0.0

 # Tensorboard summary
 writer = SummaryWriter()

 for epoch in range(num_epochs):
 print('Epoch {}/{}'.format(epoch, num_epochs - 1))
 print('-' * 10)

 # Each epoch has a training and validation phase
 for phase in ['train', 'valid']:
 if phase == 'train':
 model.train() # Set model to training mode
 else:
 model.eval() # Set model to evaluate mode

 running_loss = 0.0
 running_corrects = 0

 # Iterate over data.
 for inputs, labels in dataloaders[phase]:
 # inputs = inputs.to("cuda")
 # labels = labels.to("cuda")

 inputs = inputs.to(device, non_blocking=True)
 labels = labels.to(device, non_blocking=True)

 # zero the parameter gradients
 optimizer.zero_grad()

 # forward
 # track history if only in train
 with torch.set_grad_enabled(phase == 'train'):
 outputs = model(inputs)
 _, preds = torch.max(outputs, 1)
 loss = criterion(outputs, labels)

 # backward + optimize only if in training phase
 if phase == 'train':
 loss.backward()
 optimizer.step()

 # statistics
 running_loss += loss.item() * inputs.size(0)
 running_corrects += torch.sum(preds == labels.data)
 if phase == 'train':
 scheduler.step()

 epoch_loss = running_loss / dataset_sizes[phase]
 epoch_acc = running_corrects.double() / dataset_sizes[phase]

 print('{} Loss: {:.4f} Acc: {:.4f}'.format(
 phase, epoch_loss, epoch_acc))

 # Record training loss and accuracy for each phase
 if phase == 'train':
 writer.add_scalar('Train/Loss', epoch_loss, epoch)
 writer.add_scalar('Train/Accuracy', epoch_acc, epoch)
 writer.flush()
 else:
 writer.add_scalar('Valid/Loss', epoch_loss, epoch)
 writer.add_scalar('Valid/Accuracy', epoch_acc, epoch)
 writer.flush()

 # deep copy the model
 if phase == 'valid' and epoch_acc > best_acc:
 best_acc = epoch_acc
 best_model_wts = copy.deepcopy(model.state_dict())

 print()

 time_elapsed = time.time() - since
 print('Training complete in {:.0f}m {:.0f}s'.format(
 time_elapsed // 60, time_elapsed % 60))
 print('Best val Acc: {:4f}'.format(best_acc))

 # load best model weights
 model.load_state_dict(best_model_wts)
 return model


# Train the model
model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler,
 num_epochs=num_epochs)

优化器调整建议 #

optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)
    
# Learning rate decay
exp_lr_scheduler = lr_scheduler.ReduceLROnPlateau(optimizer_ft, 'min', patience=5, factor=0.5)

if phase == 'train':
  # scheduler.step()
    scheduler.step(loss)

调整后的训练曲线 #

超参数优化方案 #

1. 学习率与优化器调整 #

• 分层学习率：ViT微调时，对预训练主干用小学习率（1e-5_{1e-4），分类头用大学习率（1e-3}1e-2），避免破坏预训练权重。当前统一用0.001的学习率易导致主干过拟合。
• 优化器替换：尝试AdamW替代SGD，参数设置如optim.AdamW(model_ft.parameters(), lr=1e-4, weight_decay=1e-4)，对ViT微调稳定性更强。
• 调度器修正：ReduceLROnPlateau应传入验证损失而非训练损失，避免训练噪声干扰学习率调整逻辑。

2. 数据增强与输入修复 #

• 修正裁剪逻辑：当前RandomResizedCrop(size=32)会把384尺寸图像缩到32再放大，严重破坏特征，改为RandomResizedCrop(size=384, scale=(0.8,1.0))，匹配预训练输入尺寸。
• 增强手段扩容：添加随机擦除（transforms.RandomErasing()）、颜色抖动（transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2)），提升模型泛化能力。

3. 训练策略优化 #

• 阶段式训练：先冻结ViT主干参数，只训练分类头10~20轮，再解冻主干用小学习率微调，避免初始训练破坏预训练特征。
• 早停机制：添加验证损失连续5~10轮不下降则停止训练的逻辑，避免过拟合。
• 批次调整：GPU显存充足时，将batch size调到64或128，提升梯度估计稳定性。

4. 超参数搜索方法 #

• 网格搜索：针对学习率（1e-5,5e-5,1e-4）、权重衰减（1e-5,1e-4,1e-3）等关键参数组合测试，筛选最优组合。
• 贝叶斯优化：用Optuna或Ray Tune工具自动搜索超参数，适合小数据集的高效调参。

内容的提问来源于stack exchange，提问作者Khawar Islam