DQN模型训练失效或异常缓慢问题求助（附实现代码）

问题描述 #

我正在为博士研究构建DQN模型，在应用真实数据前先用模拟数据（dummy data）做测试。之前用简单Q Learning方法时效果不错，但切换到DQN后，训练要么失效要么异常缓慢，已经开启GPU加速但无济于事，怀疑是模拟数据集规模或其他未知因素导致，寻求专业指导。

修复后的实现代码 #

import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import random
import pandas as pd
from collections import deque

# 设置计算设备
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print("运行设备:", device)

# 生成模拟数据
data = {
    'message_size': np.random.randint(1000, 70000, size=1000),
    'cpu_usage': np.random.uniform(40, 100, size=1000),
    'submission_time': np.random.uniform(0, 300, size=1000)
}
dummy_data = pd.DataFrame(data)

# 超参数设置
MAX_BLOCK_SIZE = 32768
GAMMA = 0.9
EPSILON = 1.0
EPSILON_MIN = 0.01
EPSILON_DECAY = 0.99
BATCH_SIZE = 32
EPISODES = 1000
MAX_STEPS_PER_EPISODE = 20
TARGET_UPDATE_FREQ = 10

# DQN模型定义
class DQN(nn.Module):
    def __init__(self, input_dim, num_actions):
        super(DQN, self).__init__()
        self.fc1 = nn.Linear(input_dim, 64)
        self.fc2 = nn.Linear(64, 64)
        self.fc3 = nn.Linear(64, num_actions)
    
    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        return self.fc3(x)

# 计算最大可能动作数
max_possible_action = int(dummy_data['submission_time'].max() // MAX_BLOCK_SIZE) + 1

# 初始化模型、优化器与经验回放池
dqn = DQN(input_dim=2, num_actions=max_possible_action).to(device)
target_model = DQN(input_dim=2, num_actions=max_possible_action).to(device)
target_model.load_state_dict(dqn.state_dict())
optimizer = optim.Adam(dqn.parameters(), lr=0.001)
memory = deque(maxlen=2000)

# 动作选择函数
def block_choice(state):
    max_action = int(state[0] // MAX_BLOCK_SIZE) + 1
    if random.random() < EPSILON:
        return random.randint(1, max_action)
    else:
        state_tensor = torch.FloatTensor(state).unsqueeze(0).to(device)
        q_values = dqn(state_tensor)
        valid_q_values = q_values[0, :max_action]
        return torch.argmax(valid_q_values).item() + 1

# 奖励函数
def utility_function_rewarding(total_latency, cpu_per_block, max_latency=300, max_cpu=100):
    latency_reward = max(0, 1 - (total_latency / max_latency))
    cpu_reward = max(0, 1 - (cpu_per_block / max_cpu))
    return latency_reward + cpu_reward

# DQN训练函数
def dqn_training(batch_size):
    if len(memory) < batch_size:
        return
    batch = random.sample(memory, batch_size)
    states, actions, rewards, next_states, dones = zip(*batch)
    
    states = torch.FloatTensor(states).to(device)
    rewards = torch.FloatTensor(rewards).unsqueeze(1).to(device)
    next_states = torch.FloatTensor(next_states).to(device)
    dones = torch.FloatTensor(dones).unsqueeze(1).to(device)

    # 获取当前状态下所选动作的Q值
    state_action_values = dqn(states).gather(1, torch.LongTensor(actions).unsqueeze(1).to(device) - 1)
    # 获取下一状态的最大Q值（来自目标网络）
    next_state_values = target_model(next_states).max(1)[0].unsqueeze(1)
    expected_values = rewards + (GAMMA * next_state_values * (1 - dones))

    loss = nn.functional.mse_loss(state_action_values, expected_values)
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

# 存储经验到回放池
def store_transition(state, action, reward, next_state, done):
    memory.append((state, action, reward, next_state, done))

# 主训练循环
for episode in range(EPISODES):
    print(f"开始第 {episode + 1}/{EPISODES} 轮训练")
    row = dummy_data.sample().iloc[0]
    state = [row['submission_time'], row['cpu_usage']]
    total_reward = 0
    done = False
    steps = 0

    while not done and steps < MAX_STEPS_PER_EPISODE:
        action = block_choice(state)
        # 基于当前动作生成下一状态（替代原随机采样逻辑）
        next_latency = state[0] * (1 - 0.05 * action)  # 示例：块数量越多，延迟降低
        next_cpu = state[1] / action  # 块数量越多，单块CPU占用越低
        next_state = [next_latency, next_cpu]

        reward = utility_function_rewarding(next_latency, next_cpu)
        total_reward += reward
        done = (steps == MAX_STEPS_PER_EPISODE - 1)
        store_transition(state, action, reward, next_state, done)
        
        state = next_state
        dqn_training(BATCH_SIZE)
        steps += 1
    
    # 衰减探索率
    if EPSILON > EPSILON_MIN:
        EPSILON *= EPSILON_DECAY

    # 定期更新目标网络
    if episode % TARGET_UPDATE_FREQ == 0:
        target_model.load_state_dict(dqn.state_dict())

    print(f"第 {episode + 1}/{EPISODES} 轮训练完成 - 总奖励: {total_reward:.2f}")

核心问题与修复说明 #

• 动作空间与模型输出不匹配：原模型仅输出1维值，无法对应离散动作空间。修改输出层维度为最大可能动作数，确保每个动作有独立Q值。
• 训练循环死循环：原代码仅最后一轮训练才终止循环，导致前序轮次无限运行。改为每轮训练固定步数后终止。
• 状态转移无因果关联：原逻辑随机采样下一状态，动作与状态变化完全无关，模型无法学习有效策略。调整为基于当前动作生成下一状态，建立动作-状态的逻辑关联。
• 学习率过高：原Adam优化器学习率0.1过大，导致参数震荡无法收敛，调整为DQN常用的0.001。
• 缺失目标网络更新：新增定期同步目标网络参数的逻辑，避免Q值估计波动，提升训练稳定性。
• 动作索引逻辑错误：修正动作的0/1基转换问题，确保模型输出的索引与实际可选动作范围匹配。

内容的提问来源于stack exchange，提问作者Konstantinos Voulgaridis