PyTorch实现PPO时遇RuntimeError：二次反向传播及原地操作错误

核心问题分析 #

你的错误根源不是retain_graph的问题，而是代码存在几个关键逻辑错误：

• 策略/价值网络混淆：原PolicyNetwork只输出动作概率，但get_value方法直接返回该概率作为状态价值，完全不符合PPO设计（价值网络需输出状态的标量价值，而非动作分布）。
• 未定义变量：update_policy中的ppo_epochs未定义，属于语法错误。
• 单样本循环更新：逐样本计算损失并反向传播，效率低下且易引发计算图重复使用的冲突。
• 张量处理混乱：xvalues的requires_grad设置逻辑错误，advantages和returns的detach处理不当。

修复后的完整代码 #

import torch
import torch.nn as nn
import torch.optim as optim
from torch.distributions import Categorical
import torch.nn.functional as F
import numpy as np

# 联合策略-价值网络：共享底层特征，分支输出动作概率和状态价值
class ActorCriticNetwork(nn.Module):
    def __init__(self, input_size, output_size):
        super(ActorCriticNetwork, self).__init__()
        self.fc = nn.Linear(input_size, 128)
        # 策略分支：输出动作概率分布
        self.actor_fc = nn.Linear(128, output_size)
        # 价值分支：输出状态价值（标量）
        self.critic_fc = nn.Linear(128, 1)

    def forward(self, x):
        x = torch.relu(self.fc(x))
        probs = torch.softmax(self.actor_fc(x), dim=-1)
        value = self.critic_fc(x)
        return probs, value

class PPOAgent:
    def __init__(self, input_size, output_size, lr=1e-3, gamma=0.99, epsilon=0.2, value_coef=0.5, entropy_coef=0.01, ppo_epochs=10, batch_size=32):
        self.policy = ActorCriticNetwork(input_size, output_size)
        self.optimizer = optim.Adam(self.policy.parameters(), lr=lr)
        self.gamma = gamma
        self.epsilon = epsilon
        self.value_coef = value_coef
        self.entropy_coef = entropy_coef
        self.ppo_epochs = ppo_epochs  # 定义PPO迭代轮数
        self.batch_size = batch_size

    def select_action(self, state):
        state = torch.from_numpy(state).float().unsqueeze(0)  # 增加batch维度
        probs, value = self.policy(state)
        m = Categorical(probs)
        action = m.sample()
        return action.item(), m.log_prob(action).item(), value.item()

    def compute_gae(self, rewards, dones, values, next_values):
        # 广义优势估计（GAE），提升训练稳定性
        advantages = []
        advantage = 0.0
        for t in reversed(range(len(rewards))):
            delta = rewards[t] + self.gamma * next_values[t] * (1 - dones[t]) - values[t]
            advantage = delta + self.gamma * 0.95 * (1 - dones[t]) * advantage
            advantages.insert(0, advantage)
        advantages = torch.tensor(advantages, dtype=torch.float32)
        advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8)  # 标准化优势
        returns = advantages + torch.tensor(values, dtype=torch.float32)
        return advantages, returns

    def update_policy(self, states, actions, rewards, log_probs, values, next_values, dones):
        # 转换为张量并统一维度
        states = torch.tensor(np.array(states), dtype=torch.float32)
        actions = torch.tensor(actions, dtype=torch.int64).unsqueeze(1)
        old_log_probs = torch.tensor(log_probs, dtype=torch.float32).unsqueeze(1)
        advantages, returns = self.compute_gae(rewards, dones, values, next_values)
        returns = returns.unsqueeze(1)

        # PPO多轮迭代更新
        for _ in range(self.ppo_epochs):
            # 随机打乱数据，分批次处理
            indices = torch.randperm(len(states))
            for start in range(0, len(states), self.batch_size):
                end = start + self.batch_size
                batch_indices = indices[start:end]
                batch_states = states[batch_indices]
                batch_actions = actions[batch_indices]
                batch_old_log_probs = old_log_probs[batch_indices]
                batch_advantages = advantages[batch_indices].unsqueeze(1)
                batch_returns = returns[batch_indices]

                # 前向传播得到新的概率、价值、对数概率
                new_probs, new_values = self.policy(batch_states)
                dist = Categorical(new_probs)
                new_log_probs = dist.log_prob(batch_actions.squeeze(1)).unsqueeze(1)

                # 裁剪式策略损失
                ratio = torch.exp(new_log_probs - batch_old_log_probs)
                surr1 = ratio * batch_advantages
                surr2 = torch.clamp(ratio, 1 - self.epsilon, 1 + self.epsilon) * batch_advantages
                policy_loss = -torch.min(surr1, surr2).mean()

                # 价值损失
                value_loss = F.mse_loss(new_values, batch_returns)

                # 熵损失（鼓励探索）
                entropy_loss = dist.entropy().mean()

                # 总损失
                total_loss = policy_loss + self.value_coef * value_loss - self.entropy_coef * entropy_loss

                # 反向传播与优化
                self.optimizer.zero_grad()
                total_loss.backward()
                self.optimizer.step()

关键修复点说明 #

• 分离策略与价值分支：改用ActorCriticNetwork同时输出动作概率和状态价值，解决原代码中价值网络的逻辑错误。
• 使用GAE计算优势：替代原始returns计算，提升训练稳定性。
• 批量处理数据：避免逐样本循环，减少计算图冲突，提升训练效率。
• 修复未定义变量：添加ppo_epochs和batch_size初始化，解决语法错误。
• 统一张量维度：添加batch维度，避免维度不匹配问题。
• 移除不必要的retain_graph：批量处理后，每次反向传播都是独立计算图，无需保留图结构，彻底解决反向传播冲突。

内容的提问来源于stack exchange，提问作者heyula