基于Keras与OpenAI Gym的深度强化学习项目报错求助

错误原因分析 #

当前抛出的ValueError: Input 0 of layer "sequential" is incompatible with the layer: expected shape=(None, 4), found shape=(32, 1, 4)核心原因是输入数据维度不匹配，同时代码还存在Q值计算、目标值更新等逻辑错误，加上Gym/Keras版本更新带来的API细节变化，导致一系列问题。

修复后的完整代码 #

import gym
import numpy as np
from keras.models import Sequential, clone_model
from keras.layers import Dense, Activation
from keras.optimizers import Adam
from collections import deque
import random

# 创建环境
env_name = 'CartPole-v1'
env = gym.make(env_name)

num_observations = env.observation_space.shape[0]
num_actions = env.action_space.n

# 创建神经网络
model = Sequential()
model.add(Dense(32, input_shape=(num_observations,), activation='relu'))  # 用num_observations更通用
model.add(Dense(32, activation='relu'))
model.add(Dense(num_actions, activation='linear'))

# 创建并同步目标网络
target_model = clone_model(model)
target_model.set_weights(model.get_weights())

# 定义参数
EPOCHS = 1000
BATCH_SIZE = 32
epsilon = 1.0
EPSILON_REDUCE = 0.995
MIN_EPSILON = 0.01  # 新增最小epsilon，保留探索性
LEARNING_RATE = 0.001
GAMMA = 0.95
REPLAY_FREQ = 4  # 每4步训练一次，提升效率
update_target_model = 10

def epsilon_greedy_action_selection(model, epsilon, observation):
    if np.random.random() > epsilon:
        prediction = model.predict(observation, verbose=0)  # 关闭verbose减少冗余输出
        action = np.argmax(prediction)
    else:
        action = np.random.randint(0, num_actions)
    return action

replay_buffer = deque(maxlen=20000)

def replay(replay_buffer, batch_size, model, target_model):
    if len(replay_buffer) < batch_size:
        return
    
    samples = random.sample(replay_buffer, batch_size)
    states, actions, rewards, new_states, terminateds, truncateds = zip(*samples)

    # 修正输入维度：去掉多余的(1,)维度，从(batch_size,1,4)转为(batch_size,4)
    states = np.array(states).squeeze()
    new_states = np.array(new_states).squeeze()

    targets = target_model.predict(states, verbose=0)
    q_values = model.predict(new_states, verbose=0)

    for i in range(batch_size):
        # 修正Q值计算：取当前样本所有动作的最大Q值
        max_q = np.max(q_values[i])
        target = targets[i].copy()
        
        if terminateds[i] or truncateds[i]:
            target[actions[i]] = rewards[i]
        else:
            target[actions[i]] = rewards[i] + GAMMA * max_q
        
        targets[i] = target

    # 训练模型，关闭verbose减少输出
    model.fit(states, targets, epochs=1, verbose=0)

def update_model_handler(epoch, update_target_model, model, target_model):
    if epoch > 0 and epoch % update_target_model == 0:
        target_model.set_weights(model.get_weights())
        print(f"同步目标网络权重，当前epoch：{epoch}")

# 编译模型
model.compile(loss='mse', optimizer=Adam(learning_rate=LEARNING_RATE))

best_so_far = 0

for epoch in range(EPOCHS):
    observation, _ = env.reset()  # 显式忽略info参数
    observation = np.asarray(observation).reshape([1, num_observations])
    terminated = False
    truncated = False
    points = 0
    step_count = 0

    while not terminated and not truncated:
        action = epsilon_greedy_action_selection(model, epsilon, observation)
        next_observation, reward, terminated, truncated, _ = env.step(action)
        next_observation = np.asarray(next_observation).reshape([1, num_observations])

        replay_buffer.append((observation, action, reward, next_observation, terminated, truncated))
        
        observation = next_observation
        points += 1
        step_count += 1

        # 每N步训练一次，避免每次step都训练拖慢速度
        if step_count % REPLAY_FREQ == 0:
            replay(replay_buffer, BATCH_SIZE, model, target_model)

    # 衰减epsilon，不低于最小值
    epsilon = max(MIN_EPSILON, epsilon * EPSILON_REDUCE)

    update_model_handler(epoch, update_target_model, model, target_model)

    if points > best_so_far:
        best_so_far = points
    if epoch % 25 == 0:
        print(f"epoch: {epoch} | 得分: {points} | epsilon: {round(epsilon,4)} | 历史最佳: {best_so_far}")

env.close()

关键修复点说明 #

• 输入维度修正：经验回放中取出的states每个都是(1,4)的形状，批量后变成(32,1,4)，通过np.squeeze()去掉多余维度，转为模型期望的(32,4)
• Q值计算逻辑修正：原代码max(q_values[i][0])错误，因为q_values[i]是一维数组（对应当前样本的所有动作Q值），改为np.max(q_values[i])获取最大Q值
• 目标值更新修正：原代码target[0][actions[i]]错误，因为targets[i]是一维数组（对应当前样本的目标Q值），直接通过target[actions[i]]更新对应动作的目标值
• 优化训练频率：原代码每次step都调用训练，效率极低，改为每4步训练一次，平衡训练效率和稳定性
• 增加最小epsilon限制：避免epsilon衰减到0，保留一定探索性以应对环境变化
• API适配：显式处理Gym返回的info参数，避免潜在的解包错误
• 目标网络初始化同步：克隆模型后立即同步权重，确保初始状态一致

内容的提问来源于stack exchange，提问作者Skipper1504