加载预训练DQN模型后性能骤降问题求助

背景 #

我在一个强化学习项目中使用Deep Q-Network（DQN）模型，训练完成后保存模型，后续加载进行评估时，发现加载后的模型性能远低于原预训练模型。

问题 #

加载训练好的模型后，性能大幅下降，模型表现仿佛未经过训练，而训练完成保存前模型性能正常。

训练代码 #

class DQLAgent():
    def __init__(self, env, model_path=None):
        self.env = env
        self.state_size = env.observation_space.shape[0]
        self.action_size = env.action_space.n
        self.gamma = 0.95
        self.learning_rate = 0.001
        self.epsilon = 1
        self.epsilon_decay = 0.995
        self.epsilon_min = 0.01
        self.memory = deque(maxlen=2000)
        
        if model_path:
            self.model = load_model(model_path)  # Load model if path provided
        else:
            self.model = self.build_model()  # Build new model otherwise

    def build_model(self):
        model = Sequential()
        model.add(Dense(48, input_dim=self.state_size, activation='tanh'))
        model.add(Dense(self.action_size, activation='linear'))
        model.compile(loss='mean_squared_error', optimizer=Adam(learning_rate=self.learning_rate))
        return model

    def remember(self, state, action, reward, next_state, done):
        self.memory.append((state, action, reward, next_state, done))

    def act(self, state):
        if np.random.rand() <= self.epsilon:
            return self.env.action_space.sample()
        else:
            act_values = self.model.predict(state,verbose = 0)
            return np.argmax(act_values[0])

    def replay(self, batch_size):
        if len(self.memory) < batch_size:
            return
        minibatch = random.sample(self.memory, batch_size)
        for state, action, reward, next_state, done in minibatch:
            target = reward if done else reward + self.gamma * np.amax(self.model.predict(next_state,verbose = 0)[0])
            train_target = self.model.predict(state,verbose = 0)
            train_target[0][action] = target
            self.model.fit(state, train_target, verbose=0)

    def adaptiveEGreedy(self):
        if self.epsilon > self.epsilon_min:
            self.epsilon *= self.epsilon_decay

# Initialize gym environment and the agent
env = gym.make('CartPole-v1')
agent = DQLAgent(env)

episodes = 50
batch_size = 32
round_results = []

for e in range(episodes):
    state = env.reset()
    state = np.reshape(state, [1, 4])
    total_reward = 0

    while True:
        action = agent.act(state)
        next_state, reward, done, _ = env.step(action)
        next_state = np.reshape(next_state, [1, 4])
        agent.remember(state, action, reward, next_state, done)
        state = next_state
        agent.replay(batch_size)
        agent.adaptiveEGreedy()
        total_reward += reward

        if done:
            print(f'Episode: {e+1}, Total reward: {total_reward}')
            round_results.append(total_reward)
            break

agent.model.save('dql_cartpole_model.keras')

评估代码 #

model_path = 'dql_cartpole_model.keras'  # Update this path

env = gym.make('CartPole-v1')
agent = DQLAgent(env, model_path=model_path)  

episodes = 100
round_results = []

for e in range(episodes):
    state = env.reset()
    state = np.reshape(state, [1, 4])
    total_reward = 0

    while True:
        action = agent.act(state)
        next_state, reward, done, _ = env.step(action)
        next_state = np.reshape(next_state, [1, 4])
        state = next_state
        total_reward += reward

        if done:
            print(f'Episode: {e+1}, Total reward: {total_reward}')
            round_results.append(total_reward)
            break

# Plot the rewards
plt.plot(round_results)
plt.title('Rewards per Episode')
plt.xlabel('Episode')
plt.ylabel('Total Reward')
plt.show()

尝试的解决方法 #

我在DQLAgent类中添加了保存和加载epsilon与memory的方法：

def save_agent_state(self, file_path):
        with open(file_path, 'wb') as file:
            pickle.dump({
                'epsilon': self.epsilon,
                'memory': list(self.memory)  # Convert deque to list before saving
            }, file)

def load_agent_state(self, file_path):
        with open(file_path, 'rb') as file:
            agent_state = pickle.load(file)
            self.epsilon = agent_state['epsilon']
            self.memory = deque(agent_state['memory'], maxlen=2000)

训练结束时添加：

agent.model.save('dql_cartpole_model.keras')
agent.save_agent_state('agent_state.pkl')

评估时添加：

model_path = 'dql_cartpole_model.keras'  # Update this path
agent.load_agent_state('agent_state.pkl')

但问题仍然存在，加载后的模型性能依旧很差。

内容的提问来源于stack exchange，提问作者Unknown