Python报错ValueError:无法将float NaN转换为整数的解决求助

运行以下Chainer框架的DQN训练代码时：

class Q_Network(chainer.Chain):

    def __init__(self, input_size, hidden_size, output_size):
        super(Q_Network, self).__init__(
            fc1 = L.Linear(input_size, hidden_size),
            fc2 = L.Linear(hidden_size, hidden_size),
            fc3 = L.Linear(hidden_size, output_size)
        )

    def __call__(self, x):
        h = F.relu(self.fc1(x))
        h = F.relu(self.fc2(h))
        y = self.fc3(h)
        return y

    def reset(self):
        self.zerograds()

Q = Q_Network(input_size=env.history_t+1, hidden_size=100, output_size=3)
Q_ast = copy.deepcopy(Q)
optimizer = chainer.optimizers.Adam()
optimizer.setup(Q)

epoch_num = 50
step_max = len(env.data)-1
memory_size = 200
batch_size = 20
epsilon = 1.0
epsilon_decrease = 1e-3
epsilon_min = 0.1
start_reduce_epsilon = 200
train_freq = 10
update_q_freq = 20
gamma = 0.97
show_log_freq = 5

memory = []
total_step = 0
total_rewards = []
total_losses = []

start = time.time()
for epoch in range(epoch_num):

    pobs = env.reset()
    step = 0
    done = False
    total_reward = 0
    total_loss = 0

    while not done and step < step_max:

        # select act
        pact = np.random.randint(3)
        if np.random.rand() > epsilon:
            pact = Q(np.array(pobs, dtype=np.float32).reshape(1, -1))
            pact = np.argmax(pact.data)

        # act
        obs, reward, done = env.step(pact)

        # add memory
        memory.append((pobs, pact, reward, obs, done))
        if len(memory) > memory_size:
            memory.pop(0)

        # train or update q
        if len(memory) == memory_size:
            if total_step % train_freq == 0:
                shuffled_memory = np.random.permutation(memory)
                memory_idx = range(len(shuffled_memory))
                for i in memory_idx[::batch_size]:
                    batch = np.array(shuffled_memory[i:i+batch_size])
                    b_pobs = np.array(batch[:, 0].tolist(), dtype=np.float32).reshape(batch_size, -1)
                    b_pact = np.array(batch[:, 1].tolist(), dtype=np.int32)
                    b_reward = np.array(batch[:, 2].tolist(), dtype=np.int32)
                    b_obs = np.array(batch[:, 3].tolist(), dtype=np.float32).reshape(batch_size, -1)
                    b_done = np.array(batch[:, 4].tolist(), dtype=np.bool)

                    q = Q(b_pobs)
                    maxq = np.max(Q_ast(b_obs).data, axis=1)
                    target = copy.deepcopy(q.data)
                    for j in range(batch_size):
                        target[j, b_pact[j]] = b_reward[j]+gamma*maxq[j]*(not b_done[j])
                    Q.reset()
                    loss = F.mean_squared_error(q, target)
                    total_loss += loss.data
                    loss.backward()
                    optimizer.update()

            if total_step % update_q_freq == 0:
                Q_ast = copy.deepcopy(Q)

        # epsilon
        if epsilon > epsilon_min and total_step > start_reduce_epsilon:
            epsilon -= epsilon_decrease

        # next step
        total_reward += reward
        pobs = obs
        step += 1
        total_step += 1

    total_rewards.append(total_reward)
    total_losses.append(total_loss)

    if (epoch+1) % show_log_freq == 0:
        log_reward = sum(total_rewards[((epoch+1)-show_log_freq):])/show_log_freq
        log_loss = sum(total_losses[((epoch+1)-show_log_freq):])/show_log_freq
        elapsed_time = time.time()-start
        print('\t'.join(map(str, [epoch+1, epsilon, total_step, log_reward, log_loss, elapsed_time])))
        start = time.time()
                
return Q, total_losses, total_rewards

执行Q, total_losses, total_rewards = train_dqn(Environment1(train))时触发错误：ValueError: cannot convert float NaN to integer，尝试int(numpy.nan)无效。

该错误源于训练过程中Q网络输出出现NaN，导致将其转为整数动作时失败。常见诱因：

• 状态数据或奖励值本身包含NaN
• 梯度爆炸导致网络权重变为NaN
• 批量数据处理时维度不匹配或数据类型错误

以下是加入数据校验、梯度裁剪、权重初始化正则化的修复版本：

import numpy as np
import copy
import time
import chainer
from chainer import links as L
from chainer import functions as F

class Q_Network(chainer.Chain):

    def __init__(self, input_size, hidden_size, output_size):
        super(Q_Network, self).__init__(
            fc1 = L.Linear(input_size, hidden_size),
            fc2 = L.Linear(hidden_size, hidden_size),
            fc3 = L.Linear(hidden_size, output_size)
        )
        # 初始化权重，加入小范围随机值防止NaN初始值
        for param in self.params():
            param.data = np.random.normal(0, 0.01, param.data.shape).astype(np.float32)

    def __call__(self, x):
        h = F.relu(self.fc1(x))
        h = F.relu(self.fc2(h))
        y = self.fc3(h)
        return y

    def reset(self):
        self.zerograds()

def train_dqn(env):
    Q = Q_Network(input_size=env.history_t+1, hidden_size=100, output_size=3)
    Q_ast = copy.deepcopy(Q)
    optimizer = chainer.optimizers.Adam()
    optimizer.setup(Q)
    # 添加梯度裁剪防止爆炸
    optimizer.add_hook(chainer.optimizer.GradientClipping(1.0))

    epoch_num = 50
    step_max = len(env.data)-1
    memory_size = 200
    batch_size = 20
    epsilon = 1.0
    epsilon_decrease = 1e-3
    epsilon_min = 0.1
    start_reduce_epsilon = 200
    train_freq = 10
    update_q_freq = 20
    gamma = 0.97
    show_log_freq = 5

    memory = []
    total_step = 0
    total_rewards = []
    total_losses = []

    start = time.time()
    for epoch in range(epoch_num):

        pobs = env.reset()
        # 校验初始状态是否含NaN
        if np.isnan(pobs).any():
            raise ValueError("Initial state contains NaN")
        step = 0
        done = False
        total_reward = 0
        total_loss = 0

        while not done and step < step_max:

            # select act
            pact = np.random.randint(3)
            if np.random.rand() > epsilon:
                q_values = Q(np.array(pobs, dtype=np.float32).reshape(1, -1))
                q_data = q_values.data
                # 检查Q值是否含NaN，若有则随机选择动作
                if np.isnan(q_data).any():
                    pact = np.random.randint(3)
                else:
                    pact = np.argmax(q_data)

            # act
            obs, reward, done = env.step(pact)
            # 校验奖励和新状态是否含NaN
            if np.isnan(reward) or np.isnan(obs).any():
                reward = 0.0  # 替换NaN奖励为0
                obs = np.nan_to_num(obs)  # 替换状态中的NaN为0

            # add memory，过滤含NaN的样本
            if not (np.isnan(pobs).any() or np.isnan(reward) or np.isnan(obs).any()):
                memory.append((pobs, pact, reward, obs, done))
            if len(memory) > memory_size:
                memory.pop(0)

            # train or update q
            if len(memory) == memory_size:
                if total_step % train_freq == 0:
                    shuffled_memory = np.random.permutation(memory)
                    memory_idx = range(len(shuffled_memory))
                    for i in memory_idx[::batch_size]:
                        batch = np.array(shuffled_memory[i:i+batch_size])
                        # 确保批量数据无NaN
                        batch = batch[~np.isnan(batch).any(axis=1)]
                        if len(batch) < batch_size:
                            continue  # 跳过不足批量的样本
                        b_pobs = np.array(batch[:, 0].tolist(), dtype=np.float32).reshape(len(batch), -1)
                        b_pact = np.array(batch[:, 1].tolist(), dtype=np.int32)
                        b_reward = np.array(batch[:, 2].tolist(), dtype=np.float32)  # 改为float避免整数转换问题
                        b_obs = np.array(batch[:, 3].tolist(), dtype=np.float32).reshape(len(batch), -1)
                        b_done = np.array(batch[:, 4].tolist(), dtype=np.bool)

                        q = Q(b_pobs)
                        maxq = np.max(Q_ast(b_obs).data, axis=1)
                        # 检查maxq是否含NaN
                        maxq = np.nan_to_num(maxq)
                        target = copy.deepcopy(q.data)
                        for j in range(len(batch)):
                            target[j, b_pact[j]] = b_reward[j] + gamma * maxq[j] * (not b_done[j])
                        Q.reset()
                        loss = F.mean_squared_error(q, target)
                        total_loss += loss.data if not np.isnan(loss.data) else 0
                        loss.backward()
                        optimizer.update()

                if total_step % update_q_freq == 0:
                    Q_ast = copy.deepcopy(Q)

            # epsilon
            if epsilon > epsilon_min and total_step > start_reduce_epsilon:
                epsilon = max(epsilon_min, epsilon - epsilon_decrease)  # 确保不低于最小值

            # next step
            total_reward += reward
            pobs = obs
            step += 1
            total_step += 1

        total_rewards.append(total_reward)
        total_losses.append(total_loss)

        if (epoch+1) % show_log_freq == 0:
            log_reward = sum(total_rewards[((epoch+1)-show_log_freq):])/show_log_freq
            log_loss = sum(total_losses[((epoch+1)-show_log_freq):])/show_log_freq
            elapsed_time = time.time()-start
            print('\t'.join(map(str, [epoch+1, epsilon, total_step, log_reward, log_loss, elapsed_time])))
            start = time.time()
                    
    return Q, total_losses, total_rewards

• 网络初始化时给权重赋予小范围随机值，避免初始NaN
• 添加梯度裁剪钩子，防止梯度爆炸导致权重NaN
• 动作选择前检查Q值是否含NaN，异常时 fallback 到随机动作
• 对状态、奖励数据做NaN校验，替换异常值为合理值
• 批量数据处理时过滤含NaN的样本，避免污染训练
• 将奖励数据类型改为float，避免整数转换时的NaN问题

内容的提问来源于stack exchange，提问作者user19329602