输入大量数据与多轮epochs训练后神经网络无法有效学习的问题

我用Python构建了一个神经网络，用于区分三元组的和是否小于1.5，但网络无法正常学习。训练100轮后输出仍接近[[0.56, 0.37]]，达不到预期的[[1, 0]]或[[0, 1]]。即使增加epochs、简化网络结构也没用。代码如下：

import numpy as np
import random

##Layers
class ForwardLayer:
    def __init__(self, ins, outs):
        self.w = np.random.random((ins, outs)) * 2 - 1
        self.b = np.random.random((1, outs)) * 2 - 1
        self.cw = np.zeros((ins, outs))
        self.cb = np.zeros((1, outs))
        self.prev = np.zeros((1, outs))
        self.ins = ins
        self.outs = outs
        self.z = np.zeros((1, outs))
        self.n = 0
    def ff(self, a):
        self.prev = a
        self.z = np.dot(a, self.w) + self.b
        return self.act(self.z)
    def act(self, a):
        return 1/(1+np.exp(-a))
    def act_d(self, z):
        return self.act(z)*(1-self.act(z))
    def bp(self, error):
        zerr = error * self.act_d(self.z)
        self.cb += zerr
        self.cw += np.dot(self.prev.transpose(), zerr)
        self.n += 1
        return np.dot(zerr, self.w.transpose())
    def updates(self, lr):
        self.b -= self.cb*lr/self.n
        self.w -= self.cw*lr/self.n
        self.cw = np.zeros((self.ins, self.outs))
        self.cb = np.zeros((1, self.outs))
    def show(self):
        print(self.w)
        print(self.b)

class Flatten:
    def __init__(self):
        self.ins = None
        self.outs = 0
    def ff(self, a):
        self.ins = a.shape
        self.outs = a.size
        return a.reshape(1, self.outs)
    def bp(self, error):
        return error.reshape(self.ins)
    def updates(self, lr):
        pass
    def show(self):
        print("Flatten!")

class DumbLayer:
    def __init__(self):
        pass
    def ff(self, a):
        return a
    def bp(self, error):
        return error
    def updates(self, lr):
        pass
    def show(self):
        pass

#Network
class Network:
    def __init__(self, layers, lr):
        self.layers = layers
        self.lr = lr
    def ff(self, a):
        for l in self.layers:
            a = l.ff(a)
        return a
    def error(self, out, exp):
        return 2*(out - exp)
    def prac(self, a, exp):
        out = self.ff(a)
        err = self.error(out, exp)
        for l in self.layers[::-1]:
            err = l.bp(err)
    def update(self):
        for l in self.layers:
            l.updates(self.lr)
    def out(self):
        for l in self.layers:
            l.show()

net = Network([Flatten(), ForwardLayer(3, 2)], 1)
in1 = np.array([0.1, 0.05, 0.05])
in2 = np.array([0.8, 0.8, 0.8])
out1 = np.array([[1, 0]])
out2 = np.array([[0, 1]])

net.out()
#data
data = []
for _ in range(1000):
    x = random.random()
    y = random.random()
    z = random.random()
    if x+y+z < 1.5:
        data.append((np.array([x, y, z]), out1))
    else:
        data.append((np.array([x, y, z]), out2))

for _ in range(100):
    for i, o in data:
        net.prac(i, o)
        net.update()
print(net.ff(in1))
print(net.ff(in2))
net.out()

问题根源 #

• 训练逻辑冲突：代码里ForwardLayer是按批量梯度下降设计（累计梯度后取平均更新），但实际每处理一个样本就调用update()，相当于随机梯度下降，导致梯度被频繁重置，训练不稳定。
• 损失函数不匹配：用sigmoid输出配合MSE类损失，容易出现梯度消失，减慢收敛速度。
• 输入未归一化：三元组和的范围是0-3，输入特征尺度不一致，影响网络学习效率。
• 学习率过大：学习率设为1，对于sigmoid激活来说容易导致参数震荡，无法收敛。

修复后的代码 #

import numpy as np
import random

##Layers
class ForwardLayer:
    def __init__(self, ins, outs):
        self.w = np.random.random((ins, outs)) * 2 - 1
        self.b = np.random.random((1, outs)) * 2 - 1
        self.cw = np.zeros((ins, outs))
        self.cb = np.zeros((1, outs))
        self.prev = np.zeros((1, outs))
        self.ins = ins
        self.outs = outs
        self.z = np.zeros((1, outs))
        self.n = 0
    def ff(self, a):
        self.prev = a
        self.z = np.dot(a, self.w) + self.b
        return self.act(self.z)
    def act(self, a):
        return 1/(1+np.exp(-a))
    def act_d(self, z):
        return self.act(z)*(1-self.act(z))
    def bp(self, error):
        zerr = error * self.act_d(self.z)
        self.cb += zerr
        self.cw += np.dot(self.prev.transpose(), zerr)
        self.n += 1
        return np.dot(zerr, self.w.transpose())
    def updates(self, lr):
        self.b -= self.cb * lr / self.n
        self.w -= self.cw * lr / self.n
        self.cw = np.zeros((self.ins, self.outs))
        self.cb = np.zeros((1, self.outs))
        self.n = 0  # 重置批量计数
    def show(self):
        print(self.w)
        print(self.b)

class Flatten:
    def __init__(self):
        self.ins = None
        self.outs = 0
    def ff(self, a):
        self.ins = a.shape
        self.outs = a.size
        return a.reshape(1, self.outs)
    def bp(self, error):
        return error.reshape(self.ins)
    def updates(self, lr):
        pass
    def show(self):
        print("Flatten!")

class DumbLayer:
    def __init__(self):
        pass
    def ff(self, a):
        return a
    def bp(self, error):
        return error
    def updates(self, lr):
        pass
    def show(self):
        pass

#Network
class Network:
    def __init__(self, layers, lr):
        self.layers = layers
        self.lr = lr
    def ff(self, a):
        for l in self.layers:
            a = l.ff(a)
        return a
    # 改用交叉熵损失的梯度，适配sigmoid输出的二分类任务
    def error(self, out, exp):
        eps = 1e-8  # 避免log(0)的情况
        return (out - exp) / ((1 - out + eps) * (out + eps))
    def prac(self, a, exp):
        out = self.ff(a)
        err = self.error(out, exp)
        for l in self.layers[::-1]:
            err = l.bp(err)
    def update(self):
        for l in self.layers:
            l.updates(self.lr)
    def out(self):
        for l in self.layers:
            l.show()

# 输入归一化：将三元组特征压缩到0-1区间
def normalize_input(x):
    return x / 3.0

net = Network([Flatten(), ForwardLayer(3, 2)], lr=0.1)  # 降低学习率
in1 = np.array([0.1, 0.05, 0.05])
in2 = np.array([0.8, 0.8, 0.8])
out1 = np.array([[1, 0]])
out2 = np.array([[0, 1]])

#data
data = []
for _ in range(1000):
    x = random.random()
    y = random.random()
    z = random.random()
    if x+y+z < 1.5:
        data.append((np.array([x, y, z]), out1))
    else:
        data.append((np.array([x, y, z]), out2))

# 批量训练：每轮遍历所有样本后再更新参数
for epoch in range(100):
    for i, o in data:
        normalized_i = normalize_input(i)
        net.prac(normalized_i, o)
    net.update()
    # 每10轮打印一次训练状态
    if epoch % 10 == 0:
        pred1 = net.ff(normalize_input(in1))
        pred2 = net.ff(normalize_input(in2))
        print(f"Epoch {epoch} | 输入1预测: {pred1.round(2)} | 输入2预测: {pred2.round(2)}")

print("\n最终测试结果：")
print("输入1预测：", net.ff(normalize_input(in1)).round(2))
print("输入2预测：", net.ff(normalize_input(in2)).round(2))
net.out()

关键改动说明 #

• 调整训练流程：将update()移到每轮循环末尾，实现批量梯度下降，让网络累计所有样本的梯度后再更新，训练更稳定。
• 更换损失梯度：使用交叉熵损失的梯度替代MSE梯度，解决sigmoid激活下的梯度消失问题，加速收敛。
• 输入归一化：将输入除以3，把特征尺度压缩到0-1区间，帮助网络更快捕捉有效特征。
• 降低学习率：将学习率从1调整为0.1，避免参数震荡，让网络平稳收敛。

内容的提问来源于stack exchange，提问作者Neil Peter