神经网络权重更新异常排查：手写数字识别模型无法学习

我编写了一个包含2个隐藏层、使用softmax激活函数的手写数字识别神经网络，但模型无法学习，运行时始终收敛到预测0。现附上代码，请求检查Run类中calc_new_hidden1()、calc_new_hidden2()、calc_new_input()这几个权重矩阵更新函数的正确性，我怀疑错误可能出在calc_new_input()函数中。

from cmath import exp

import numpy as np

from tensorflow.keras.datasets import mnist


class Run:
    def __init__(self, num_inputs, num_hidden1, num_hidden2, num_outputs):
        self.num_inputs = num_inputs
        self.num_hidden1 = num_hidden1
        self.num_hidden2 = num_hidden2
        self.num_outputs = num_outputs
        self.learningrate = 0.001
        self.get = GetInput()
        self.count = 0
        self.countTrue = 0
        self.count1 = 0 
        self.sum = 0
        self.past = 0

        self.inputLayer = Layer(num_inputs, num_hidden1)
        self.hiddenLayer1 = Layer(num_hidden1, num_hidden2)
        self.hiddenLayer2  = Layer(num_hidden2, num_outputs)

    def getinput(self):
        input, expected = self.get.get(self.count)
        self.count +=1
        self.count1 += 1
        return input, expected

    def runNN(self, input):
        self.inputLayer.calc_output_1(input)
        self.hiddenLayer1.calc_output_1(self.inputLayer.fin_outputs)
        self.hiddenLayer2.calc_output_1(self.hiddenLayer1.fin_outputs)
        self.NN_Output = self.hiddenLayer2.fin_outputs

    def calculate_cost(self, expected):
        error = 0
        for i in range(self.num_outputs):
            error += (self.NN_Output[i][0] - expected[i][0])**2 / self.num_outputs
    
        list = []
        list1 = []
        for each in self.NN_Output:
            list.append(float(each[0]))
    
        self.sum += list.index(max(list))
    
        for each in expected:
            list1.append(float(each[0]))

        if list1.index(max(list1)) == list.index(max(list)):
            self.countTrue += 1
    
        print(round(self.countTrue/self.count1, 3))

        if self.count1 % 1000 == 0:
            print(self.sum / 1000)
            print('')
            self.past = 0
            self.sum = 0
            self.count1 = 0
            self.countTrue = 0
        return error

    def calc_new_hidden1(self, expected):
        delta = self.NN_Output - expected
        change = np.multiply(delta, self.hiddenLayer2.fin_outputs)
        change_weights = np.matmul(change, np.transpose(self.hiddenLayer2.inputs)) * self.learningrate
        change_bias = change * self.learningrate
        self.hiddenLayer2.amend(change_weights, change_bias)

    def calc_new_hidden2(self, expected):
        delta = self.NN_Output - expected
        change = np.multiply(np.matmul(np.transpose(self.hiddenLayer2.getter()[0]), delta), self.hiddenLayer1.fin_outputs)
        change_weights = np.matmul(change, np.transpose(self.hiddenLayer1.inputs)) * self.learningrate
        change_bias = change * self.learningrate
        self.hiddenLayer1.amend(change_weights, change_bias)

    def calc_new_input(self, expected):
        delta = (self.NN_Output - expected)
        change = np.multiply(np.matmul(np.transpose(self.hiddenLayer1.getter()[0]), np.matmul(np.transpose(self.hiddenLayer2.getter()[0]), delta)), self.inputLayer.fin_outputs)
        change_weights = np.matmul(change, np.transpose(self.inputLayer.inputs)) * self.learningrate
        change_bias = change * self.learningrate
        self.inputLayer.amend(change_weights, change_bias)


class Layer:
    def __init__(self, num_inputs, num_outputs):
        self.__weights = np.random.uniform(-0.5, 0.5, (num_outputs, num_inputs))
        self.__bias = np.matrix([[float(0)] for x in range(num_outputs)])

    def calc_output_1(self, inputs):
        self.inputs = inputs
        self.__output_1 = np.matmul(self.__weights, inputs) + self.__bias
        self.softmax()

    def softmax(self):
        sum = 0
        for each in self.__output_1:
            sum += np.exp(float(each[0]))
    
        list1 = []
        for each in self.__output_1:
            list1.append([float(np.exp(each[0])/sum)])
    
        self.fin_outputs = np.matrix(list1)

    def amend(self, change_weights, change_bias):
        self.__weights -= change_weights
        self.__bias -= change_bias

    def getter(self):
        return self.__weights, self.__bias

class GetInput:
    def __init__(self):
        (self.X_train, self.Y_train), (X_test, Y_test) = mnist.load_data()
        self.X_train = self.X_train.reshape(self.X_train.shape[0], 28, 28, 1)
        x_test = X_test.reshape(X_test.shape[0], 28, 28, 1)

    def get(self, i):
        list = []
        newPhoto = self.X_train[i].astype('float32')/255
        for each in newPhoto:
            for n in each:
                list.append([float(n)])
        input = np.matrix(list)

        list = []
        expect = self.Y_train[i]
        for each in range(10):
            if each == expect:
                list.append([1])
            else:
                list.append([0])
        expected = np.matrix(list)
        
        return input, expected

if __name__ == "__main__":
    initiate = Run(784, 600, 400, 10)
    while True:
        input, expected = initiate.getinput()
        initiate.runNN(input)
        initiate.calculate_cost(expected)
        initiate.calc_new_hidden1(expected)
        initiate.calc_new_hidden2(expected)
        initiate.calc_new_input(expected)

核心问题分析与修复 #

你的模型无法学习的根本原因是反向传播的梯度计算完全不符合链式法则，同时存在激活函数选择、权重初始化、学习率设置等问题，具体如下：

1. 反向传播梯度计算错误 #

输出层（hiddenLayer2）错误

• 你错误地在输出层delta上额外乘了self.hiddenLayer2.fin_outputs，对于MSE损失+softmax输出，正确的输出层delta就是self.NN_Output - expected，多余的乘法会完全破坏梯度方向。
• 函数命名混淆：calc_new_hidden1()实际更新的是输出层权重，应该改成calc_new_output()避免混乱。

隐藏层梯度计算错误

• 隐藏层错误使用softmax激活：softmax只适合输出层，隐藏层用它会导致梯度消失（导数趋近于0），无法传递有效梯度。
• 遗漏激活函数导数：反向传播时，隐藏层的delta需要乘以激活函数的导数，你直接乘了输出值，完全不符合链式法则。
• calc_new_input()中转置矩阵顺序错误：应该先乘输出层权重转置，再乘第二个隐藏层权重转置，顺序搞反会导致梯度维度不匹配或方向错误。

2. 其他影响学习的问题 #

• 学习率0.001过小：对于784→600→400→10的网络，这个学习率不足以让权重有效更新，建议调到0.01。
• 权重初始化方差过大：np.random.uniform(-0.5, 0.5)会导致初始输出值过大，激活函数进入饱和区，建议用Xavier初始化。
• 无限循环：主循环没有终止条件，会一直遍历训练集直到报错。

修复后的关键代码 #

第一步：重构Layer类，支持不同激活函数 #

class Layer:
    def __init__(self, num_inputs, num_outputs, activation='sigmoid'):
        # Xavier初始化，避免初始权重方差过大
        self.__weights = np.random.uniform(-np.sqrt(6/(num_inputs+num_outputs)), 
                                          np.sqrt(6/(num_inputs+num_outputs)), 
                                          (num_outputs, num_inputs))
        self.__bias = np.zeros((num_outputs, 1))
        self.activation = activation

    def calc_output_1(self, inputs):
        self.inputs = inputs
        self.z = np.matmul(self.__weights, inputs) + self.__bias
        if self.activation == 'sigmoid':
            self.fin_outputs = 1 / (1 + np.exp(-self.z))
        elif self.activation == 'softmax':
            # 防止数值溢出的softmax实现
            exp_z = np.exp(self.z - np.max(self.z))
            self.fin_outputs = exp_z / np.sum(exp_z, axis=0, keepdims=True)
        else:
            self.fin_outputs = self.z

    # 添加激活函数导数计算
    def activation_derivative(self):
        if self.activation == 'sigmoid':
            return self.fin_outputs * (1 - self.fin_outputs)
        elif self.activation == 'softmax':
            return self.fin_outputs * (1 - self.fin_outputs)
        else:
            return np.ones_like(self.fin_outputs)

    def amend(self, change_weights, change_bias):
        self.__weights -= change_weights
        self.__bias -= change_bias

    def getter(self):
        return self.__weights, self.__bias

第二步：修正Run类的反向传播函数 #

class Run:
    def __init__(self, num_inputs, num_hidden1, num_hidden2, num_outputs):
        self.num_inputs = num_inputs
        self.num_hidden1 = num_hidden1
        self.num_hidden2 = num_hidden2
        self.num_outputs = num_outputs
        self.learningrate = 0.01  # 调大学习率
        self.get = GetInput()
        self.count = 0
        self.countTrue = 0
        self.count1 = 0 
        self.sum = 0
        self.past = 0

        # 指定各层激活函数：隐藏层用sigmoid，输出层用softmax
        self.inputLayer = Layer(num_inputs, num_hidden1, activation='sigmoid')
        self.hiddenLayer1 = Layer(num_hidden1, num_hidden2, activation='sigmoid')
        self.hiddenLayer2 = Layer(num_hidden2, num_outputs, activation='softmax')

    # ... 其他函数（getinput、runNN、calculate_cost）不变 ...

    # 修正输出层权重更新
    def calc_new_output(self, expected):
        delta = self.NN_Output - expected
        change_weights = np.matmul(delta, np.transpose(self.hiddenLayer2.inputs)) * self.learningrate
        change_bias = np.sum(delta, axis=1, keepdims=True) * self.learningrate
        self.hiddenLayer2.amend(change_weights, change_bias)
        return delta

    # 修正第二个隐藏层权重更新
    def calc_new_hidden2(self, output_delta):
        delta = np.multiply(np.matmul(np.transpose(self.hiddenLayer2.getter()[0]), output_delta), 
                           self.hiddenLayer1.activation_derivative())
        change_weights = np.matmul(delta, np.transpose(self.hiddenLayer1.inputs)) * self.learningrate
        change_bias = np.sum(delta, axis=1, keepdims=True) * self.learningrate
        self.hiddenLayer1.amend(change_weights, change_bias)
        return delta

    # 修正输入层到第一个隐藏层的权重更新
    def calc_new_input(self, hidden2_delta):
        delta = np.multiply(np.matmul(np.transpose(self.hiddenLayer1.getter()[0]), hidden2_delta), 
                           self.inputLayer.activation_derivative())
        change_weights = np.matmul(delta, np.transpose(self.inputLayer.inputs)) * self.learningrate
        change_bias = np.sum(delta, axis=1, keepdims=True) * self.learningrate
        self.inputLayer.amend(change_weights, change_bias)

    # 封装训练步骤
    def train_step(self, input, expected):
        self.runNN(input)
        self.calculate_cost(expected)
        output_delta = self.calc_new_output(expected)
        hidden2_delta = self.calc_new_hidden2(output_delta)
        self.calc_new_input(hidden2_delta)

第三步：修改主循环，添加终止条件 #

if __name__ == "__main__":
    initiate = Run(784, 600, 400, 10)
    max_steps = 60000  # MNIST训练集总样本数
    for _ in range(max_steps):
        input, expected = initiate.getinput()
        initiate.train_step(input, expected)

额外优化建议 #

• 改用小批量训练：每次取32/64个样本计算平均梯度，比单样本训练收敛更快更稳定。
• 替换隐藏层激活函数：用ReLU替代sigmoid，进一步缓解梯度消失问题。
• 添加损失跟踪：除了准确率，实时打印损失值，更直观观察模型学习状态。

内容的提问来源于stack exchange，提问作者Dhdhdh