MNIST手写数字分类神经网络准确率卡11.35%，求故障排查

问题描述 #

我用Python从零构建基于MNIST数据集的手写数字分类神经网络，准确率一直卡在11.35%，无法定位错误。参考了一份能达到60%准确率的代码，但我的代码几乎一致却无法复现效果。

我的代码实现 #

神经网络核心代码 #

"""1. 输入层784个神经元
   2. 隐藏层1共128个神经元
   3. 隐藏层2共64个神经元
   4. 输出层10个神经元"""
def softmax(input):
    y = np.exp(input - input.max())
    activated = y/ np.sum(y, axis=0)
    return activated
 
def softmax_grad(x):
    exps = np.exp(x-x.max())
    return exps / np.sum(exps,axis = 0) * (1 - exps /np.sum(exps,axis = 0))

def sigmoid(input):
    activated = 1/(1 + np.exp(-input))
    return activated

def sigmoid_grad(input):
    grad = input*(1-input)
    return grad


class DenseNN:
    def __init__(self,d0,d1,d2,d3):
        self.params = {'w1': nn.Xavier.initialize(d0, d1),
                       'w2': nn.Xavier.initialize(d1, d2),
                       'w3': nn.Xavier.initialize(d2, d3)}
        
    def forward(self,a0):
        params = self.params
        params['a0'] = a0
        params['z1'] = np.dot(params['w1'],params['a0'])
        params['a1'] = sigmoid(params['z1'])
        
        params['z2'] = np.dot(params['w2'],params['a1'])
        params['a2'] = sigmoid(params['z2'])
        
        params['z3'] = np.dot(params['w3'],params['a2'])
        params['a3'] = softmax(params['z3'])
        
        return params['a3']
        
    def backprop(self,y_true,y_pred):
        params = self.params
        w_change = {}
        error = softmax_grad(params['z3'])*((y_pred - y_true)/y_true.shape[0])
        w_change['w3'] = np.outer(error,params['a2'])
        
        error = np.dot(params['w3'].T,error)*sigmoid_grad(params['a2'])
        w_change['w2'] = np.outer(error,params['a1'])
        
        error = np.dot(params['w2'].T,error)*sigmoid_grad(params['a1'])
        w_change['w1'] = np.outer(error,params['a0'])
        return w_change
        
    def update_weights(self,learning_rate,w_change):
        self.params['w1'] -= learning_rate*w_change['w1']
        self.params['w2'] -= learning_rate*w_change['w2']
        self.params['w3'] -= learning_rate*w_change['w3']

    def train(self,epochs,lr):
        for epoch in range(epochs):
            for i in range(60000):
                a0 = np.array([x_train[i]]).T
                o = np.array([y_train[i]]).T
                y_pred = self.forward(a0)
                w_change = self.backprop(o,y_pred)
                self.update_weights(lr,w_change)
                # print(self.compute_accuracy()*100)
                # print(calc_mse(a3, o))
            print((self.compute_accuracy())*100)
    
    def compute_accuracy(self):
        '''
            对输入做前向传播，检查输出最大值的索引是否与标签y的索引一致，
            统计所有预测结果的正确率平均值
        '''
        predictions = []

        for i in range(10000):
            idx = i
            a0 = x_test[idx]
            a0 = np.array([a0]).T
            #print("acc a1",np.shape(a1))
            o = y_test[idx]
            o = np.array([o]).T
            #print("acc o",np.shape(o))
            output = self.forward(a0)
            pred = np.argmax(output)
            predictions.append(pred == np.argmax(o))
        return np.mean(predictions)

数据加载代码 #

# 加载CSV格式数据集
train_data = pd.read_csv('../Datasets/MNIST/mnist_train.csv')
test_data = pd.read_csv('../Datasets/MNIST/mnist_test.csv')

# 训练数据处理
x_train = train_data.drop('label',axis=1).to_numpy()
y_train = pd.get_dummies(train_data['label']).values

# 测试数据处理
x_test = test_data.drop('label',axis=1).to_numpy()
y_test = pd.get_dummies(test_data['label']).values

# 归一化处理
fac = 0.99 / 255
x_train = np.asfarray(x_train) * fac + 0.01
x_test = np.asfarray(x_test) * fac + 0.01

# 打印维度信息
print(np.shape(x_train)) #(60000,784)
print(np.shape(y_train)) #(60000,10)
print(np.shape(x_test)) #(10000,784)
print(np.shape(y_test)) #(10000,10)
print((x_train))

问题解决 #

感谢Bartosz Mikulski的提示，问题出在Xavier权重初始化的实现上。修改权重初始化代码后，准确率得到明显提升；再添加偏置参数后，效果进一步优化。

修改后的权重初始化代码 #

self.params = {
    'w1':np.random.randn(d1, d0) * np.sqrt(1. / d1),
    'w2':np.random.randn(d2, d1) * np.sqrt(1. / d2),
    'w3':np.random.randn(d3, d2) * np.sqrt(1. / d3),
    'b1':np.random.randn(d1, 1)  * np.sqrt(1. / d1),
    'b2':np.random.randn(d2, 1)  * np.sqrt(1. / d2),
    'b3':np.random.randn(d3, 1)  * np.sqrt(1. / d3),
}

优化后的效果 #

• 修改权重初始化后，准确率脱离11.35%的瓶颈，开始显著提升
• 添加偏置参数后，准确率进一步提高到理想水平

内容的提问来源于stack exchange，提问作者anonymouse