从零实现的前馈全连接ANN训练Loss随Epoch上升问题求助

我基于玩具训练集从零开发了首个采用批量学习模式的前馈全连接ANN，使用反向传播计算损失函数关于权重和偏置的梯度，采用梯度下降作为学习规则。但训练时输出的训练Loss随Epoch增加而不断增大：

E(0) on TrS is: [[7.83898769]]
E(1) on TrS is: [[10.00738465]]
E(2) on TrS is: [[10.76653098]]
E(3) on TrS is: [[15.94001008]]
E(4) on TrS is: [[23.80650667]]
E(5) on TrS is: [[28.65805023]]
E(6) on TrS is: [[29.56550719]]
E(7) on TrS is: [[30.5424694]]
E(8) on TrS is: [[34.26980112]]
E(9) on TrS is: [[39.9948856]]

代码文件 #

loss_functions.py #

import numpy as np

def sum_of_squares(c, t, y, derivative=False):
    ret = 0
    for k in range(c):
        ret += np.square(y - t)
    ret = 1 / 2 * ret
    if derivative:
        return y - t
    return ret

activation_functions.py #

import numpy as np


def sigmoid(a, derivative=False):
    f_a = 1 / (1 + np.exp(-a))
    df_a = np.multiply(f_a, (1 - f_a)) 
    if derivative:
        return df_a
    return f_a


def identity(a, derivative=False):
    f = a
    df = np.ones(np.shape(a))
    if derivative:
        return df
    return f

main.py #

from activation_functions import *
from loss_functions import *


class NeuralNetwork:

    def _init_(self):
        self.layers = []

    def add_layer(self, layer):
        self.layers.append(layer)

    def create(self):
        for i, layer in enumerate(self.layers):
            if i == 0:
                layer.type = "input"
            else:
                if i == len(self.layers) - 1:
                    layer.type = "output"
                else:
                    layer.type = "hidden"
                layer.configure(self.layers[i - 1].neurons)

    def train(self, X, targets):
        MAX_EPOCHS = 10
        loss_function = sum_of_squares
        E = 0  # errore sull'intero DS
        for epoch in range(MAX_EPOCHS):
            for i, x in enumerate(X):
                target = targets[i]
                prediction = self.forward_prop(x.T)
                E_n = loss_function(c, target, prediction)
                E += E_n  
                self.back_prop(target, local_loss=sum_of_squares)
            print("E(%d) on TrS is:" % epoch, E)  # increasing!!!
            self.learning_rule(l_rate=0.05)  

    def forward_prop(self, z):
        for layer in self.layers:
            z = layer.forward_prop_step(z)
        return z

    def back_prop(self, target, local_loss):
        for i, layer in enumerate(self.layers[:0:-1]):
            next_layer = self.layers[-i]  
            prev_layer = self.layers[-i - 2] 
            layer.back_prop_step(next_layer, prev_layer, target, local_loss)

    def learning_rule(self, l_rate):
        # GD
        for layer in self.layers:
            if layer.type != "input":
                layer.weight -= l_rate * layer.dE_dW
                layer.bias -= l_rate * layer.dE_db


class Layer:

    def _init_(self, neurons, type=None, activation=None):
        self.dE_dW = 0 
        self.dE_db = 0
        self.dEn_db = None  # based on the n-th item
        self.dEn_dW = None  # based on the n-th item
        self.dact_a = None  
        self.out = None
        self.weight = None  
        self.bias = None 
        self.w_sum = None  
        self.neurons = neurons 
        self.type = type  
        self.activation = activation 
        self.deltas = None  

    def configure(self, prev_layer_neurons):
        self.weight = np.asmatrix(np.random.normal(0, 0.5, (self.neurons, prev_layer_neurons)))
        self.bias = np.asmatrix(np.random.normal(0, 0.5, self.neurons)).T 
        if self.activation is None:
            if self.type == "hidden":
                self.activation = sigmoid
            elif self.type == "output":
                self.activation = identity

    def forward_prop_step(self, z):
        if self.type == "input":
            self.out = z
        else:
            self.w_sum = np.dot(self.weight, z) + self.bias
            self.out = self.activation(self.w_sum)
        return self.out

    def back_prop_step(self, next_layer, prev_layer, target, local_loss):
        if self.type == "input":
            pass
        elif self.type == "output":
            self.dact_a = self.activation(self.w_sum, derivative=True) 
            self.deltas = np.multiply(self.dact_a, local_loss(c, target, self.out, derivative=True)) 
        else:
            self.dact_a = self.activation(self.w_sum, derivative=True)  
            self.deltas = np.multiply(self.dact_a, np.dot(next_layer.weight.T, next_layer.deltas))

        self.dEn_dW = np.dot(self.deltas, prev_layer.out.T)

        self.dEn_db = self.deltas

        self.dE_dW += self.dEn_dW

        self.dE_db += self.dEn_db


if _name_ == '_main_':
    net = NeuralNetwork() 

    for m in (2, 4, 4, 1):
        net.add_layer(Layer(m))

    net.create()

    X = np.asmatrix([
        [1, 0],
        [1, 1],
        [0, 1],
        [0, 0]
    ])

    targets = np.asarray([1, 0, 0, 0])

    net.train(X, targets)  

已尝试的修复措施 #

• 检查代码bug
• 降低学习率（l_rate）
• 增大MAX_EPOCHS取值
• 将梯度下降公式中的减号替换为加号

问题根源与修复方案 #

1. 构造函数命名错误 #

Python类的构造函数必须使用双下划线__init__，代码中写成单下划线_init_，导致类初始化逻辑完全未执行：

• 将NeuralNetwork类的_init_改为__init__
• 将Layer类的_init_改为__init__
• 将主入口的if _name_ == '_main_'改为if __name__ == '__main__'

2. 未定义变量c #

sum_of_squares函数的第一个参数c（输出层神经元数）在调用时从未定义，导致逻辑混乱：

• 在train方法中添加c = self.layers[-1].neurons获取输出层神经元数
• 确保back_prop_step调用local_loss时传入正确的c参数

3. 批量梯度下降的梯度累积未重置 #

每个epoch结束后未重置dE_dW和dE_db的累积值，导致后续epoch梯度叠加历史错误数据：

• 在train方法的epoch循环开头，添加重置代码：

  self.reset_gradients()
  E = 0
  

• 在NeuralNetwork类中新增方法：

  def reset_gradients(self):
      for layer in self.layers:
          if layer.type != "input":
              layer.dE_dW = 0
              layer.dE_db = 0
  

4. 反向传播的层索引逻辑错误 #

back_prop方法中遍历层的索引错误，导致next_layer和prev_layer指向错误层级：

• 修改back_prop方法：

  def back_prop(self, target, local_loss):
      c = self.layers[-1].neurons
      # 从输出层反向遍历所有非输入层
      for i in reversed(range(1, len(self.layers))):
          layer = self.layers[i]
          prev_layer = self.layers[i-1]
          if layer.type == "output":
              layer.back_prop_step(None, prev_layer, target, local_loss, c)
          else:
              next_layer = self.layers[i+1]
              layer.back_prop_step(next_layer, prev_layer, target, local_loss, c)
  

• 同步修改Layer.back_prop_step方法，添加c参数：

  def back_prop_step(self, next_layer, prev_layer, target, local_loss, c):
      if self.type == "input":
          pass
      elif self.type == "output":
          self.dact_a = self.activation(self.w_sum, derivative=True)
          self.deltas = np.multiply(self.dact_a, local_loss(c, target, self.out, derivative=True))
      else:
          self.dact_a = self.activation(self.w_sum, derivative=True)
          self.deltas = np.multiply(self.dact_a, np.dot(next_layer.weight.T, next_layer.deltas))
  
      self.dEn_dW = np.dot(self.deltas, prev_layer.out.T)
      self.dEn_db = self.deltas
      self.dE_dW += self.dEn_dW
      self.dE_db += self.dEn_db
  

5. 目标值维度不匹配 #

输入的targets是一维数组，与网络输出的矩阵维度不匹配，导致损失计算错误：

• 在train方法中，将目标值转换为匹配维度的矩阵：target = np.asmatrix(targets[i]).T

修复后的核心train方法片段 #

def train(self, X, targets):
    MAX_EPOCHS = 10
    loss_function = sum_of_squares
    c = self.layers[-1].neurons
    for epoch in range(MAX_EPOCHS):
        self.reset_gradients()
        E = 0  # 每个epoch重置总损失
        for i, x in enumerate(X):
            target = np.asmatrix(targets[i]).T
            prediction = self.forward_prop(x.T)
            E_n = loss_function(c, target, prediction)
            E += E_n  
            self.back_prop(target, local_loss=sum_of_squares)
        print("E(%d) on TrS is:" % epoch, E)
        self.learning_rule(l_rate=0.05)

内容的提问来源于stack exchange，提问作者abc123