Parametric ReLU参数梯度dalpha1恒为0，参数无法更新求排查建议

我构建了一个包含1个隐藏层、以Parametric ReLU作为隐藏层激活函数的神经网络，实现代码如下：

import numpy as np
from scipy.special import expit as sigmoid
from scipy.special import softmax as sm
import pandas as pd
import math
from sklearn.metrics import mean_squared_error
from sklearn.metrics import accuracy_score
from sklearn.metrics import log_loss
from math import sqrt
from math import log

class NeuralNet:
    def __init__(self, num_features, num_hidden1 ,alpha,alpha1, max_epochs, num_output, _EPSILON):
        super().__init__()
        self.num_features=num_features  # number of input nodes (features)
        self.num_hidden1=num_hidden1  # number of hidden nodes for 1st hidden layer
        self.alpha=alpha  # learning rate
        self.alpha1=alpha1 # alpha for para RelU
        self.max_epochs=max_epochs # maximum number of epochs
        self.num_output=num_output # number of output nodes
        self._EPSILON=_EPSILON
        self.loss = [] #list to store losses per 100 epochs 
        self.trainingaccur=[] # list to store training accuracy per 100 epochs 
        self.devaccur=[]
        self.Weights_Input_to_H1=np.random.randn(self.num_hidden1, self.num_features)*(0.1)
        self.Bias_Input_to_H1=np.zeros([self.num_hidden1,1])
        self.Weights_H1_to_output=np.random.randn(self.num_output, self.num_hidden1)*(0.1)
        self.Bias_H1_to_output=np.zeros([self.num_output,1])
        self.dWeights_Input_to_H1=np.zeros([self.num_hidden1, self.num_features])
        self.dBias_Input_to_H1=np.zeros([self.num_hidden1,1])
        self.dWeights_H1_to_output=np.zeros([self.num_output, self.num_hidden1])
        self.dBias_H1_to_output=np.zeros([self.num_output,1])
        
        

    def relU(self,X):
        return np.maximum(X, 0)

    def Para_relU(self,alpha,X):
        return np.maximum(X,alpha*X)

    def Para_deriv_wrt_X(self,alpha,X):
        X[X>0]=1
        X[X<=0]=alpha
        return X
        

    def Para_deriv_wrt_alpha(self,alpha,X):
        return np.where(X<=0,alpha*X,0)

        

    def deriv(self,X):
        X[X>0]=1
        X[X<=0]=0
        return X
        
        

    def softmax(self,x):
        e=np.exp(x)
        for i in range(e.shape[1]):
            e[:,i]=e[:,i]/np.sum(e[:,i])
        return e

    
    # Forward pass implementation
    def forward(self, X):
        self.z1=np.dot((self.Weights_Input_to_H1),(X))+self.Bias_Input_to_H1
        self.a1=self.Para_relU(self.alpha1,self.z1)
        self.z2=np.dot((self.Weights_H1_to_output),(self.a1))+self.Bias_H1_to_output
        self.a2=self.softmax((self.z2))
        return self.a2
        
        
    
    # Backpropagation implementation
    def backprop(self, X, t):
      
        self.dz2=(self.a2.reshape(self.num_output,-1)-t.reshape(self.num_output,-1))/((self.num_output)*(X.shape[1]))
        self.dBias_H1_to_output=np.sum(self.dz2,axis=1,keepdims=True)
        self.dWeights_H1_to_output=np.dot((self.dz2),self.a1.T)
        self.dz1=(np.dot(self.Weights_H1_to_output.T,self.dz2)) * (self.Para_deriv_wrt_X(self.alpha1,self.z1))
        self.dalpha1=(np.dot(self.Weights_H1_to_output.T,self.dz2)) * (self.Para_deriv_wrt_alpha(self.alpha1,self.z1))
        self.dalpha1=np.sum(self.dalpha1)
        self.dBias_Input_to_H1=np.sum(self.dz1,axis=1,keepdims=True)
        self.dWeights_Input_to_H1=np.dot((self.dz1),X.T)
        
        
    # Fit function implementation
    def fit(self, x_train_data, y_train_data,x_dev_data,y_dev_data):
        for step in range(self.max_epochs):
            self.forward(x_train_data)
            self.backprop(x_train_data, y_train_data)
            self.Bias_H1_to_output=self.Bias_H1_to_output-((self.alpha)*(self.dBias_H1_to_output))
            self.Weights_H1_to_output=self.Weights_H1_to_output-((self.alpha)*(self.dWeights_H1_to_output))
            self.Bias_Input_to_H1=self.Bias_Input_to_H1-((self.alpha)*(self.dBias_Input_to_H1))
            self.Weights_Input_to_H1=self.Weights_Input_to_H1-((self.alpha)*(self.dWeights_Input_to_H1))
            self.alpha1=self.alpha1-((self.alpha)*(self.dalpha1))
            

            if step % 100 == 0:
                self.CCloss=log_loss(np.transpose(y_train_data),np.transpose(self.a2),eps=self._EPSILON,normalize=True)
                self.trainingaccuracy=accuracy_score(np.argmax(y_train_data,axis=0),np.argmax(self.forward(x_train_data),axis=0))
                self.devaccuracy=accuracy_score(np.argmax(y_dev_data,axis=0),np.argmax(self.forward(x_dev_data),axis=0))
                print(f'step: {step},  loss: {self.CCloss:3.150f}') 
                print(accuracy_score(np.argmax(y_train_data,axis=0),np.argmax(self.forward(x_train_data),axis=0)))
                print(accuracy_score(np.argmax(y_dev_data,axis=0),np.argmax(self.forward(x_dev_data),axis=0)))
                print(self.dalpha1)
                print(self.alpha1)
                self.loss.append(self.CCloss)
                self.trainingaccur.append(self.trainingaccuracy)
                self.devaccur.append(self.devaccuracy)
                
              
            
    def predict(self,X,y=None):
        self.forward(X)
        if(self.num_output>1):
            y_hat=np.argmax(self.a2, axis=0)
            temp=accuracy_score(y_hat,y)
        else:
            y_hat=np.where(self.a2>0.5,1,0)
            temp=accuracy_score(y_hat,y)
        return temp,y_hat

在实现Parametric ReLU参数alpha1的梯度dalpha1时，我发现打印出的dalpha1值始终为0，通过外部单元调试也仅得到接近0的数值。已核对dalpha1各组成项的维度符合预期，尝试将fit方法中的self.dalpha1替换为完整的np.sum表达式，但alpha1参数仍无法更新，希望得到可能的排查方向建议。

排查方向建议 #

• 修正Parametric ReLU对alpha的导数计算
  原代码中Para_deriv_wrt_alpha函数返回np.where(X<=0, alpha*X, 0)，但根据Parametric ReLU的定义，激活函数max(x, alpha*x)对alpha的导数应为：当x<=0时导数是x，而非alpha*x。这是核心错误，会直接导致梯度为0或极小值。修正后的实现：

  def Para_deriv_wrt_alpha(self, alpha, X):
      return np.where(X <= 0, X, 0)
  

• 避免原地修改z1数组
  原Para_deriv_wrt_X函数直接修改输入的X（即self.z1），会破坏前向传播时的原始z1值，导致后续dalpha1计算使用的是被修改后的数组。应改为返回新数组，不修改原数据：

  def Para_deriv_wrt_X(self, alpha, X):
      return np.where(X > 0, 1, alpha)
  

• 检查梯度缩放因子是否过度
  反向传播中dz2的计算除以了(self.num_output)*(X.shape[1])，这会大幅缩小梯度值，可能让dalpha1趋近于0。交叉熵损失对softmax输出的梯度通常是a2 - t，无需额外除以样本数和输出数的乘积，可尝试移除该缩放因子。

• 验证初始alpha1及z1的负区间激活情况
  如果初始alpha1设为0或极小值，可能导致没有神经元的z1<=0，此时对alpha的导数全为0。建议将初始alpha1设为0.25这类常用值，并在forward函数中添加print(np.sum(self.z1 <= 0))，确认有神经元触发了Parametric ReLU的负区间分支。

• 检查dalpha1的非零元素及求和逻辑
  在backprop中添加print(np.count_nonzero(self.dalpha1))，确认dalpha1是否存在非零值；同时检查np.sum(self.dalpha1)是否对所有维度求和，确保得到正确的标量梯度。

内容的提问来源于stack exchange，提问作者Yuan Zhi Lee