NGBoost自定义逆威布尔分布损失函数返回-inf问题求助

我参考NGBoost官方的拉普拉斯分布示例，实现了一个继承自scipy.stats.invweibull的自定义逆威布尔分布类，编写了inv_weibull.py文件，并且已经修改ngboost/distns/__init__.py导入该分布。但在波士顿数据集及自定义数据集测试时，损失函数持续返回-inf，迭代直接终止。以下是完整代码及运行结果，请求排查问题。

参考的拉普拉斯分布示例 #

class LaplaceLogScore(LogScore):

    def score(self, Y):
        return -self.dist.logpdf(Y)

    def d_score(self, Y):
        D = np.zeros((len(Y), 2)) # first col is dS/d𝜇, second col is dS/d(log(b))
        D[:, 0] = np.sign(self.loc - Y)/self.scale
        D[:, 1] = 1 - np.abs(self.loc - Y)/self.scale
        return D

class Laplace(RegressionDistn):

    n_params = 2
    scores = [LaplaceLogScore]

    def __init__(self, params):
        # save the parameters
        self._params = params

        # create other objects that will be useful later
        self.loc = params[0]
        self.logscale = params[1]
        self.scale = np.exp(params[1]) # since params[1] is log(scale)
        self.dist = dist(loc=self.loc, scale=self.scale)

    def fit(Y):
        m, s = dist.fit(Y) # use scipy's implementation
        return np.array([m, np.log(s)])

    def sample(self, m):
        return np.array([self.dist.rvs() for i in range(m)])

    def __getattr__(self, name): # gives us access to Laplace.mean() required for RegressionDist.predict()
        if name in dir(self.dist):
            return getattr(self.dist, name)
        return None

    @property
    def params(self):
        return {'loc':self.loc, 'scale':self.scale}

自定义逆威布尔分布代码（inv_weibull.py） #

import numpy as np
import scipy as sp
from scipy.stats import invweibull
from ngboost.distns import RegressionDistn
from ngboost.scores import LogScore, CRPScore

class InverseWeibullLogScore(LogScore):
    def score(self, Y):
        return -self.dist.logpdf(Y)

    def d_score(self, Y):
        D = np.zeros((len(Y), 2))
        D[:, 0] = -self.alpha * np.power(self.beta * Y, -self.alpha) * np.log(self.beta * Y)
        D[:, 1] = -self.alpha * np.power(self.beta * Y, -self.alpha - 1) * np.log(self.beta * Y)
        return D


    def metric(self):
        FI = np.zeros((self.alpha.shape[0], 2, 2))
        FI[:, 0, 0] = -self.alpha / (self.beta**2)
        FI[:, 1, 1] = self.alpha * (self.alpha + 1) / (self.beta**2)
        FI[:, 0, 1] = 0  # Cross derivatives are zero
        FI[:, 1, 0] = 0  # Cross derivatives are zero
        return FI


class InverseWeibullCRPScore(CRPScore):
    def score(self, Y):
        Z = (Y - self.loc) / self.scale
        return self.scale * (
            Z * (2 * sp.stats.norm.cdf(Z) - 1)
            + 2 * sp.stats.norm.pdf(Z)
            - 1 / np.sqrt(np.pi)
        )

    def d_score(self, Y):
        Z = (Y - self.loc) / self.scale
        D = np.zeros((len(Y), 2))
        D[:, 0] = -(2 * sp.stats.norm.cdf(Z) - 1) * self.dist.pdf(Y) / self.scale
        D[:, 1] = self.score(Y) + (Y - self.loc) * D[:, 0]
        return D

    def metric(self):
        FI = np.zeros((self.alpha.shape[0], 2, 2))
        FI[:, 0, 0] = -self.alpha / (self.beta * self.beta)
        FI[:, 1, 1] = self.alpha * (self.alpha + 1) / (self.beta * self.beta)
        return FI



class InverseWeibull(RegressionDistn):
    """
    Implements the inverse Weibull distribution for NGBoost.

    The inverse Weibull distribution has two parameters, alpha and beta.
    This distribution has both LogScore and InverseWeibullCRPScore implemented for it.
    """

    n_params = 2
    scores = [InverseWeibullLogScore, InverseWeibullCRPScore]

    def __init__(self, params):
        super().__init__(params)
        self.alpha = np.exp(params[0])
        self.beta = np.exp(params[1])
        self.dist = invweibull(self.alpha, scale=self.beta)

    def __getattr__(self, name):
        if name in dir(self.dist):
            return getattr(self.dist, name)
        return None

    @staticmethod
    def fit(Y):
        alpha, _, beta = invweibull.fit(Y, floc=0)
        return np.array([alpha, np.log(beta)])


    
    def sample(self, m):
        return np.array([self.rvs() for _ in range(m)])


    @property
    def params(self):
        return {"alpha": self.alpha, "beta": self.beta}

测试代码 #

from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split

from ngboost import NGBoost,NGBRegressor
from ngboost.distns import InverseWeibull
import numpy as np
import pandas as pd
from sklearn.datasets import load_boston

boston_dataset = load_boston()
boston = pd.DataFrame(boston_dataset.data, columns=boston_dataset.feature_names)
boston['MEDV'] = boston_dataset.target
X = pd.DataFrame(np.c_[boston['LSTAT'], boston['RM']], columns = ['LSTAT','RM'])
Y = boston['MEDV']

X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)


ngb = NGBRegressor(Dist=InverseWeibull).fit(X_train,Y_train)
Y_preds = ngb.predict(X_test)
Y_dists = ngb.pred_dist(X_test)

# test Mean Squared Error
test_MSE = mean_squared_error(Y_preds, Y_test)
print("Test MSE", test_MSE)

# test Negative Log Likelihood
test_NLL = -Y_dists.logpdf(Y_test).mean()
print("Test NLL", test_NLL)

运行结果 #

[iter 0] loss=inf val_loss=0.0000 scale=1.0000 norm=0.0000
== Quitting at iteration / GRAD 0
Test MSE 101.68170913380303
Test NLL inf

导入配置修改 #

已修改ngboost/distns/__init__.py添加：

from .inv_weibull import InverseWeibull

问题排查要点 #

1. CRPScore实现完全错误
   你编写的InverseWeibullCRPScore直接照搬了正态分布的CRPS计算逻辑，但逆威布尔分布的CRPS公式与正态分布完全不同，这会导致分数计算出现inf或异常值。建议先移除InverseWeibullCRPScore，仅保留InverseWeibullLogScore进行测试，后续再补充逆威布尔分布对应的CRPS公式。

2. 参数导数公式错误
   d_score中的导数推导不符合逆威布尔分布的对数似然逻辑：
   逆威布尔分布的对数似然为 ln(α) - α ln(βy) - (βy)^(-α)，由于模型优化的是log(α)和log(β)（因为alpha = np.exp(params[0])、beta = np.exp(params[1])），需要重新推导这两个参数的导数，当前公式会导致梯度计算溢出。

3. Fisher信息矩阵（metric）错误
   当前metric方法返回的FI矩阵值是错误的，错误的Fisher矩阵会干扰NGBoost的梯度更新步长与方向，引发数值不稳定问题。

4. 初始参数传递错误
   fit方法返回的是原始alpha值，但模型期望params[0]是log(alpha)（因为alpha = np.exp(params[0])），这会导致初始alpha被指数放大后异常过大，进而让logpdf计算溢出。需要修改fit方法返回[np.log(alpha), np.log(beta)]。

5. 数值稳定性问题
   当beta*Y趋近于0时，log(beta*Y)会变为负无穷，导致d_score中的计算出现inf。建议给beta*Y添加极小值（如1e-8）避免对数运算溢出。

内容的提问来源于stack exchange，提问作者SMCC9145