手动实现梯度下降拟合sin(x)多项式，训练时损失持续上升求助

我需要编写程序，用梯度下降法拟合sin(x)的多项式模型，要求不能用PyTorch的autograd功能，必须手动实现梯度计算逻辑。我已经多次检查代码，并且用简单数据验证了前向传播和反向传播的计算结果都是正确的，但实际训练过程中模型损失却持续上升。这是我的课程作业，恳请帮忙排查问题。

代码实现 #

import torch
from torch.utils.data import Dataset,DataLoader
import matplotlib.pyplot as plt

##DataSet and DataLoader
class MyDataset(Dataset):
    def __init__(self):
        self.x = torch.linspace(0,10,500)
        self.y = torch.sin(self.x)
    
    def __getitem__(self, index):
        return (self.x[index],self.y[index])
    
    def __len__(self):
        return len(self.x)

dataset = MyDataset()
dataloader = DataLoader(dataset,batch_size=50,num_workers=0,shuffle=True)

##Model
class MyModel():
    def __init__(self,n):
        assert n>=1 
        self.n = n
        self.parameters = torch.randn((1,n+1)) #[1,n+1]，Θ0，Θ1...Θn
    
    def backward(self,weight,x):
        """
        weight : (batch,1) y_pred-y
        x : (batch,1)
        """
        x = x.expand(x.shape[0],self.n).clone() #size:(batch,1) -> (batch,n)
        x = torch.concat((torch.ones(x.shape[0],1),x),dim=1) #size:(batch,n) -> (batch,n+1)
        
        gradient_tensor = torch.zeros_like(self.parameters)
        for i in range(self.n+1):
            gradient_tensor[0,i] =  torch.mean(torch.pow(x[:,i],i).unsqueeze(-1) * weight,dim=0)
        return gradient_tensor
    
    def forward(self,x):
        x = x.expand(x.shape[0],self.n).clone() #size:(batch,1) -> (batch,n)
        x = torch.concat((torch.ones(x.shape[0],1),x),dim=1) #,size:(batch,n) -> (batch,n+!)
        for i in range(self.n + 1): 
            x[:,i] = torch.pow(x[:,i],i)
        return x @ self.parameters.T
    
    def step(self,gradient_tensor,lr):
        self.parameters = self.parameters - lr * gradient_tensor

    def __call__(self,x):
        return self.forward(x)

model = MyModel(5) #5 polinomial

##process of training
def train(model:MyModel,epoch=500,lr=1e-5,name=''):
    loss_list = []
    for i in range(epoch):
        total_loss = 0
        for x,y in dataloader:
            x,y = x.unsqueeze(-1),y.unsqueeze(-1) #size:(50) -> (50,1)
            y_pred = model(x)
            
            y_difference = (y_pred - y)
            loss = (0.5 * (y_difference ** 2)).mean()
            total_loss += loss
            
            gradient_tensor = model.backward(y_difference,x)
            model.step(gradient_tensor,lr)   
        print(f"epoch:{i}/{epoch} loss:{total_loss}")
        loss_list.append(total_loss)
    print(f"finish training model {name}")
    return loss_list

##train
loss_3 = train(model,name='3',epoch=10,lr=1e-8)

前向传播验证（5阶多项式） #

x = torch.tensor([0,torch.pi/2],dtype=torch.float64).unsqueeze(-1)
y = torch.tensor([0,1],dtype=torch.float64).unsqueeze(-1)
model.parameters = torch.tensor([1,1,1,1,1,1],dtype=torch.float64).unsqueeze(0)
y_pred = model(x)
y_pred

##output
#tensor([[ 1.0000],
#       [24.5652]])

temp = torch.pi/2
1 + temp + temp ** 2 + temp ** 3 + temp ** 4 + temp ** 5
##output:24.565165351269908

反向传播验证 #

y_difference = y_pred - y
gradient_tensor = model.backward(y_difference,x)
gradient_tensor #output:tensor([[ 12.2826,  18.5080,  29.0724,  45.6668,  71.7332, 112.6782]])

weight = y_difference[1].item()
temp_gradient = torch.tensor([[1,0,0,0,0,0],
                              [weight * 1,weight * (torch.pi/2),weight * (torch.pi/2) ** 2,weight * (torch.pi/2) ** 3,weight * (torch.pi/2) ** 4,weight * (torch.pi/2) ** 5]],dtype=torch.float64)
print(temp_gradient.mean(dim=0)) #output:tensor([ 12.2826,  18.5080,  29.0724,  45.6668,  71.7332, 112.6782])

核心问题：特征尺度不匹配+学习率过小 #

你的代码中，多项式特征的尺度差异极大（比如x∈[0,10]时，x^{5的最大值是100000，而x}0始终是1），这会导致不同参数的梯度量级相差悬殊，小学习率无法更新大尺度特征对应的参数；同时你设置的1e-8学习率过小，根本无法有效驱动参数向最优值更新，反而可能因数值精度问题导致损失波动上升。

具体修复步骤 #

1. 特征标准化
   对多项式特征进行标准化，将每个特征缩放到相近尺度（均值0，方差1），消除尺度差异对梯度的影响：

   def __init__(self,n):
       assert n>=1 
       self.n = n
       self.parameters = torch.randn((1,n+1)) #[1,n+1]，Θ0，Θ1...Θn
       # 预计算训练数据的多项式特征均值和方差
       x_train = torch.linspace(0,10,500).unsqueeze(-1)
       self.poly_features = self._generate_poly_features(x_train)
       self.feature_mean = self.poly_features.mean(dim=0)
       self.feature_std = self.poly_features.std(dim=0)
       # 避免常数项除以0
       self.feature_std[self.feature_std == 0] = 1.0
   
   def _generate_poly_features(self,x):
       x = x.expand(x.shape[0],self.n).clone()
       x = torch.concat((torch.ones(x.shape[0],1),x),dim=1)
       for i in range(self.n + 1):
           x[:,i] = torch.pow(x[:,i],i)
       return x
   
   def forward(self,x):
       x_poly = self._generate_poly_features(x)
       # 标准化特征
       x_poly = (x_poly - self.feature_mean) / self.feature_std
       return x_poly @ self.parameters.T
   
   def backward(self,weight,x):
       x_poly = self._generate_poly_features(x)
       x_poly = (x_poly - self.feature_mean) / self.feature_std
       gradient_tensor = torch.zeros_like(self.parameters)
       for i in range(self.n+1):
           gradient_tensor[0,i] = torch.mean(x_poly[:,i].unsqueeze(-1) * weight,dim=0)
       return gradient_tensor
   

2. 调整学习率
   特征归一化后，将学习率调整为1e-3或1e-2，确保参数能有效更新。

3. 优化损失计算逻辑
   当前total_loss直接累加每个batch的损失，应除以batch数得到平均损失，避免损失值随batch数放大：

   def train(model:MyModel,epoch=500,lr=1e-3,name=''):
       loss_list = []
       batch_num = len(dataloader)
       for i in range(epoch):
           total_loss = 0
           for x,y in dataloader:
               x,y = x.unsqueeze(-1),y.unsqueeze(-1)
               y_pred = model(x)
               
               y_difference = (y_pred - y)
               loss = (0.5 * (y_difference ** 2)).mean()
               total_loss += loss.item() # 转成数值累加，减少内存占用
               
               gradient_tensor = model.backward(y_difference,x)
               model.step(gradient_tensor,lr)   
           avg_loss = total_loss / batch_num
           print(f"epoch:{i}/{epoch} loss:{avg_loss:.6f}")
           loss_list.append(avg_loss)
       print(f"finish training model {name}")
       return loss_list
   

额外建议 #

• 可尝试添加学习率衰减策略，比如每训练50个epoch将学习率减半，避免后期参数震荡。
• 5阶多项式已足够拟合[0,10]区间的sin(x)，过高阶数容易引发过拟合。

内容的提问来源于stack exchange，提问作者LYAJJ