为何JavaScript神经网络反向传播函数会使权重与偏置变为NaN？

问题描述 #

我用JavaScript编写了一个小型神经网络，从C#转用JavaScript让我十分恼火，因为我认为JavaScript更容易出现bug。我已修复最初的错误，但现在无法执行反向传播，因为处理后weights和biases全部变为NaN！请问我的BackPropagation Function为何会导致权重变为NaN？

初始化代码：var neuralNet = new NeuralNet(10, 10, 1);

原代码 #

class NeuralNet {
// const for learningRate
static LEARNING_RATE = 0.33;
// define float[][] for biases
biases;
// define float[][] for values
values;
// define float[][] for errors
errors;
// define float[][][] for weights
weights;

// constructor
constructor(inputSize, hiddenSize, outputSize) {
    // set biases
    this.biases = [
        // hidden layer
        new Array(hiddenSize).fill(0).map(NeuralNet.random),
        // output layer
        new Array(outputSize).fill(0).map(NeuralNet.random)
    ];
    // set values
    this.values = [
        // input layer
        new Array(inputSize).fill(0),
        // hidden layer
        new Array(hiddenSize).fill(0),
        // output layer
        new Array(outputSize).fill(0)
    ];
    // set errors
    this.errors = [
        // hidden layer
        new Array(hiddenSize).fill(0).map(() => new Array(inputSize).fill(0)),
        // output layer
        new Array(outputSize).fill(0).map(() => new Array(hiddenSize).fill(0))
    ];
    // set weights
    this.weights = [
        // hidden layer
        new Array(hiddenSize).fill(0).map(() => new Array(inputSize).fill(0).map(NeuralNet.random)),
        // output layer
        new Array(outputSize).fill(0).map(() => new Array(hiddenSize).fill(0).map(NeuralNet.random))
    ];
}

// define a random
static random() {
    // return random number between -1 and 1
    return Math.random() * 2 - 1;
}

// tanh function
static tanh(x) {
    // return tanh of x
    return Math.tanh(x);
}

// tanh activation function
feedForward() {
    // for each layer
    for (var i = 1; i < this.values.length; i++) {
        // for each neuron
        for (var j = 0; j < this.values[i].length; j++) {
            // set value to bias
            this.values[i][j] = this.biases[i - 1][j];
            // for each weight
            for (var k = 0; k < this.weights[i - 1].length; k++) {
                // add weight * value
                this.values[i][j] += this.weights[i - 1][k][j] * this.values[i][k];
            }
            // apply tanh
            this.values[i][j] = NeuralNet.tanh(this.values[i][j]);
        }
    }
}

// get output of neural net
getOutput(inpts) {
    // set input
    this.setInput(inpts);
    // feed forward
    this.feedForward();
    // return the output
    return this.values[this.values.length - 1];
}

// backpropagation that doesn't make all values NaN
backpropagation(expected) {
    // for each layer
    for (var i = this.values.length - 1; i > 0; i--) {
        // for each neuron
        for (var j = 0; j < this.values[i].length; j++) {
            // if output layer
            if (i == this.values.length - 1) {
                // set error
                this.errors[i - 1][j] = (expected[j] - this.values[i][j]) * (1 - this.values[i][j] * this.values[i][j]);
            } else {
                // set error
                this.errors[i - 1][j] = 0;
                // for each weight
                for (var k = 0; k < this.weights[i].length; k++) {
                    // add weight * error
                    this.errors[i - 1][j] += this.weights[i][k][j] * this.errors[i][k];
                }
                // multiply error with tanh
                this.errors[i - 1][j] *= (1 - this.values[i][j] * this.values[i][j]);
            }
            // for each weight
            for (var k = 0; k < this.weights[i - 1].length; k++) {
                // add error * value * learning rate
                this.weights[i - 1][k][j] += this.errors[i - 1][j] * this.values[i - 1][k] * NeuralNet.LEARNING_RATE;
            }
            // add error * learning rate
            this.biases[i - 1][j] += this.errors[i - 1][j] * NeuralNet.LEARNING_RATE;
        }
    }
}

问题根源及修复方案 #

1. Errors数组结构完全错误 #

你把errors定义成了二维数组，但实际上每个神经元的错误是单个数值，不是数组。当你用数组参与数值计算（比如减法、乘法）时，结果会变成NaN，这是权重/偏置变NaN的核心原因。

修复后的errors定义：

this.errors = [
    // 隐藏层每个神经元对应一个错误值
    new Array(hiddenSize).fill(0),
    // 输出层每个神经元对应一个错误值
    new Array(outputSize).fill(0)
];

2. FeedForward中的权重与输入值索引错误 #

计算当前层神经元值时，你错误地使用了当前层未计算的values[i][k]，而应该用前一层的输出值values[i-1][k]；同时权重的索引顺序也搞反了，weights[i-1][k][j]应该改为weights[i-1][j][k]（权重矩阵结构是[当前层神经元数][前一层神经元数]）。

修复后的feedForward函数：

feedForward() {
    for (var i = 1; i < this.values.length; i++) {
        for (var j = 0; j < this.values[i].length; j++) {
            this.values[i][j] = this.biases[i - 1][j];
            // 遍历前一层的所有神经元，而非当前层
            for (var k = 0; k < this.values[i-1].length; k++) {
                this.values[i][j] += this.weights[i - 1][j][k] * this.values[i - 1][k];
            }
            this.values[i][j] = NeuralNet.tanh(this.values[i][j]);
        }
    }
}

3. 反向传播中的权重更新索引错误 #

更新权重时，同样搞反了权重的索引顺序，weights[i-1][k][j]应该改为weights[i-1][j][k]，同时循环的遍历对象应该是前一层的神经元数量，而非当前层权重数组的长度。

修复后的反向传播权重更新部分：

// 遍历前一层的所有神经元
for (var k = 0; k < this.values[i-1].length; k++) {
    this.weights[i - 1][j][k] += this.errors[i - 1][j] * this.values[i - 1][k] * NeuralNet.LEARNING_RATE;
}

4. 补充缺失的setInput方法 #

原代码中getOutput调用了setInput但未实现，这会导致输入层值无法正确设置，后续计算也会出错。需添加：

setInput(inpts) {
    this.values[0] = [...inpts];
}

修复后的完整核心代码 #

class NeuralNet {
static LEARNING_RATE = 0.33;
biases;
values;
errors;
weights;

constructor(inputSize, hiddenSize, outputSize) {
    this.biases = [
        new Array(hiddenSize).fill(0).map(NeuralNet.random),
        new Array(outputSize).fill(0).map(NeuralNet.random)
    ];
    this.values = [
        new Array(inputSize).fill(0),
        new Array(hiddenSize).fill(0),
        new Array(outputSize).fill(0)
    ];
    // 修正errors结构
    this.errors = [
        new Array(hiddenSize).fill(0),
        new Array(outputSize).fill(0)
    ];
    this.weights = [
        new Array(hiddenSize).fill(0).map(() => new Array(inputSize).fill(0).map(NeuralNet.random)),
        new Array(outputSize).fill(0).map(() => new Array(hiddenSize).fill(0).map(NeuralNet.random))
    ];
}

static random() {
    return Math.random() * 2 - 1;
}

static tanh(x) {
    return Math.tanh(x);
}

// 修正feedForward索引
feedForward() {
    for (var i = 1; i < this.values.length; i++) {
        for (var j = 0; j < this.values[i].length; j++) {
            this.values[i][j] = this.biases[i - 1][j];
            for (var k = 0; k < this.values[i-1].length; k++) {
                this.values[i][j] += this.weights[i - 1][j][k] * this.values[i - 1][k];
            }
            this.values[i][j] = NeuralNet.tanh(this.values[i][j]);
        }
    }
}

// 补充setInput方法
setInput(inpts) {
    this.values[0] = [...inpts];
}

getOutput(inpts) {
    this.setInput(inpts);
    this.feedForward();
    return this.values[this.values.length - 1];
}

// 修正反向传播的权重索引和循环逻辑
backpropagation(expected) {
    for (var i = this.values.length - 1; i > 0; i--) {
        for (var j = 0; j < this.values[i].length; j++) {
            if (i == this.values.length - 1) {
                this.errors[i - 1][j] = (expected[j] - this.values[i][j]) * (1 - this.values[i][j] * this.values[i][j]);
            } else {
                this.errors[i - 1][j] = 0;
                for (var k = 0; k < this.weights[i].length; k++) {
                    this.errors[i - 1][j] += this.weights[i][k][j] * this.errors[i][k];
                }
                this.errors[i - 1][j] *= (1 - this.values[i][j] * this.values[i][j]);
            }
            // 遍历前一层神经元数量
            for (var k = 0; k < this.values[i-1].length; k++) {
                this.weights[i - 1][j][k] += this.errors[i - 1][j] * this.values[i - 1][k] * NeuralNet.LEARNING_RATE;
            }
            this.biases[i - 1][j] += this.errors[i - 1][j] * NeuralNet.LEARNING_RATE;
        }
    }
}
}

内容的提问来源于stack exchange，提问作者Mr Facysus