基于Python从零实现Word2Vec时Loss居高不下的求助

我正在学习Word2Vec模型，尝试基于Python从零实现它。沿用找到的教程代码，仅将训练数据替换为《爱丽丝梦游仙境》第一章内容，但训练时Loss始终维持在30000左右，无法降至1以下。我认为前向传播、反向传播、损失函数定义均无问题，已尝试调整词嵌入维度n、窗口大小、学习率等参数，均未达预期，恳请帮忙排查问题。

以下是我的Jupyter Notebook代码：

#--- IMPORT DEPENDENCIES ------------------------------------------------------+

import numpy as np
import re
from collections import defaultdict

#---Import for graph-----------------------------------------------------------+

import matplotlib.pyplot as plt
from IPython.display import clear_output

#--- CONSTANTS ----------------------------------------------------------------+


class word2vec():
    def __init__ (self):
        self.n = settings['n']
        self.eta = settings['learning_rate']
        self.epochs = settings['epochs']
        self.window = settings['window_size']
        self.losses = [] # added
        pass
    
    
    # GENERATE TRAINING DATA
    def generate_training_data(self, settings, corpus):

        # GENERATE WORD COUNTS
        word_counts = defaultdict(int)
        for row in corpus:
            for word in row:
                word_counts[word] += 1

        self.v_count = len(word_counts.keys())

        # GENERATE LOOKUP DICTIONARIES
        self.words_list = sorted(list(word_counts.keys()),reverse=False)
        self.word_index = dict((word, i) for i, word in enumerate(self.words_list))
        self.index_word = dict((i, word) for i, word in enumerate(self.words_list))

        training_data = []
        # CYCLE THROUGH EACH SENTENCE IN CORPUS
        for sentence in corpus:
            sent_len = len(sentence)

            # CYCLE THROUGH EACH WORD IN SENTENCE
            for i, word in enumerate(sentence):
                
                #w_target  = sentence[i]
                w_target = self.word2onehot(sentence[i])

                # CYCLE THROUGH CONTEXT WINDOW
                w_context = []
                for j in range(i-self.window, i+self.window+1):
                    if j!=i and j<=sent_len-1 and j>=0:
                        w_context.append(self.word2onehot(sentence[j]))
                training_data.append([w_target, w_context])
        return np.array(training_data)


    # SOFTMAX ACTIVATION FUNCTION
    def softmax(self, x):
        e_x = np.exp(x - np.max(x))
        return e_x / e_x.sum(axis=0)


    # CONVERT WORD TO ONE HOT ENCODING
    def word2onehot(self, word):
        word_vec = [0 for i in range(0, self.v_count)]
        word_index = self.word_index[word]
        word_vec[word_index] = 1
        return word_vec


    # FORWARD PASS
    def forward_pass(self, x):
        h = np.dot(self.w1.T, x)
        u = np.dot(self.w2.T, h)
        y_c = self.softmax(u)
        return y_c, h, u
                

    # BACKPROPAGATION
    def backprop(self, e, h, x):
        dl_dw2 = np.outer(h, e)  
        dl_dw1 = np.outer(x, np.dot(self.w2, e.T))

        # UPDATE WEIGHTS
        self.w1 = self.w1 - (self.eta * dl_dw1)
        self.w2 = self.w2 - (self.eta * dl_dw2)
        pass


    # TRAIN W2V model
    def train(self, training_data):
        # INITIALIZE WEIGHT MATRICES
        self.w1 = np.random.uniform(-0.8, 0.8, (self.v_count, self.n))     # embedding matrix
        self.w2 = np.random.uniform(-0.8, 0.8, (self.n, self.v_count))     # context matrix
        
        # CYCLE THROUGH EACH EPOCH
        for i in range(0, self.epochs):

            self.loss = 0

            # CYCLE THROUGH EACH TRAINING SAMPLE
            for w_t, w_c in training_data:

                # FORWARD PASS
                y_pred, h, u = self.forward_pass(w_t)
                
                # CALCULATE ERROR
                EI = np.sum([np.subtract(y_pred, word) for word in w_c], axis=0)

                # BACKPROPAGATION
                self.backprop(EI, h, w_t)

                # CALCULATE LOSS
                self.loss += -np.sum([u[word.index(1)] for word in w_c]) + len(w_c) * np.log(np.sum(np.exp(u)))
                #self.loss += -2*np.log(len(w_c)) -np.sum([u[word.index(1)] for word in w_c]) + (len(w_c) * np.log(np.sum(np.exp(u))))
                
            print('EPOCH:',i, 'LOSS:', self.loss)
            self.losses.append(self.loss)
            self.plot_losses()
        pass
    
    # Add this method
    def plot_losses(self):
        clear_output(wait=True)
        plt.plot(self.losses)
        plt.ylabel('Loss')
        plt.xlabel('Epoch')
        plt.show()
    
    # input a word, returns a vector (if available)
    def word_vec(self, word):
        w_index = self.word_index[word]
        v_w = self.w1[w_index]
        return v_w


    # input a vector, returns nearest word(s)
    def vec_sim(self, vec, top_n):

        # CYCLE THROUGH VOCAB
        word_sim = {}
        for i in range(self.v_count):
            v_w2 = self.w1[i]
            theta_num = np.dot(vec, v_w2)
            theta_den = np.linalg.norm(vec) * np.linalg.norm(v_w2)
            theta = theta_num / theta_den

            word = self.index_word[i]
            word_sim[word] = theta

        words_sorted = sorted(word_sim.items(), key=lambda word_sim: word_sim[1], reverse=True)

        for word, sim in words_sorted[:top_n]:
            print(word, sim)
            
        pass

    # input word, returns top [n] most similar words
    def word_sim(self, word, top_n):
        
        w1_index = self.word_index[word]
        v_w1 = self.w1[w1_index]

        # CYCLE THROUGH VOCAB
        word_sim = {}
        for i in range(self.v_count):
            v_w2 = self.w1[i]
            theta_num = np.dot(v_w1, v_w2)
            theta_den = np.linalg.norm(v_w1) * np.linalg.norm(v_w2)
            theta = theta_num / theta_den

            word = self.index_word[i]
            word_sim[word] = theta

        words_sorted = sorted(word_sim.items(), key=lambda word_sim: word_sim[1], reverse=True)

        for word, sim in words_sorted[:top_n]:
            print(word, sim)
            
        pass

import string
import nltk
nltk.download('punkt')

# 파일 읽기 및 전처리 함수
def preprocess(file_name):
    with open(file_name, 'r') as f:
        text = f.read().lower()
        text = text.translate(str.maketrans('', '', string.punctuation))  # punctuation 제거
        tokenized_text = nltk.word_tokenize(text)  # tokenizing
        return tokenized_text

# 학습시키기 위한 텍스트 파일 불러오기
file_name = 'alice_wonderland_small.txt'  # 이곳에 사용하려는 텍스트 파일 이름 입력
corpus = [preprocess(file_name)]  # 전처리 후 corpus로 사용

settings = {
    'n': 100,  # dimension of word embeddings
    'window_size': 2,  # context window +/- center word
    'min_count': 5,  # minimum word count
    'epochs': 100,  # number of training epochs
    'neg_samp': 5,  # number of negative words to use during training
    'learning_rate': 0.01  # learning rate
}
np.random.seed(0)

w2v = word2vec()

# generate training data
training_data = w2v.generate_training_data(settings, corpus)

# train word2vec model
w2v.train(training_data)

问题排查与优化建议： #

• 训练数据量不足：仅用单章文本的语料规模太小，Word2Vec需要大量上下文关联来学习有效嵌入。建议扩充至整本书或更大规模语料，让模型有足够样本学习词汇间的语义关系。
• 权重初始化范围过大：当前用[-0.8, 0.8]初始化权重，范围过大导致初始Loss极高，优化难度大。建议缩小范围至[-0.01, 0.01]，让模型从更接近最优解的初始状态开始训练。
• 低频词未过滤：配置了min_count:5但代码未实现过滤逻辑，稀有词会增加模型优化难度且无法有效学习嵌入。在generate_training_data中添加逻辑：仅保留出现次数≥min_count的词汇，缩减词汇表规模。
• 学习率与训练轮数调整：当前0.01的学习率对于小规模数据可能收敛过慢，可尝试初始用0.1，随训练轮数逐步衰减（如每轮乘以0.95）；同时增加训练轮数至200-500轮，观察Loss下降趋势。
• 关注Loss趋势而非绝对值：当前Loss是样本损失的累加值，小规模数据下绝对值本身会偏高，更应该看每轮Loss是否持续下降。可以修改Loss计算为平均Loss（总Loss除以训练样本数），更直观判断收敛情况。

内容的提问来源于stack exchange，提问作者James Jang