大学项目Chatbot训练报错ValueError：未知损失函数categorical crossentropy

问题描述 #

我正在为大学项目搭建Chatbot，跟着YouTube教程学习且毫无相关经验。此前代码运行正常，添加random.shuffle后出现ValueError，报错信息为“Unknown loss function: categorical crossentropy. Please ensure this object is passed to the custom_objects argument”。试过谷歌上的修复方法但无效，怀疑需要重装TensorFlow，希望得到帮助。

运行日志 #

C:\Users\Kimbe\.conda\envs\tf.2\python.exe C:\Users\Kimbe\PycharmProjects\chatbot\training.py 
C:\Users\Kimbe\PycharmProjects\chatbot\training.py:53: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.
  training = np.array(training)
2022-11-23 21:38:00.366897: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'nvcuda.dll'; dlerror: nvcuda.dll not found
2022-11-23 21:38:00.367881: W tensorflow/stream_executor/cuda/cuda_driver.cc:263] failed call to cuInit: UNKNOWN ERROR (303)
2022-11-23 21:38:00.371587: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:169] retrieving CUDA diagnostic information for host: Kims-Surface
2022-11-23 21:38:00.371782: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:176] hostname: Kims-Surface
2022-11-23 21:38:00.372191: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX AVX2
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
C:\Users\Kimbe\.conda\envs\tf.2\lib\site-packages\keras\optimizers\optimizer_v2\gradient_descent.py:111: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.
  super().__init__(name, **kwargs)
Epoch 1/200
Traceback (most recent call last):
  File "C:\Users\Kimbe\PycharmProjects\chatbot\training.py", line 69, in <module>
    model.fit(np.array(train_x), np.array(train_y), epochs=200, batch_size=5, verbose=1)
  File "C:\Users\Kimbe\.conda\envs\tf.2\lib\site-packages\keras\utils\traceback_utils.py", line 70, in error_handler
    raise e.with_traceback(filtered_tb) from None
  File "C:\Users\Kimbe\AppData\Local\Temp\__autograph_generated_filecynafcyn.py", line 15, in tf__train_function
    retval_ = ag__.converted_call(ag__.ld(step_function), (ag__.ld(self), ag__.ld(iterator)), None, fscope)
ValueError: in user code:

    File "C:\Users\Kimbe\.conda\envs\tf.2\lib\site-packages\keras\engine\training.py", line 1160, in train_function  *
        return step_function(self, iterator)
    File "C:\Users\Kimbe\.conda\envs\tf.2\lib\site-packages\keras\engine\training.py", line 1146, in step_function  **
        outputs = model.distribute_strategy.run(run_step, args=(data,))
    File "C:\Users\Kimbe\.conda\envs\tf.2\lib\site-packages\keras\engine\training.py", line 1135, in run_step  **
        outputs = model.train_step(data)
    File "C:\Users\Kimbe\.conda\envs\tf.2\lib\site-packages\keras\engine\training.py", line 994, in train_step
        loss = self.compute_loss(x, y, y_pred, sample_weight)
    File "C:\Users\Kimbe\.conda\envs\tf.2\lib\site-packages\keras\engine\training.py", line 1052, in compute_loss
        return self.compiled_loss(
    File "C:\Users\Kimbe\.conda\envs\tf.2\lib\site-packages\keras\engine\compile_utils.py", line 240, in __call__
        self.build(y_pred)
    File "C:\Users\Kimbe\.conda\envs\tf.2\lib\site-packages\keras\engine\compile_utils.py", line 182, in build
        self._losses = tf.nest.map_structure(
    File "C:\Users\Kimbe\.conda\envs\tf.2\lib\site-packages\keras\engine\compile_utils.py", line 353, in _get_loss_object
        loss = losses_mod.get(loss)
    File "C:\Users\Kimbe\.conda\envs\tf.2\lib\site-packages\keras\losses.py", line 2649, in get
        return deserialize(identifier)
    File "C:\Users\Kimbe\.conda\envs\tf.2\lib\site-packages\keras\losses.py", line 2603, in deserialize
        return deserialize_keras_object(
    File "C:\Users\Kimbe\.conda\envs\tf.2\lib\site-packages\keras\utils\generic_utils.py", line 769, in deserialize_keras_object
        raise ValueError(

    ValueError: Unknown loss function: categorical crossentropy. Please ensure this object is passed to the `custom_objects` argument. See https://www.tensorflow.org/guide/keras/save_and_serialize#registering_the_custom_object for details.


Process finished with exit code 1

原代码 #

import random
import json
import pickle
import numpy as np

import nltk
from nltk.stem import WordNetLemmatizer

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Activation, Dropout
from tensorflow.keras.optimizers import SGD

lemmatizer = WordNetLemmatizer()

intents = json.loads(open('intents.json').read())

words = []
classes = []
documents = []
ignore_letters = ['?', '!', '.', ',']

for intent in intents['intents']:
    for pattern in intent['patterns']:
        word_list = nltk.word_tokenize(pattern)
        words.extend(word_list)
        documents.append((word_list, intent['tag']))
        if intent['tag'] not in classes:
            classes.append(intent['tag'])

words = [lemmatizer.lemmatize(word) for word in words if word not in ignore_letters]
words = sorted(set(words))

classes = sorted(set(classes))

pickle.dump(words, open('words.pkl', 'wb'))
pickle.dump(words, open('classes.pkl', 'wb'))

training = []
output_empty = [0] * len(classes)

for document in documents:
    bag = []
    word_patterns = document[0]
    word_patterns = [lemmatizer.lemmatize(word.lower()) for word in word_patterns]
    for word in words:
        bag.append(1) if word in word_patterns else bag.append(0)

    output_row = list(output_empty)
    output_row[classes.index(document[1])] = 1
    training.append([bag, output_row])

    random.shuffle(training)
    training = np.array(training)


    train_x = list(training[:, 0])
    train_y = list(training[:, 1])

    model = Sequential()
    model.add(Dense(128, input_shape=(len(train_x[0]),), activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(64, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(len(train_y[0]), activation='softmax'))

    sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
    model.compile(loss='categorical crossentropy', optimizer=sgd, metrics=['accuracy'])

    model.fit(np.array(train_x), np.array(train_y), epochs=200, batch_size=5, verbose=1)
    model.save('Chatbot_model.model')

    print("Done")

解决方案 #

你的代码存在几个关键问题，不需要重装TensorFlow，修正后即可运行：

1. 缩进错误（核心问题） #

你把random.shuffle、数组转换、模型构建训练等代码都写在了遍历documents的for循环内部，导致每处理一条数据就打乱一次、重新创建模型并开始训练，完全不符合训练逻辑。这些代码应该放在for循环外面，等所有训练数据生成完成后再执行。

2. Loss函数名称错误 #

报错的直接原因是loss函数名称写错了：应该是categorical_crossentropy（下划线连接），而不是categorical crossentropy（空格）。

3. 其他小问题 #

• 保存classes到pickle时写错了，把classes写成了words，导致后续加载会出错；
• SGD的lr参数已过时，应该用learning_rate；
• 创建numpy数组时的警告，需要指定dtype=object来处理不规则序列。

修改后的完整代码 #

import random
import json
import pickle
import numpy as np

import nltk
from nltk.stem import WordNetLemmatizer

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Activation, Dropout
from tensorflow.keras.optimizers import SGD

lemmatizer = WordNetLemmatizer()

intents = json.loads(open('intents.json').read())

words = []
classes = []
documents = []
ignore_letters = ['?', '!', '.', ',']

for intent in intents['intents']:
    for pattern in intent['patterns']:
        word_list = nltk.word_tokenize(pattern)
        words.extend(word_list)
        documents.append((word_list, intent['tag']))
        if intent['tag'] not in classes:
            classes.append(intent['tag'])

words = [lemmatizer.lemmatize(word) for word in words if word not in ignore_letters]
words = sorted(set(words))

classes = sorted(set(classes))

# 修正：保存classes而不是words
pickle.dump(words, open('words.pkl', 'wb'))
pickle.dump(classes, open('classes.pkl', 'wb'))

training = []
output_empty = [0] * len(classes)

# 只在循环内生成训练数据
for document in documents:
    bag = []
    word_patterns = document[0]
    word_patterns = [lemmatizer.lemmatize(word.lower()) for word in word_patterns]
    for word in words:
        bag.append(1) if word in word_patterns else bag.append(0)

    output_row = list(output_empty)
    output_row[classes.index(document[1])] = 1
    training.append([bag, output_row])

# 把这些操作移到循环外面
random.shuffle(training)
# 修正：指定dtype=object消除警告
training = np.array(training, dtype=object)

train_x = list(training[:, 0])
train_y = list(training[:, 1])

model = Sequential()
model.add(Dense(128, input_shape=(len(train_x[0]),), activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(64, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(len(train_y[0]), activation='softmax'))

# 修正：用learning_rate替代过时的lr参数
sgd = SGD(learning_rate=0.01, decay=1e-6, momentum=0.9, nesterov=True)
# 修正：loss函数名称改为categorical_crossentropy
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])

model.fit(np.array(train_x), np.array(train_y), epochs=200, batch_size=5, verbose=1)
model.save('Chatbot_model.model')

print("Done")

内容的提问来源于stack exchange，提问作者Sharqzzzz