TensorFlow训练神经网络遇ValueError维度越界等问题求助

问题背景 #

在WSL环境使用TensorFlow 2.12完成Kaggle Intel图像分类任务时，训练阶段出现报错。以下是完整代码及报错信息：

导入、标签与数据路径代码 #

import tensorflow as tf
import os
from tqdm import tqdm
import numpy as np

# 标签
labels = ["buildings", "forest", "glacier", "mountain", "sea", "street"]

# 数据目录
train_data = os.path.join(os.getcwd(), "data", "seg_train", "seg_train")
test_data = os.path.join(os.getcwd(), "data", "seg_test", "seg_test")

# 输入数据形状（需包含颜色通道作为最后一维）
data_shape = (150, 150, 3)

# 构建文件路径字典
train_dirs = {}
test_dirs = {}

for label in labels:
    train_dirs[label] = os.path.join(train_data, label)
    test_dirs[label] = os.path.join(test_data, label)

数据处理辅助函数 #

# 整理带标签的文件列表，移除不符合目标形状的文件
def remove_bad_data(dirs_dict):
    indexed_files = []
    for idx, directory in enumerate(dirs_dict):
        listfiles = os.listdir(dirs_dict[directory])
        for file in listfiles:
            image_path = os.path.join(dirs_dict[directory], file)
            image = tf.io.read_file(image_path)
            image = tf.io.decode_image(image, channels=data_shape[-1])
            if image.shape == data_shape:
                indexed_files.append((image_path, idx))
            else:
                os.remove(image_path)
                print(f"移除形状为 {image.shape} 的文件：{image_path}")
    return indexed_files

# 图像预处理函数
def preprocess_images(image_path, label):
    image = tf.io.read_file(image_path)
    image = tf.io.decode_image(image, channels=data_shape[-1])/255
    return (image, label)

# 数据集流水线函数
def dataset_pipeline(dataset):
    dataset = dataset.map(map_func=preprocess_images)
    dataset = dataset.cache()
    dataset = dataset.shuffle(buffer_size=len(dataset))
    dataset = dataset.prefetch(buffer_size=tf.data.AUTOTUNE)
    return dataset

数据集创建 #

train_files = remove_bad_data(train_dirs)
test_files = remove_bad_data(test_dirs)

# 创建数据集
train_files = np.array(train_files)
train_dataset = tf.data.Dataset.from_tensor_slices((train_files[:,0].astype(str), train_files[:,1].astype(int)))
train_dataset = dataset_pipeline(dataset=train_dataset)

此时train_dataset.as_numpy_iterator().next()返回(150, 150, 3)数组和0-5的整数标签（注：原描述中0-6有误，标签共6类，索引应为0-5）。

模型定义与训练 #

# 定义模型架构
input_layer = tf.keras.Input(shape=data_shape, name="input_layer")
conv_layer = tf.keras.layers.Conv2D(filters=16, kernel_size=3, padding="same", activation="relu", name="first_conv2d")(input_layer)
maxpool_layer = tf.keras.layers.MaxPool2D(name="first_maxpool")(conv_layer)
conv_layer = tf.keras.layers.Conv2D(filters=32, kernel_size=3, padding="same", activation="relu", name="second_conv2d")(maxpool_layer)
maxpool_layer = tf.keras.layers.MaxPool2D(name="second_maxpool")(conv_layer)
conv_layer = tf.keras.layers.Conv2D(filters=64, kernel_size=3, padding="same", activation="relu", name="third_conv2d")(maxpool_layer)
maxpool_layer = tf.keras.layers.MaxPool2D(name="third_maxpool")(conv_layer)
conv_layer = tf.keras.layers.Conv2D(filters=128, kernel_size=3, padding="same", activation="relu", name="fourth_conv2d")(maxpool_layer)
maxpool_layer = tf.keras.layers.MaxPool2D(name="fourth_maxpool")(conv_layer)
flatten_layer = tf.keras.layers.Flatten(name="flatten_layer")(maxpool_layer)
dense_layer = tf.keras.layers.Dense(units=128, activation="relu")(flatten_layer)
output_layer = tf.keras.layers.Dense(units=len(labels), name="output_layer")(dense_layer)

# 初始化模型
model = tf.keras.Model(inputs=input_layer, outputs=output_layer, name="intel_classification")
model.compile(optimizer='adam', loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy'])
tf.keras.utils.plot_model(model, "model.png", show_shapes=True)

history = model.fit(train_dataset)

报错情况 #

1. 使用SparseCategoricalCrossentropy(from_logits=True)时报错：

ValueError: slice index 0 of dimension 0 out of bounds. for '{{node strided_slice}} = StridedSlice[Index=DT_INT32, T=DT_INT32, begin_mask=0, ellipsis_mask=0, end_mask=0, new_axis_mask=0, shrink_axis_mask=1](Shape, strided_slice/stack, strided_slice/stack_1, strided_slice/stack_2)' with input shapes: [0], [1], [1], [1] and with computed input tensors: input[1] = <0>, input[2] = <1>, input[3] = <1>.

2. 改用CategoricalCrossentropy()时报错：

2023-04-18 15:25:56.800730: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_1' with dtype int64 and shape [13986]
         [[{{node Placeholder/_1}}]]
2023-04-18 15:25:56.800956: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'Placeholder/_1' with dtype int64 and shape [13986]
         [[{{node Placeholder/_1}}]]
Traceback (most recent call last):
  File "/home/jw/projects/testmodels/main.py", line 109, in <module>
    history = model.fit(train_dataset)
  File "/home/jw/projects/testmodels/venv/lib/python3.10/site-packages/keras/utils/traceback_utils.py", line 70, in error_handler
    raise e.with_traceback(filtered_tb) from None
  File "/tmp/__autograph_generated_file7w1vt_rv.py", line 15, in tf__train_function
    retval_ = ag__.converted_call(ag__.ld(step_function), (ag__.ld(self), ag__.ld(iterator)), None, fscope)
ValueError: in user code:

    File "/home/jw/projects/testmodels/venv/lib/python3.10/site-packages/keras/engine/training.py", line 1284, in train_function  *
        return step_function(self, iterator)
    File "/home/jw/projects/testmodels/venv/lib/python3.10/site-packages/keras/engine/training.py", line 1268, in step_function  **
        outputs = model.distribute_strategy.run(run_step, args=(data,))
    File "/home/jw/projects/testmodels/venv/lib/python3.10/site-packages/keras/engine/training.py", line 1249, in run_step  **
        outputs = model.train_step(data)
    File "/home/jw/projects/testmodels/venv/lib/python3.10/site-packages/keras/engine/training.py", line 1051, in train_step
        loss = self.compute_loss(x, y, y_pred, sample_weight)
    File "/home/jw/projects/testmodels/venv/lib/python3.10/site-packages/keras/engine/training.py", line 1109, in compute_loss
        return self.compiled_loss(
    File "/home/jw/projects/testmodels/venv/lib/python3.10/site-packages/keras/engine/compile_utils.py", line 265, in __call__
        loss_value = loss_obj(y_t, y_p, sample_weight=sw)
    File "/home/jw/projects/testmodels/venv/lib/python3.10/site-packages/keras/losses.py", line 142, in __call__
        losses = call_fn(y_true, y_pred)
    File "/home/jw/projects/testmodels/venv/lib/python3.10/site-packages/keras/losses.py", line 268, in call  **
        return ag_fn(y_true, y_pred, **self._fn_kwargs)
    File "/home/jw/projects/testmodels/venv/lib/python3.10/site-packages/keras/losses.py", line 1984, in categorical_crossentropy
        return backend.categorical_crossentropy(
    File "/home/jw/projects/testmodels/venv/lib/python3.10/site-packages/keras/backend.py", line 5559, in categorical_crossentropy
        target.shape.assert_is_compatible_with(output.shape)

    ValueError: Shapes () and (None, 6) are incompatible

问题分析 #

1. 第一个报错根源：数据集未设置批量（batch）。model.fit期望输入是批量数据（图像shape为(batch_size, 150, 150, 3)，标签shape为(batch_size,)），但当前数据集输出的是单个样本（图像shape(150,150,3)，标签shape()），导致TensorFlow内部处理时出现维度越界。
2. 第二个报错根源：损失函数与标签类型不匹配。CategoricalCrossentropy要求标签为one-hot编码格式（shape(batch_size, 6)），但当前标签是整数类型（shape(batch_size,)），因此出现形状不兼容错误。而SparseCategoricalCrossentropy正是为整数标签设计的，原本的损失函数选择是正确的，只需解决批量问题即可。

修复方案 #

方案1：修复数据集流水线，添加批量操作（推荐） #

修改dataset_pipeline函数，添加batch操作，同时优化shuffle的buffer_size（使用固定值避免内存占用过高）：

# 定义数据集流水线函数
def dataset_pipeline(dataset, batch_size=32):
    dataset = dataset.map(map_func=preprocess_images)
    dataset = dataset.cache()
    dataset = dataset.shuffle(buffer_size=1000)  # 用固定值替代len(dataset)，减少内存压力
    dataset = dataset.batch(batch_size)  # 添加批量操作
    dataset = dataset.prefetch(buffer_size=tf.data.AUTOTUNE)
    return dataset

调用时指定批量大小（或使用默认值）：

train_dataset = dataset_pipeline(dataset=train_dataset, batch_size=32)

保持原损失函数SparseCategoricalCrossentropy(from_logits=True)不变，重新运行训练即可。

方案2：改用CategoricalCrossentropy（需转换标签格式） #

若坚持使用CategoricalCrossentropy，需将整数标签转为one-hot编码，并为模型输出层添加softmax激活：

1. 修改preprocess_images函数，将标签转为one-hot：

def preprocess_images(image_path, label):
    image = tf.io.read_file(image_path)
    image = tf.io.decode_image(image, channels=data_shape[-1])/255
    label = tf.one_hot(label, depth=len(labels))  # 转为one-hot编码
    return (image, label)

2. 修改模型输出层，添加softmax激活：

output_layer = tf.keras.layers.Dense(units=len(labels), activation="softmax", name="output_layer")(dense_layer)

3. 编译模型时使用CategoricalCrossentropy()：

model.compile(optimizer='adam', loss=tf.keras.losses.CategoricalCrossentropy(), metrics=['accuracy'])

同时也要为数据集添加batch操作（同方案1）。

完整修复后代码示例（方案1） #

修改后的数据集流水线函数和训练代码：

# 定义数据集流水线函数
def dataset_pipeline(dataset, batch_size=32):
    dataset = dataset.map(map_func=preprocess_images)
    dataset = dataset.cache()
    dataset = dataset.shuffle(buffer_size=1000)
    dataset = dataset.batch(batch_size)
    dataset = dataset.prefetch(buffer_size=tf.data.AUTOTUNE)
    return dataset

# 创建数据集
train_files = remove_bad_data(train_dirs)
test_files = remove_bad_data(test_dirs)
train_files = np.array(train_files)
train_dataset = tf.data.Dataset.from_tensor_slices((train_files[:,0].astype(str), train_files[:,1].astype(int)))
train_dataset = dataset_pipeline(dataset=train_dataset, batch_size=32)

# 模型定义与训练（保持不变）
input_layer = tf.keras.Input(shape=data_shape, name="input_layer")
conv_layer = tf.keras.layers.Conv2D(filters=16, kernel_size=3, padding="same", activation="relu", name="first_conv2d")(input_layer)
maxpool_layer = tf.keras.layers.MaxPool2D(name="first_maxpool")(conv_layer)
conv_layer = tf.keras.layers.Conv2D(filters=32, kernel_size=3, padding="same", activation="relu", name="second_conv2d")(maxpool_layer)
maxpool_layer = tf.keras.layers.MaxPool2D(name="second_maxpool")(conv_layer)
conv_layer = tf.keras.layers.Conv2D(filters=64, kernel_size=3, padding="same", activation="relu", name="third_conv2d")(maxpool_layer)
maxpool_layer = tf.keras.layers.MaxPool2D(name="third_maxpool")(conv_layer)
conv_layer = tf.keras.layers.Conv2D(filters=128, kernel_size=3, padding="same", activation="relu", name="fourth_conv2d")(maxpool_layer)
maxpool_layer = tf.keras.layers.MaxPool2D(name="fourth_maxpool")(conv_layer)
flatten_layer = tf.keras.layers.Flatten(name="flatten_layer")(maxpool_layer)
dense_layer = tf.keras.layers.Dense(units=128, activation="relu")(flatten_layer)
output_layer = tf.keras.layers.Dense(units=len(labels), name="output_layer")(dense_layer)

model = tf.keras.Model(inputs=input_layer, outputs=output_layer, name="intel_classification")
model.compile(optimizer='adam', loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy'])
tf.keras.utils.plot_model(model, "model.png", show_shapes=True)

history = model.fit(train_dataset, epochs=10)  # 可指定训练轮数

内容的提问来源于stack exchange，提问作者Dante