U-Net音频分类模型Concatenate层形状不匹配问题求助

问题背景 #

尝试用U-Net模型做音频分类，预处理后数据集形状如下：

• X_train shape: (3806, 2809)
• X_test shape: (952, 2809)
• y_train shape: (3806, 1)
• y_test shape: (952, 1)

已尝试padding和cropping操作，但始终存在维度不匹配问题，以下是U-Net架构代码及错误信息：

U-Net代码 #

import tensorflow as tf
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input, Conv2D, MaxPooling2D, UpSampling2D, Concatenate, Dropout, ZeroPadding2D

def unet(input_size=(53, 53, 1), num_classes=2):
    inputs = Input(input_size)
    
    # Encoder
    conv1 = Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(inputs)
    conv1 = Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv1)
    pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
    
    conv2 = Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool1)
    conv2 = Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv2)
    pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
    
    conv3 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool2)
    conv3 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv3)
    pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
    
    conv4 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool3)
    conv4 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv4)
    drop4 = Dropout(0.5)(conv4)
    pool4 = MaxPooling2D(pool_size=(2, 2))(drop4)

    conv5 = Conv2D(1024, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool4)
    conv5 = Conv2D(1024, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv5)
    drop5 = Dropout(0.5)(conv5)

    # Decoder
    up6 = UpSampling2D(size=(2, 2))(drop5)
    conv6 = Conv2D(512, 2, activation='relu', padding='same', kernel_initializer='he_normal')(up6)
    
    # Padding
    pad_top = (conv4.shape[1] - conv6.shape[1]) // 2
    pad_bottom = conv4.shape[1] - conv6.shape[1] - pad_top
    pad_left = (conv4.shape[2] - conv6.shape[2]) // 2
    pad_right = conv4.shape[2] - conv6.shape[2] - pad_left
    padded_conv4 = ZeroPadding2D(padding=((pad_top, pad_bottom), (pad_left, pad_right)))(conv4)
    
    merge6 = Concatenate(axis=3)([padded_conv4, conv6])
    conv6 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge6)
    conv6 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv6)

    up7 = UpSampling2D(size=(2, 2))(conv6)
    conv7 = Conv2D(256, 2, activation='relu', padding='same', kernel_initializer='he_normal')(up7)
    
    # Padding
    pad_top = (conv3.shape[1] - conv7.shape[1]) // 2
    pad_bottom = conv3.shape[1] - conv7.shape[1] - pad_top
    pad_left = (conv3.shape[2] - conv7.shape[2]) // 2
    pad_right = conv3.shape[2] - conv7.shape[2] - pad_left
    padded_conv3 = ZeroPadding2D(padding=((pad_top, pad_bottom), (pad_left, pad_right)))(conv3)
    
    merge7 = Concatenate(axis=3)([padded_conv3, conv7])
    conv7 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge7)
    conv7 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv7)

    up8 = UpSampling2D(size=(2, 2))(conv7)
    conv8 = Conv2D(128, 2, activation='relu', padding='same', kernel_initializer='he_normal')(up8)
    
    # Padding
    pad_top = (conv2.shape[1] - conv8.shape[1]) // 2
    pad_bottom = conv2.shape[1] - conv8.shape[1] - pad_top
    pad_left = (conv2.shape[2] - conv8.shape[2]) // 2
    pad_right = conv2.shape[2] - conv8.shape[2] - pad_left
    padded_conv2 = ZeroPadding2D(padding=((pad_top, pad_bottom), (pad_left, pad_right)))(conv2)
    
    merge8 = Concatenate(axis=3)([padded_conv2, conv8])
    conv8 = Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge8)
    conv8 = Conv2D(128, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv8)

    up9 = UpSampling2D(size=(2, 2))(conv8)
    conv9 = Conv2D(64, 2, activation='relu', padding='same', kernel_initializer='he_normal')(up9)
    
    # Padding
    pad_top = (conv1.shape[1] - conv9.shape[1]) // 2
    pad_bottom = conv1.shape[1] - conv9.shape[1] - pad_top
    pad_left = (conv1.shape[2] - conv9.shape[2]) // 2
    pad_right = conv1.shape[2] - conv9.shape[2] - pad_left
    padded_conv1 = ZeroPadding2D(padding=((pad_top, pad_bottom), (pad_left, pad_right)))(conv1)
    
    merge9 = Concatenate(axis=3)([padded_conv1, conv9])
    conv9 = Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge9)
    conv9 = Conv2D(64, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv9)
    
    conv10 = Conv2D(num_classes, 1, activation='sigmoid')(conv9)  # Use 'sigmoid' for binary classification

    model = Model(inputs=inputs, outputs=conv10)
    
    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
    
    return model

错误信息 #

ValueError                                Traceback (most recent call last)
Cell In[73], line 1
----> 1 model = unet(input_size=(53, 53, 1))

Cell In[72], line 55, in unet(input_size, num_classes)
     52 pad_right = conv3.shape[2] - conv7.shape[2] - pad_left
     53 padded_conv3 = ZeroPadding2D(padding=((pad_top, pad_bottom), (pad_left, pad_right)))(conv3)
---> 55 merge7 = Concatenate(axis=3)([padded_conv3, conv7])
     56 conv7 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge7)
     57 conv7 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv7)

File ~\anaconda3\lib\site-packages\keras\src\utils\traceback_utils.py:70, in filter_traceback.<locals>.error_handler(*args, **kwargs)
     67     filtered_tb = _process_traceback_frames(e.__traceback__)
     68     # To get the full stack trace, call:
     69     # `tf.debugging.disable_traceback_filtering()`
---> 70     raise e.with_traceback(filtered_tb) from None
     71 finally:
     72     del filtered_tb

File ~\anaconda3\lib\site-packages\keras\src\layers\merging\concatenate.py:131, in Concatenate.build(self, input_shape)
    125 unique_dims = set(
    126     shape[axis]
    127     for shape in shape_set
    128     if shape[axis] is not None
    129 )
    130 if len(unique_dims) > 1:
---> 131     raise ValueError(err_msg)

ValueError: A `Concatenate` layer requires inputs with matching shapes except for the concatenation axis. Received: input_shape=[(None, 14, 14, 256), (None, 12, 12, 256)]

错误原因 #

错误核心是padding对象搞反了：当前代码给encoder输出的特征图（比如conv3）加padding，导致它的尺寸比decoder上采样后的特征图（conv7）更大，无法拼接。正确逻辑应该是给decoder的特征图加padding，让它和encoder对应层的尺寸一致，或者对encoder特征图做cropping。

另外，输入尺寸53x53不是2的幂次，经过多次下采样/上采样后，尺寸无法完美匹配，加剧了维度问题。

解决方案 #

方案1：修正padding对象 #

将所有padding操作从encoder特征图转移到decoder特征图，确保拼接时两者尺寸一致。修改后的Decoder部分代码如下：

# Decoder
up6 = UpSampling2D(size=(2, 2))(drop5)
conv6 = Conv2D(512, 2, activation='relu', padding='same', kernel_initializer='he_normal')(up6)

# 修正：给decoder的conv6加padding，匹配conv4尺寸
pad_top = (conv4.shape[1] - conv6.shape[1]) // 2
pad_bottom = conv4.shape[1] - conv6.shape[1] - pad_top
pad_left = (conv4.shape[2] - conv6.shape[2]) // 2
pad_right = conv4.shape[2] - conv6.shape[2] - pad_left
padded_conv6 = ZeroPadding2D(padding=((pad_top, pad_bottom), (pad_left, pad_right)))(conv6)

merge6 = Concatenate(axis=3)([conv4, padded_conv6])
conv6 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge6)
conv6 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv6)

up7 = UpSampling2D(size=(2, 2))(conv6)
conv7 = Conv2D(256, 2, activation='relu', padding='same', kernel_initializer='he_normal')(up7)

# 修正：给decoder的conv7加padding，匹配conv3尺寸
pad_top = (conv3.shape[1] - conv7.shape[1]) // 2
pad_bottom = conv3.shape[1] - conv7.shape[1] - pad_top
pad_left = (conv3.shape[2] - conv7.shape[2]) // 2
pad_right = conv3.shape[2] - conv7.shape[2] - pad_left
padded_conv7 = ZeroPadding2D(padding=((pad_top, pad_bottom), (pad_left, pad_right)))(conv7)

merge7 = Concatenate(axis=3)([conv3, padded_conv7])
conv7 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge7)
conv7 = Conv2D(256, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv7)

# 后续merge8、merge9做同样修改，给conv8、conv9加padding

方案2：改用Conv2DTranspose替代UpSampling2D #

Conv2DTranspose可以直接控制输出尺寸，避免手动计算padding的麻烦。示例如下：

# 替换Decoder中的UpSampling2D+Conv2D为Conv2DTranspose
up6 = Conv2DTranspose(512, (2,2), strides=(2,2), padding='same', kernel_initializer='he_normal')(drop5)
# 直接和conv4拼接，无需额外padding
merge6 = Concatenate(axis=3)([conv4, up6])
conv6 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge6)
conv6 = Conv2D(512, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv6)

方案3：调整输入尺寸为2的幂次 #

将输入尺寸改为64x64（2的6次方），这样每次下采样和上采样的尺寸都是整数倍，无需额外padding/cropping：

def unet(input_size=(64, 64, 1), num_classes=2):
    # 后续代码不变

同时需要将预处理后的音频数据（2809维度）reshape为64x64，可通过补零或裁剪实现：

# 补零示例
X_train = np.pad(X_train, ((0,0), (0, 64*64 - 2809)), mode='constant')
X_train = X_train.reshape(-1, 64, 64, 1)

额外提示 #

音频分类任务中，通常先将一维音频转换为二维频谱图（比如Mel频谱图）再输入CNN，直接用一维数据reshape为二维可能丢失时序信息，建议先做频谱转换。

内容的提问来源于stack exchange，提问作者Sunflower_Poet