如何在Keras/PyTorch构建含半固定Gabor滤波器的卷积神经网络？

当然没问题！不管是用PyTorch还是Keras，都能通过自定义卷积层来实现这种「一半固定Gabor滤波器+一半可学习滤波器」的CNN结构，完全贴合你提到的论文思路。下面我分别给出两种框架的具体实现方案：

首先我们需要一个生成Gabor滤波器的工具函数，用来生成固定的滤波器组。Gabor的核心参数（方向、尺度、带宽等）可以根据你的任务需求调整：

import torch
import torch.nn as nn
import numpy as np

def generate_gabor_filters(kernel_size, num_filters, sigma=1.0, lambd=2.0, gamma=0.5):
    filters = []
    # 生成不同方向的Gabor滤波器
    for theta in np.linspace(0, np.pi, num_filters, endpoint=False):
        x, y = np.meshgrid(np.linspace(-(kernel_size//2), kernel_size//2, kernel_size),
                           np.linspace(-(kernel_size//2), kernel_size//2, kernel_size))
        # Gabor滤波器核心公式
        rot_x = x * np.cos(theta) + y * np.sin(theta)
        rot_y = -x * np.sin(theta) + y * np.cos(theta)
        gabor = np.exp(-(rot_x**2 + gamma**2 * rot_y**2)/(2*sigma**2)) * np.cos(2*np.pi*rot_x/lambd)
        # 归一化保证数值稳定
        gabor = (gabor - gabor.mean()) / gabor.std()
        filters.append(gabor)
    # 转为PyTorch卷积要求的张量形状 [num_filters, in_channels, kernel_size, kernel_size]
    filters = np.array(filters)[:, np.newaxis, :, :]
    return torch.tensor(filters, dtype=torch.float32)

接下来自定义混合卷积层，初始化时把一半滤波器设为固定Gabor（禁止梯度更新），另一半设为可学习的随机初始化参数：

class GaborMixedConv2d(nn.Module):
    def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0):
        super().__init__()
        self.in_channels = in_channels
        self.out_channels = out_channels
        self.kernel_size = kernel_size
        self.stride = stride
        self.padding = padding
        
        # 分配固定Gabor和可学习滤波器的数量
        num_gabor = out_channels // 2
        num_learnable = out_channels - num_gabor
        
        # 生成固定Gabor滤波器，设置requires_grad=False避免训练时更新
        self.gabor_filters = nn.Parameter(generate_gabor_filters(kernel_size, num_gabor), requires_grad=False)
        
        # 初始化可学习卷积权重，用Kaiming初始化保证训练稳定性
        self.learnable_weights = nn.Parameter(torch.randn(num_learnable, in_channels, kernel_size, kernel_size))
        nn.init.kaiming_normal_(self.learnable_weights, mode='fan_out', nonlinearity='relu')
        
        # 可选的偏置参数
        self.bias = nn.Parameter(torch.zeros(out_channels)) if out_channels > 0 else None

    def forward(self, x):
        # 处理多通道输入：将单通道Gabor滤波器复制到对应输入通道数
        if self.in_channels > 1:
            gabor_filters = self.gabor_filters.repeat(1, self.in_channels, 1, 1)
        else:
            gabor_filters = self.gabor_filters
        
        # 拼接固定和可学习滤波器
        all_filters = torch.cat([gabor_filters, self.learnable_weights], dim=0)
        
        # 执行卷积操作
        out = nn.functional.conv2d(x, all_filters, bias=self.bias, stride=self.stride, padding=self.padding)
        return out

使用示例 #

# 构建一个简单的分类CNN
model = nn.Sequential(
    GaborMixedConv2d(in_channels=3, out_channels=16, kernel_size=3, padding=1),
    nn.ReLU(),
    nn.MaxPool2d(2),
    nn.Conv2d(16, 32, 3, padding=1),
    nn.ReLU(),
    nn.AdaptiveAvgPool2d(1),
    nn.Flatten(),
    nn.Linear(32, 10)
)

在Keras（TensorFlow后端）中，我们通过自定义Layer类来实现同样的逻辑：

首先还是生成Gabor滤波器的函数，注意要适配Keras的卷积权重格式：

import tensorflow as tf
from tensorflow.keras import layers, Model
import numpy as np

def generate_gabor_filters(kernel_size, num_filters, sigma=1.0, lambd=2.0, gamma=0.5):
    filters = []
    for theta in np.linspace(0, np.pi, num_filters, endpoint=False):
        x, y = np.meshgrid(np.linspace(-(kernel_size//2), kernel_size//2, kernel_size),
                           np.linspace(-(kernel_size//2), kernel_size//2, kernel_size))
        rot_x = x * np.cos(theta) + y * np.sin(theta)
        rot_y = -x * np.sin(theta) + y * np.cos(theta)
        gabor = np.exp(-(rot_x**2 + gamma**2 * rot_y**2)/(2*sigma**2)) * np.cos(2*np.pi*rot_x/lambd)
        gabor = (gabor - gabor.mean()) / gabor.std()
        filters.append(gabor)
    # 转为Keras卷积要求的形状 [kernel_size, kernel_size, in_channels, num_filters]
    filters = np.array(filters).transpose(1,2,0)[..., np.newaxis]
    return tf.convert_to_tensor(filters, dtype=tf.float32)

然后自定义混合卷积层：

class GaborMixedConv2D(layers.Layer):
    def __init__(self, out_channels, kernel_size, stride=1, padding='same', **kwargs):
        super().__init__(**kwargs)
        self.out_channels = out_channels
        self.kernel_size = kernel_size
        self.stride = stride
        self.padding = padding
        
        # 分配固定Gabor和可学习滤波器的数量
        self.num_gabor = out_channels // 2
        self.num_learnable = out_channels - self.num_gabor

    def build(self, input_shape):
        self.in_channels = input_shape[-1]
        
        # 生成固定Gabor滤波器，用tf.stop_gradient禁止梯度更新
        self.gabor_filters = tf.stop_gradient(
            # 复制到对应输入通道数
            tf.repeat(generate_gabor_filters(self.kernel_size, self.num_gabor), self.in_channels, axis=-2)
        )
        
        # 初始化可学习卷积权重
        self.learnable_weights = self.add_weight(
            shape=(self.kernel_size, self.kernel_size, self.in_channels, self.num_learnable),
            initializer='he_normal',
            trainable=True,
            name='learnable_weights'
        )
        
        # 初始化偏置参数
        self.bias = self.add_weight(
            shape=(self.out_channels,),
            initializer='zeros',
            trainable=True,
            name='bias'
        )
        super().build(input_shape)

    def call(self, inputs):
        # 拼接固定和可学习滤波器
        all_filters = tf.concat([self.gabor_filters, self.learnable_weights], axis=-1)
        
        # 执行卷积操作
        return tf.nn.conv2d(
            inputs,
            all_filters,
            strides=[1, self.stride, self.stride, 1],
            padding=self.padding.upper()
        ) + self.bias

使用示例 #

# 构建一个简单的图像分类模型
inputs = layers.Input(shape=(28,28,3))
x = GaborMixedConv2D(16, 3)(inputs)
x = layers.ReLU()(x)
x = layers.MaxPool2D()(x)
x = layers.Conv2D(32, 3, padding='same')(x)
x = layers.ReLU()(x)
x = layers.GlobalAveragePooling2D()(x)
outputs = layers.Dense(10, activation='softmax')(x)

model = Model(inputs, outputs)
model.summary()

内容的提问来源于stack exchange，提问作者Bihy