基于多编码器自定义CLIP模型生成Grad-CAM热力图异常求助

我在为一款自定义CLIP模型生成Grad-CAM热力图时遇到异常：生成的热力图无高亮区域，打印结果显示Grad-CAM值全为0。该模型与标准CLIP结构不同，采用多编码器拼接后输入CLIP。

原代码 #

import tensorflow as tf
from tensorflow import keras
import numpy as np
import cv2
import matplotlib.pyplot as plt

# Assuming you have already trained the clip_model
# Create a new model for Grad-CAM generation
input_image = keras.Input(shape=(312, 312, 3))

global_activations = clip_model.global_vision_encoder(input_image)
local_activations = clip_model.local_vision_encoder(input_image)

# Create separate Grad-CAM models for impression and finding embeddings
impression_gradcam_model = keras.Model(
    inputs=[input_image, clip_model.impression_input],
    outputs=[global_activations, local_activations]
)

finding_gradcam_model = keras.Model(
    inputs=[input_image, clip_model.finding_input],
    outputs=[global_activations, local_activations]
)

# Function to preprocess the input image
def preprocess_image(image_path):
    image = tf.io.read_file(image_path)
    image = tf.image.decode_jpeg(image, channels=3)
    image = tf.image.resize(image, (312, 312))
    image = tf.expand_dims(image, axis=0)
    return image

# Example usage
image_path = r"C:\Users\zahid\Desktop\3130_IM-1472-1001.dcm.png"
impression = "1.No acute cardiopulmonary findings."
finding = "No focal consolidation. No visualized pneumothorax. No large pleural effusions. The heart size and cardiomediastinal silhouette are grossly unremarkable"

# Load and resize the input image
input_image_resized = cv2.resize(cv2.imread(image_path), (312, 312))

def compute_gradcam(model, inputs, activations):
    with tf.GradientTape() as tape:
        tape.watch(inputs[0])
        _, local_activations = model(inputs)
        class_output = tf.reduce_mean(local_activations, axis=[0])
    grads = tape.gradient(class_output, inputs[0])
    pooled_grads = tf.reduce_mean(grads, axis=[0, 1, 2])
    pooled_grads = tf.expand_dims(pooled_grads, axis=-1)
    activations = tf.cast(activations, pooled_grads.dtype)  # Cast activations to the same data type as pooled_grads
    gradcam = tf.reduce_sum(tf.multiply(activations, pooled_grads), axis=[0, 1])
    gradcam = tf.maximum(gradcam, 0)
    gradcam = (gradcam - tf.reduce_min(gradcam)) / (tf.reduce_max(gradcam) - tf.reduce_min(gradcam) + 1e-8)
    return gradcam

impression_inputs = [preprocess_image(image_path), tf.constant([impression], dtype=tf.string)]
_, impression_activations = impression_gradcam_model(impression_inputs)
impression_gradcam = compute_gradcam(impression_gradcam_model, impression_inputs, impression_activations)

finding_inputs = [preprocess_image(image_path), tf.constant([finding], dtype=tf.string)]
_, finding_activations = finding_gradcam_model(finding_inputs)
finding_gradcam = compute_gradcam(finding_gradcam_model, finding_inputs, finding_activations)

print("Impression Grad-CAM values:\n", impression_gradcam)
print("Finding Grad-CAM values:\n", finding_gradcam)

def normalize_gradcam(gradcam):
    gradcam_norm = (gradcam - tf.reduce_min(gradcam)) / (tf.reduce_max(gradcam) - tf.reduce_min(gradcam) + 1e-8)
    return gradcam_norm

impression_gradcam_norm = normalize_gradcam(impression_gradcam)
finding_gradcam_norm = normalize_gradcam(finding_gradcam)

print(f"Impression Grad-CAM heatmap values: min={tf.reduce_min(impression_gradcam_norm):.2f}, max={tf.reduce_max(impression_gradcam_norm):.2f}")
print(f"Finding Grad-CAM heatmap values: min={tf.reduce_min(finding_gradcam_norm):.2f}, max={tf.reduce_max(finding_gradcam_norm):.2f}")

def apply_colormap_and_overlay(gradcam_norm, input_image):
    gradcam_norm = tf.reshape(gradcam_norm, (1, 1))
    gradcam_norm_rgb = tf.image.grayscale_to_rgb(tf.expand_dims(gradcam_norm, axis=-1))
    gradcam_norm_resized = tf.image.resize(gradcam_norm_rgb, (input_image.shape[0], input_image.shape[1]))
    gradcam_heatmap = np.uint8(255 * gradcam_norm_resized.numpy())
    gradcam_overlay = cv2.addWeighted(input_image, 0.8, gradcam_heatmap, 0.5, 0)
    return gradcam_overlay


impression_gradcam_overlay = apply_colormap_and_overlay(impression_gradcam_norm.numpy(), input_image_resized)
finding_gradcam_overlay = apply_colormap_and_overlay(finding_gradcam_norm.numpy(), input_image_resized)

# Display the results
fig, axes = plt.subplots(1, 3, figsize=(15, 5))

axes[0].imshow(input_image_resized)
axes[0].set_title("Input Image")

axes[1].imshow(impression_gradcam_overlay)
axes[1].set_title("Impression Grad-CAM Heatmap")

axes[2].imshow(finding_gradcam_overlay)
axes[2].set_title("Finding Grad-CAM Heatmap")

plt.tight_layout()
plt.show()

原代码输出结果 #

Impression Grad-CAM values:
 tf.Tensor(0.0, shape=(), dtype=float32)
Finding Grad-CAM values:
 tf.Tensor(0.0, shape=(), dtype=float32)
Impression Grad-CAM heatmap values: min=0.00, max=0.00
Finding Grad-CAM heatmap values: min=0.00, max=0.00

异常热力图 #

问题原因与修复方案 #

1. Grad-CAM核心逻辑错误 #

原代码中，梯度计算的目标是local_activations的均值，而非CLIP模型的核心输出——图像与文本的匹配相似度得分。没有关联到任务目标的梯度自然为0。

• 修复：将梯度计算目标替换为图像-文本的相似度得分，这才是模型训练的核心任务指标。

2. Grad-CAM模型定义不完整 #

原Grad-CAM模型仅输出中间激活层，未包含CLIP的文本编码、相似度计算完整链路，导致梯度无法传递到最终任务目标。

• 修复：重新定义模型，让其接收图像+文本输入，同时输出相似度得分和目标激活层（local_activations）。

3. 可视化逻辑错误 #

原代码将Grad-CAM值强制reshape为(1,1)，导致整个热力图为单一值，无法体现空间分布。

• 修复：保留Grad-CAM的空间维度，将其resize到与输入图像一致的尺寸后再生成热力图。

修改后的完整代码 #

import tensorflow as tf
from tensorflow import keras
import numpy as np
import cv2
import matplotlib.pyplot as plt

# 基于已训练的clip_model构建完整Grad-CAM模型
def build_gradcam_model(clip_model):
    input_image = keras.Input(shape=(312, 312, 3))
    text_input = keras.Input(shape=(), dtype=tf.string)
    
    # 执行完整的图像编码流程
    global_feat = clip_model.global_vision_encoder(input_image)
    local_feat = clip_model.local_vision_encoder(input_image)
    image_embedding = tf.concat([global_feat, local_feat], axis=-1)
    
    # 执行完整的文本编码流程
    text_embedding = clip_model.text_encoder(text_input)
    
    # 计算图像-文本相似度（CLIP核心匹配得分）
    similarity = tf.matmul(
        tf.nn.l2_normalize(image_embedding, axis=-1),
        tf.nn.l2_normalize(text_embedding, axis=-1),
        transpose_b=True
    )
    similarity = tf.squeeze(similarity, axis=1)  # 去除冗余维度
    
    # 返回任务目标（相似度）和用于生成热力图的激活层
    return keras.Model(inputs=[input_image, text_input], outputs=[similarity, local_feat])

# 构建两个任务的Grad-CAM模型
impression_gradcam_model = build_gradcam_model(clip_model)
finding_gradcam_model = build_gradcam_model(clip_model)

# 图像预处理（添加CLIP标准预处理）
def preprocess_image(image_path):
    image = tf.io.read_file(image_path)
    image = tf.image.decode_jpeg(image, channels=3)
    image = tf.image.resize(image, (312, 312))
    image = tf.expand_dims(image, axis=0)
    # 适配CLIP的图像预处理要求
    image = tf.keras.applications.clip.preprocess_input(image)
    return image

# 修正后的Grad-CAM计算函数
def compute_gradcam(model, inputs):
    image_input, text_input = inputs
    with tf.GradientTape() as tape:
        tape.watch([image_input, model.output[1]])
        similarity, local_activations = model([image_input, text_input])
        # 以相似度得分为目标，计算对激活层的梯度
        grads = tape.gradient(similarity, local_activations)
    
    # Grad-CAM核心计算：梯度空间均值作为权重
    pooled_grads = tf.reduce_mean(grads, axis=(1, 2))
    # 加权激活层并求和得到热力图
    gradcam = tf.reduce_sum(
        tf.multiply(pooled_grads[..., tf.newaxis, tf.newaxis], local_activations),
        axis=-1
    )
    # ReLU过滤负贡献
    gradcam = tf.maximum(gradcam, 0)
    # 归一化到[0,1]区间
    gradcam = (gradcam - tf.reduce_min(gradcam)) / (tf.reduce_max(gradcam) - tf.reduce_min(gradcam) + 1e-8)
    return tf.squeeze(gradcam)  # 去除batch维度

# 示例数据
image_path = r"C:\Users\zahid\Desktop\3130_IM-1472-1001.dcm.png"
impression = "1.No acute cardiopulmonary findings."
finding = "No focal consolidation. No visualized pneumothorax. No large pleural effusions. The heart size and cardiomediastinal silhouette are grossly unremarkable"

input_image_resized = cv2.resize(cv2.imread(image_path), (312, 312))
input_image_tensor = preprocess_image(image_path)

# 计算Grad-CAM
impression_gradcam = compute_gradcam(impression_gradcam_model, [input_image_tensor, tf.constant([impression])])
finding_gradcam = compute_gradcam(finding_gradcam_model, [input_image_tensor, tf.constant([finding])])

print("Impression Grad-CAM values:\n", impression_gradcam)
print("Finding Grad-CAM values:\n", finding_gradcam)

# 修正后的热力图叠加函数
def apply_colormap_and_overlay(gradcam, input_image):
    # 将Grad-CAM resize到图像尺寸
    gradcam_resized = cv2.resize(gradcam.numpy(), (input_image.shape[1], input_image.shape[0]))
    # 生成彩色热力图
    heatmap = cv2.applyColorMap(np.uint8(255 * gradcam_resized), cv2.COLORMAP_JET)
    # 叠加到原图
    overlay = cv2.addWeighted(input_image, 0.7, heatmap, 0.3, 0)
    return overlay

# 生成并展示结果
impression_overlay = apply_colormap_and_overlay(impression_gradcam, input_image_resized)
finding_overlay = apply_colormap_and_overlay(finding_gradcam, input_image_resized)

fig, axes = plt.subplots(1, 3, figsize=(15, 5))
axes[0].imshow(cv2.cvtColor(input_image_resized, cv2.COLOR_BGR2RGB))
axes[0].set_title("输入图像")
axes[1].imshow(cv2.cvtColor(impression_overlay, cv2.COLOR_BGR2RGB))
axes[1].set_title("Impression Grad-CAM热力图")
axes[2].imshow(cv2.cvtColor(finding_overlay, cv2.COLOR_BGR2RGB))
axes[2].set_title("Finding Grad-CAM热力图")
plt.tight_layout()
plt.show()

内容的提问来源于stack exchange，提问作者Jacob