OpenCV视频背景替换：解决人物边缘白边问题

我正在使用OpenCV逐帧进行视频背景替换操作，但遇到人物边缘出现白边的问题。已尝试使用膨胀（dilation）和高斯模糊（Gaussian blur）方法，但未能解决该问题。以下是完整的复现代码：

import cv2
from rembg import remove
from PIL import Image, ImageFilter
import os
import numpy as np
from moviepy.editor import VideoFileClip, ImageSequenceClip

def smooth_edges(image):
    # Apply Gaussian blur to the alpha channel for smoothing
    alpha_channel = image[:, :, 3]
    blurred_alpha = cv2.GaussianBlur(alpha_channel, (15, 15), 0)

    # Apply dilation to the alpha channel to reduce white borders
    kernel = np.ones((5, 5), np.uint8)
    dilated_alpha = cv2.dilate(alpha_channel, kernel, iterations=1)

    # Update the alpha channel with the smoothed and dilated alpha
    image[:, :, 3] = dilated_alpha

    return image

def process_video_with_audio(input_video_path, output_video_path='output_video.mp4', frame_limit=40):
    # Check if the video file exists
    if not os.path.exists(input_video_path):
        print(f"Video file '{input_video_path}' not found.")
        return None

    # Open the video capture
    video_capture = cv2.VideoCapture(input_video_path)

    # Check if the video capture is open
    if not video_capture.isOpened():
        print(f"Failed to open the video '{input_video_path}'.")
        return None

    frame_count = 0  # Initialize a frame count
    frame_list = []  # List to store processed frames

    while frame_count < frame_limit:
        # Read one frame from the video
        ret, frame = video_capture.read()

        # Check if the frame was successfully read
        if not ret:
            print(f"Failed to read frame {frame_count} from the video.")
            break  # Exit the loop if there are no more frames

        # Specify the output path for the processed frame
        frame_name = f'frame{frame_count}.png'
        frame_path = 'masked/' + frame_name

        # Save the frame as an image
        cv2.imwrite(frame_path, frame)

        # Output path for the processed image
        output_image_path = f'masked/processed_frame{frame_count}.png'

        # Save the frame as an image before processing it
        cv2.imwrite(output_image_path, frame)

        with open(output_image_path, 'rb') as f:
            input_image = f.read()
            subject = remove(input_image, alpha_matting=True, alpha_matting_foreground_threshold=50)
            with open(f'masked/background{frame_count}.png', 'wb') as output_file:
                output_file.write(subject)

        background_img_path = 'Cover.jpg'

        if os.path.exists(background_img_path):
            background_img = Image.open(background_img_path)

            # Load the processed frame as the foreground image
            foreground_img = Image.open(f'masked/background{frame_count}.png').convert("RGBA")

            # Resize the background image to match the dimensions of the video frame
            background_img = background_img.resize((foreground_img.width, foreground_img.height))

            # Ensure the foreground image has an 'RGBA' mode with an alpha channel
            foreground_img = foreground_img.convert("RGBA")

            # Apply Gaussian blur to the alpha channel for smoothing
            blurred_alpha = foreground_img.split()[3].filter(ImageFilter.GaussianBlur(radius=5))

            # Apply dilation to the alpha channel to reduce white borders
            dilated_alpha = np.array(blurred_alpha)
            kernel = np.ones((5, 5), np.uint8)
            dilated_alpha = cv2.dilate(dilated_alpha, kernel, iterations=1)
            blurred_alpha = Image.fromarray(dilated_alpha)

            # Composite the images with the smoothed and dilated alpha channel
            composite_img = Image.merge("RGBA", foreground_img.split()[:3] + (blurred_alpha,))

            # Save the final composite image as PNG (supports transparency)
            result = Image.alpha_composite(background_img.convert("RGBA"), composite_img)

            # Append the processed frame to the list
            frame_list.append(np.array(result))  # Convert to NumPy array

            frame_count += 1  # Increment the frame count

            # DEBUG: Print statement for debugging
            print(f"Processing frame {frame_count}...")

        else:
            print(f"Background image '{background_img_path}' not found")

    # Check if there are processed frames to create a video
    if frame_list:
        # Get the audio from the original video
        original_clip = VideoFileClip(input_video_path)
        audio = original_clip.audio

        # Create an ImageSequenceClip from the processed frames
        processed_clip = ImageSequenceClip(frame_list, fps=30)

        # Set the audio of the processed video to the original audio
        processed_clip = processed_clip.set_audio(audio)

        # Set the duration of the processed video to match the specified number of frames
        processed_clip = processed_clip.subclip(0, processed_clip.duration)

        # Save the final video with audio using MoviePy
        processed_clip.write_videofile(output_video_path, codec='libx264', audio_codec='aac')

        print(f"Video created at '{output_video_path}' with audio.")
        return output_video_path
    else:
        print("No processed frames to create a video.")
        return None

# Example usage:
video_path = 'alex.mp4'  # Change this to the actual path of your video
output_path = process_video_with_audio(video_path)

# You can use the 'output_path' variable as needed.

可行解决方案 #

白边问题根源是抠图后的边缘残留原背景像素，或是alpha通道处理逻辑不当，以下是针对性调整方案：

1. 优化rembg抠图参数

当前alpha_matting的阈值设置不够精准，调高前景阈值、增加腐蚀尺寸，能更准确分离前景和背景：

# 替换原rembg调用代码
subject = remove(input_image, alpha_matting=True, 
                 alpha_matting_foreground_threshold=80,
                 alpha_matting_background_threshold=20,
                 alpha_matting_erode_size=10)

2. 调整alpha通道处理顺序

原代码先模糊再膨胀会放大白边，应该先腐蚀消除边缘亮边，再模糊柔化：

# 替换原前景alpha处理代码
alpha_channel = np.array(foreground_img.split()[3])
# 腐蚀消除白边
kernel = np.ones((3,3), np.uint8)
eroded_alpha = cv2.erode(alpha_channel, kernel, iterations=1)
# 高斯模糊柔化边缘
blurred_alpha = cv2.GaussianBlur(eroded_alpha, (7,7), 0)
blurred_alpha = Image.fromarray(blurred_alpha)

3. 边缘颜色融合

通过加权混合前景与背景的边缘像素，彻底消除白边：

# 在合成前添加此步骤
foreground_rgb = np.array(foreground_img.split()[:3])
background_rgb = np.array(background_img.convert("RGB"))
alpha = np.array(blurred_alpha) / 255.0

# 按alpha通道加权混合，消除边缘色差
blended_rgb = foreground_rgb * alpha + background_rgb * (1 - alpha)
blended_rgb = blended_rgb.astype(np.uint8)

# 重新合成带柔化边缘的前景
composite_img = Image.merge("RGBA", (
    Image.fromarray(blended_rgb[0]),
    Image.fromarray(blended_rgb[1]),
    Image.fromarray(blended_rgb[2]),
    blurred_alpha
))

4. 简化冗余步骤

原代码多次读写本地图片会损失画质且降低效率，建议直接在内存中处理帧：

# 替换原帧保存和rembg读取代码
# 把OpenCV帧转成PIL图像
pil_frame = Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
# 转成字节流传给rembg
import io
buffer = io.BytesIO()
pil_frame.save(buffer, format='PNG')
buffer.seek(0)
subject = remove(buffer.read(), alpha_matting=True, 
                 alpha_matting_foreground_threshold=80,
                 alpha_matting_background_threshold=20,
                 alpha_matting_erode_size=10)
# 直接读取处理后的字节流，不用存文件
foreground_img = Image.open(io.BytesIO(subject)).convert("RGBA")

内容的提问来源于stack exchange，提问作者Osama