使用PyAV无法正确解码摄像头采集图像帧的问题求助

我正在尝试对摄像头采集的帧进行H.264编码和解码，后续打算通过TCP传输帧数据，但本地测试时就出现了问题——运行代码后，源视频和解码后的视频显示均异常。

以下是我的实现代码：

import struct
import cv2
import socket
import av
import time
import os

class PerfTimer:
    def __init__(self, name):
        self.name = name

    def __enter__(self):
        self.start_time = time.perf_counter()

    def __exit__(self, type, value, traceback):
        end_time = time.perf_counter()
        print(f"'{self.name}' taken:", end_time - self.start_time, "seconds.")

os.environ['AV_PYTHON_AVISYNTH'] = 'C:/ffmpeg/bin'

socket_enabled = False
sock = None
if socket_enabled:
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    print("Connecting to server...")
    sock.connect(('127.0.0.1', 8000))

# Set up video capture.
print("Opening web cam...")
cap = cv2.VideoCapture(0, cv2.CAP_DSHOW)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 800)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 600)

# Initialize the encoder.
encoder = av.CodecContext.create('h264', 'w')
encoder.width = 800
encoder.height = 600
encoder.pix_fmt = 'yuv420p'
encoder.bit_rate = 5000

# Initialize the decoder.
decoder = av.CodecContext.create('h264', 'r')
decoder.width = 800
decoder.height = 600
decoder.pix_fmt = 'yuv420p'
decoder.bit_rate = 5000

print("Streaming...")
while(cap.isOpened()):
    
    # Capture the frame from the camera.
    ret, orig_frame = cap.read()

    cv2.imshow('Source Video', orig_frame)

    # Convert to YUV.
    img_yuv = cv2.cvtColor(orig_frame, cv2.COLOR_BGR2YUV_I420)

    # Create a video frame object from the num py array.
    video_frame = av.VideoFrame.from_ndarray(img_yuv, format='yuv420p')

    with PerfTimer("Encoding") as p:
        encoded_frames = encoder.encode(video_frame)

    # Sometimes the encode results in no frames encoded, so lets skip the frame.
    if len(encoded_frames) == 0:
        continue

    print(f"Decoding {len(encoded_frames)} frames...")

    for frame in encoded_frames:
        encoded_frame_bytes = bytes(frame)

        if socket_enabled:
            # Get the size of the encoded frame in bytes
            size = struct.pack('<L', len(encoded_frame_bytes))
            sock.sendall(size + encoded_frame_bytes)

        # Step 1: Create the packet from the frame.
        packet = av.packet.Packet(frame)

        # Step 2: Decode the packet.
        decoded_packets = decoder.decode(packet)

        for packet in decoded_packets:
            # Step 3: Convert the pixel format from the encoder color format to BGR for displaying.
            frame = cv2.cvtColor(packet.to_ndarray(format='yuv420p'), cv2.COLOR_YUV2BGR_I420)

            # Step 4. Display frame in window.
            cv2.imshow('Decoded Video', frame)

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

# release everything
cap.release()
sock.close()
cv2.destroyAllWindows()

问题原因及修复方案 #

1. 编码器参数不全且码率过低
   H.264编码器需要帧率、GOP间隔、编码预设等关键参数才能生成正常的编码流，原代码仅设置基础参数，且bit_rate=5000（仅5kbps）会导致严重失真。

修复：补充编码器核心参数并调整码率：

encoder = av.CodecContext.create('h264', 'w')
encoder.width = 800
encoder.height = 600
encoder.pix_fmt = 'yuv420p'
encoder.bit_rate = 500000  # 调整为500kbps，保证基本画质
encoder.framerate = 30  # 匹配摄像头帧率
encoder.gop_size = 60  # 每60帧生成一个关键帧
encoder.preset = 'medium'  # 平衡编码速度与画质
encoder.profile = 'high'  # 使用更高编码规格

2. 解码器手动参数覆盖导致不匹配
   解码器会自动从编码包中解析宽高、像素格式等信息，手动设置反而可能和实际编码流冲突。

修复：简化解码器初始化：

decoder = av.CodecContext.create('h264', 'r')

3. YUV格式转换存在对齐问题
   OpenCV的COLOR_BGR2YUV_I420转换后的数据，与av库的帧数据对齐要求不匹配，导致帧格式错乱。

修复：直接通过av库完成格式转换：

# 替换原YUV转换代码
video_frame = av.VideoFrame.from_ndarray(orig_frame, format='bgr24')
video_frame = video_frame.reformat(format='yuv420p', width=800, height=600)

4. 未处理编码器剩余帧
   编码器会缓存部分帧，循环结束时需要flush才能输出剩余数据，否则会丢失最后几帧。

修复：循环结束后添加flush逻辑：

# 循环结束后flush编码器
print("Flushing encoder...")
for packet in encoder.encode(None):
    decoded_frames = decoder.decode(packet)
    for decoded_frame in decoded_frames:
        frame_bgr = cv2.cvtColor(decoded_frame.to_ndarray(format='yuv420p'), cv2.COLOR_YUV2BGR_I420)
        cv2.imshow('Decoded Video', frame_bgr)
        cv2.waitKey(1)

5. 源视频未做读取有效性判断
   cap.read()可能返回空帧，直接显示会导致异常。

修复：添加读取成功判断：

ret, orig_frame = cap.read()
if not ret:
    break
cv2.imshow('Source Video', orig_frame)

6. Socket空指针异常
   当socket_enabled=False时，sock为None，直接调用sock.close()会报错。

修复：添加判断后再关闭：

if socket_enabled:
    sock.close()

修复后的完整代码 #

import struct
import cv2
import socket
import av
import time
import os

class PerfTimer:
    def __init__(self, name):
        self.name = name

    def __enter__(self):
        self.start_time = time.perf_counter()

    def __exit__(self, type, value, traceback):
        end_time = time.perf_counter()
        print(f"'{self.name}' taken:", end_time - self.start_time, "seconds.")

os.environ['AV_PYTHON_AVISYNTH'] = 'C:/ffmpeg/bin'

socket_enabled = False
sock = None
if socket_enabled:
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    print("Connecting to server...")
    sock.connect(('127.0.0.1', 8000))

# Set up video capture.
print("Opening web cam...")
cap = cv2.VideoCapture(0, cv2.CAP_DSHOW)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 800)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 600)

# Initialize the encoder with necessary parameters
encoder = av.CodecContext.create('h264', 'w')
encoder.width = 800
encoder.height = 600
encoder.pix_fmt = 'yuv420p'
encoder.bit_rate = 500000
encoder.framerate = 30
encoder.gop_size = 60
encoder.preset = 'medium'
encoder.profile = 'high'

# Initialize decoder without manual parameter override
decoder = av.CodecContext.create('h264', 'r')

print("Streaming...")
while cap.isOpened():
    # Capture the frame from the camera with ret check
    ret, orig_frame = cap.read()
    if not ret:
        break

    cv2.imshow('Source Video', orig_frame)

    # Convert BGR frame to AVFrame in yuv420p format directly
    video_frame = av.VideoFrame.from_ndarray(orig_frame, format='bgr24')
    video_frame = video_frame.reformat(format='yuv420p', width=800, height=600)

    with PerfTimer("Encoding"):
        encoded_packets = encoder.encode(video_frame)

    # Process encoded packets
    for packet in encoded_packets:
        encoded_bytes = bytes(packet)

        if socket_enabled:
            # Send size + data
            size = struct.pack('<L', len(encoded_bytes))
            sock.sendall(size + encoded_bytes)

        # Decode the packet
        decoded_frames = decoder.decode(packet)

        for decoded_frame in decoded_frames:
            # Convert YUV420p to BGR for display
            frame_bgr = cv2.cvtColor(decoded_frame.to_ndarray(format='yuv420p'), cv2.COLOR_YUV2BGR_I420)
            cv2.imshow('Decoded Video', frame_bgr)

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

# Flush remaining frames from encoder
print("Flushing encoder...")
for packet in encoder.encode(None):
    decoded_frames = decoder.decode(packet)
    for decoded_frame in decoded_frames:
        frame_bgr = cv2.cvtColor(decoded_frame.to_ndarray(format='yuv420p'), cv2.COLOR_YUV2BGR_I420)
        cv2.imshow('Decoded Video', frame_bgr)
        cv2.waitKey(1)

# Release resources
cap.release()
if socket_enabled:
    sock.close()
cv2.destroyAllWindows()

内容的提问来源于stack exchange，提问作者Martin Blore