如何降低树莓派4B录制RTSP流时的延迟？

问题背景 #

我有一台通过以太网传输RTSP协议的IP摄像头，解码格式为H.264，当前录制时延迟约1~2秒。设备参数如下：

• 树莓派4B/8GB内存/32GB A1级SD卡
• Python 3.11.2 + OpenCV 4.9.0
• 已设置gpu_mem=256

尝试过使用CAP_GStreamer，但延迟反而升至2.5秒，目标是将录制延迟控制在300~500ms。当前实现代码：

import cv2
import sys
import os
#need this line to be work
os.environ["OPENCV_FFMPEG_CAPTURE_OPTIONS"] = "rtsp_transport;udp"
# rtsp url
rtsp_url = "rtsp://192.168.144.25:8554/main.264"

# create viedoCapture
cap = cv2.VideoCapture(rtsp_url)

# check video is open or not
if not cap.isOpened():
    print("Error: Could not open RTSP stream.")
    sys.exit()

# get video info
fps = int(cap.get(cv2.CAP_PROP_FPS))
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))

# use h.264 to encode video
fourcc = cv2.VideoWriter_fourcc(*'H264')
out = None
recording = False

# show and save video
while True:
    ret, frame = cap.read()

    if not ret:
        print("Error: Could not read frame.")
        break

    cv2.imshow('RTSP Stream', frame)

    # record mode
    if recording:
        if out is None:
            out = cv2.VideoWriter('output.mp4', fourcc, fps, (width, height))
        out.write(frame)

    key = cv2.waitKey(1) & 0xFF

    # press q to quit
    if key == ord('q'):
        break

    # press r to switch record mode
    elif key == ord('r'):
        recording = not recording
        if recording:
            print("Recording started.")
        else:
            print("Recording stopped.")
            if out is not None:
                out.release()
                out = None

    # press p to photo
    elif key == ord('p'):
        cv2.imwrite('photo.jpg', frame)
        print("Photo taken.")

# release source
cap.release()
if out is not None:
    out.release()
cv2.destroyAllWindows()

核心优化方案 #

1. 调整FFmpeg捕获参数，强制低缓存 #

FFmpeg默认缓存策略会导致帧堆积，通过以下参数直接限制延迟：

os.environ["OPENCV_FFMPEG_CAPTURE_OPTIONS"] = "rtsp_transport;udp|buffer_size;102400|max_delay;500|fflags;nobuffer+flush_packets"

• buffer_size：缩小UDP接收缓冲区，避免过多帧堆积
• max_delay：强制限制最大延迟为500ms
• fflags：nobuffer禁用全局缓存，flush_packets让数据包到达后立即处理

2. 启用树莓派硬件加速解码 #

利用树莓派4B的VPU硬件解码H.264，替代CPU软解码减少延迟：

# 创建VideoCapture时指定FFmpeg后端，并启用硬件加速
cap = cv2.VideoCapture(rtsp_url, cv2.CAP_FFMPEG)
cap.set(cv2.CAP_PROP_HW_ACCELERATION, cv2.VIDEO_ACCELERATION_V4L2_MPEG2)
cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)  # 设置OpenCV仅缓存1帧

也可以在FFmpeg选项中直接指定硬件解码器：

os.environ["OPENCV_FFMPEG_CAPTURE_OPTIONS"] += "|vcodec;h264_v4l2m2m"

3. 多线程分离捕获与处理流程 #

将帧捕获和显示/录制拆分为独立线程，避免串行处理导致的帧积压：

import queue
import threading

frame_queue = queue.Queue(maxsize=2)  # 仅保留最新2帧，丢弃旧帧

def capture_frames():
    while cap.isOpened():
        ret, frame = cap.read()
        if not ret:
            break
        if frame_queue.full():
            frame_queue.get()  # 移除旧帧
        frame_queue.put(frame)

# 启动捕获线程
capture_thread = threading.Thread(target=capture_frames)
capture_thread.daemon = True
capture_thread.start()

# 主线程处理显示和录制
while True:
    if not frame_queue.empty():
        frame = frame_queue.get()
        cv2.imshow('RTSP Stream', frame)
        if recording and out is not None:
            out.write(frame)
    # 原有按键处理逻辑不变

4. 网络与摄像头端优化 #

• 使用千兆网线连接树莓派与摄像头，确保以太网带宽充足
• 在摄像头后台开启"低延迟流"模式（部分设备称为"实时模式"）
• 关闭摄像头的图像降噪、锐化等附加处理功能，减少端侧延迟

完整优化后代码 #

import cv2
import sys
import os
import queue
import threading

# 设置FFmpeg低延迟及硬件加速参数
os.environ["OPENCV_FFMPEG_CAPTURE_OPTIONS"] = "rtsp_transport;udp|buffer_size;102400|max_delay;500|fflags;nobuffer+flush_packets|vcodec;h264_v4l2m2m"
rtsp_url = "rtsp://192.168.144.25:8554/main.264"

# 初始化VideoCapture并配置硬件加速
cap = cv2.VideoCapture(rtsp_url, cv2.CAP_FFMPEG)
cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
cap.set(cv2.CAP_PROP_HW_ACCELERATION, cv2.VIDEO_ACCELERATION_V4L2_MPEG2)

if not cap.isOpened():
    print("Error: Could not open RTSP stream.")
    sys.exit()

fps = int(cap.get(cv2.CAP_PROP_FPS))
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))

fourcc = cv2.VideoWriter_fourcc(*'H264')
out = None
recording = False

# 帧队列限制缓存
frame_queue = queue.Queue(maxsize=2)

def capture_frames():
    while cap.isOpened():
        ret, frame = cap.read()
        if not ret:
            break
        if frame_queue.full():
            frame_queue.get()
        frame_queue.put(frame)

# 启动捕获线程
capture_thread = threading.Thread(target=capture_frames)
capture_thread.daemon = True
capture_thread.start()

while True:
    if not frame_queue.empty():
        frame = frame_queue.get()
        cv2.imshow('RTSP Stream', frame)
        
        if recording:
            if out is None:
                out = cv2.VideoWriter('output.mp4', fourcc, fps, (width, height))
            out.write(frame)
    
    key = cv2.waitKey(1) & 0xFF
    
    if key == ord('q'):
        break
    elif key == ord('r'):
        recording = not recording
        if recording:
            print("Recording started.")
        else:
            print("Recording stopped.")
            if out is not None:
                out.release()
                out = None
    elif key == ord('p'):
        cv2.imwrite('photo.jpg', frame)
        print("Photo taken.")

cap.release()
if out is not None:
    out.release()
cv2.destroyAllWindows()

内容的提问来源于stack exchange，提问作者Leon