如何实现低延迟RTSP视频流读取？代码遇黑屏与延迟优化问题

一、黑屏问题排查与修复 #

你的代码出现黑屏主要有以下几个核心原因：

1. 像素格式未适配：RTSP流解码后的AVFrame通常为YUV420P格式，你直接用CV_8UC1创建Mat仅读取Y分量，未处理UV分量，且OpenCV显示默认需要BGR格式，导致画面无法正常渲染；
2. 内存引用失效：创建的cv::Mat直接指向AVFrame的缓冲区，调用av_frame_free(&frame)后内存被释放，Mat指针变为野指针，显示内容不可控；
3. 未处理帧对齐问题：AVFrame的linesize可能大于实际宽度，直接用width创建Mat会导致画面错位或黑屏。

修复核心代码片段 #

在解码成功后添加像素格式转换，并拷贝Mat数据避免内存失效：

if (len >= 0 && got_frame)
{
    // 初始化格式转换上下文（仅需创建一次）
    static SwsContext* sws_ctx = sws_getContext(
        frame->width, frame->height, (AVPixelFormat)frame->format,
        frame->width, frame->height, AV_PIX_FMT_BGR24,
        SWS_BILINEAR, nullptr, nullptr, nullptr
    );

    cv::Mat bgr_frame(frame->height, frame->width, CV_8UC3);
    uint8_t* dst_data[1] = {bgr_frame.data};
    int dst_linesize[1] = {bgr_frame.step};

    // 转换为OpenCV支持的BGR格式
    sws_scale(sws_ctx, frame->data, frame->linesize, 0, frame->height, dst_data, dst_linesize);

    std::lock_guard<std::mutex> lock(mtx);
    frames.push(bgr_frame.clone()); // 拷贝数据，避免AVFrame释放后内存失效
}

程序结束时记得释放转换上下文：

sws_freeContext(sws_ctx);

二、低延迟优化方案 #

当前230ms延迟可通过以下配置进一步降低，目标可压至100ms以内：

1. RTSP与AVFormat层优化 #

AVDictionary *options = NULL;
// 关闭格式层缓冲，最小化延迟
av_dict_set(&options, "max_delay", "0", 0);
av_dict_set(&options, "buffer_size", "10240", 0); // 减小缓冲区大小
av_dict_set(&options, "rtsp_transport", "tcp", 0); // TCP稳定，低延迟场景可尝试UDP
av_dict_set(&options, "stimeout", "500000", 0); // 超时时间（微秒）
av_dict_set(&options, "use_wallclock_as_timestamps", "1", 0); // 使用系统时钟作为时间戳
av_dict_set(&options, "fflags", "nobuffer+flush_packets", 0); // 禁用缓冲，立即处理数据包

// 打开输入后追加格式上下文标记
avformat_open_input(&container, url, NULL, &options);
container->flags |= AVFMT_FLAG_NOBUFFER | AVFMT_FLAG_FLUSH_PACKETS;

2. 解码器低延迟配置 #

decoderContext->flags |= AV_CODEC_FLAG_LOW_DELAY; // 启用低延迟解码模式
decoderContext->flags2 |= AV_CODEC_FLAG2_FAST | AV_CODEC_FLAG2_NO_OUTPUT_TIMESTAMP;
decoderContext->skip_loop_filter = AVDISCARD_ALL; // 跳过环路滤波，牺牲画质换速度
decoderContext->skip_idct = AVDISCARD_ALL; // 跳过IDCT变换
decoderContext->skip_frame = AVDISCARD_NONREF; // 跳过非参考帧

3. 线程与队列优化 #

• 将读取线程、解码线程、显示线程完全分离，避免单线程阻塞；
• 限制帧队列长度为1-2，避免旧帧积压导致延迟：

// 显示线程中处理队列
std::lock_guard<std::mutex> lock(mtx);
// 仅保留最新帧，丢弃旧帧
if (frames.size() >= 2) {
    while (frames.size() > 1) {
        frames.pop();
    }
}
cv::Mat f = frames.front();
frames.pop();

4. 其他优化点 #

• 禁用av_dump_format等调试输出，减少IO开销；
• 使用硬件加速解码（如CUDA、VA-API），指定硬件解码器名称（如h264_cuvid、hevc_cuvid）；
• OpenCV显示改用cv::pollKey()替代cv::waitKey(1)，减少等待时间；
• 编译时开启O3优化，链接libav的优化版本。

完整优化后代码 #

#include <cstdio>
#include <cstring>
#include <iostream>
#include <queue>
#include <thread>
#include <mutex>
#include <condition_variable>
#include "opencv2/opencv.hpp"
extern "C"
{
#include <libavformat/avformat.h>
#include <libavutil/error.h>
#include <libavcodec/avcodec.h>
#include <libswscale/swscale.h>
}

#define ERR_BUFF_SIZE 1024
bool is_running = true;
std::mutex mtx;
std::condition_variable cv;
std::queue<cv::Mat> frames;

void show_thread()
{
    cv::namedWindow("RTSP Stream", cv::WINDOW_NORMAL | cv::WINDOW_KEEPRATIO);
    cv::resizeWindow("RTSP Stream", 1280, 720);

    while (is_running)
    {
        std::unique_lock<std::mutex> lock(mtx);
        cv.wait(lock, []{ return !frames.empty() || !is_running; });

        if (!frames.empty())
        {
            cv::Mat f = frames.front();
            frames.pop();
            lock.unlock();

            cv::imshow("RTSP Stream", f);
            cv::pollKey(); // 非阻塞按键检测
        }
    }

    cv::destroyAllWindows();
}

int main(int argc, char **argv)
{
    const char *url = "rtsp://admin:hik123456@192.168.10.64";

    av_register_all();
    avformat_network_init();

    AVFormatContext *container = nullptr;
    AVDictionary *options = NULL;

    // RTSP低延迟参数配置
    av_dict_set(&options, "max_delay", "0", 0);
    av_dict_set(&options, "buffer_size", "10240", 0);
    av_dict_set(&options, "rtsp_transport", "tcp", 0);
    av_dict_set(&options, "stimeout", "500000", 0);
    av_dict_set(&options, "use_wallclock_as_timestamps", "1", 0);
    av_dict_set(&options, "fflags", "nobuffer+flush_packets", 0);

    int ret = avformat_open_input(&container, url, NULL, &options);
    if (ret != 0)
    {
        char error_buffer[ERR_BUFF_SIZE];
        av_strerror(ret, error_buffer, sizeof(error_buffer));
        std::cerr << "Failed to open input: " << error_buffer << std::endl;
        return 1;
    }

    container->flags |= AVFMT_FLAG_NOBUFFER | AVFMT_FLAG_FLUSH_PACKETS;

    if (avformat_find_stream_info(container, NULL) < 0)
    {
        std::cerr << "Failed to find stream info" << std::endl;
        return 1;
    }

    int video_stream = -1;
    AVCodecParameters *codec_par = nullptr;
    for (unsigned i = 0; i < container->nb_streams; i++)
    {
        codec_par = container->streams[i]->codecpar;
        if (codec_par->codec_type == AVMEDIA_TYPE_VIDEO)
        {
            video_stream = i;
            break;
        }
    }

    if (video_stream == -1)
    {
        std::cerr << "No video stream found" << std::endl;
        return 1;
    }

    AVCodec *codec = avcodec_find_decoder(codec_par->codec_id);
    if (!codec)
    {
        std::cerr << "Failed to find codec" << std::endl;
        return 1;
    }

    AVCodecContext *decoder_ctx = avcodec_alloc_context3(codec);
    ret = avcodec_parameters_to_context(decoder_ctx, codec_par);
    if (ret < 0)
    {
        std::cerr << "Failed to copy codec parameters" << std::endl;
        return 1;
    }

    // 解码器低延迟配置
    decoder_ctx->flags |= AV_CODEC_FLAG_LOW_DELAY;
    decoder_ctx->flags2 |= AV_CODEC_FLAG2_FAST | AV_CODEC_FLAG2_NO_OUTPUT_TIMESTAMP;
    decoder_ctx->skip_loop_filter = AVDISCARD_ALL;
    decoder_ctx->skip_idct = AVDISCARD_ALL;
    decoder_ctx->skip_frame = AVDISCARD_NONREF;

    if (avcodec_open2(decoder_ctx, codec, nullptr) < 0)
    {
        std::cerr << "Failed to open codec" << std::endl;
        return 1;
    }

    std::thread t(show_thread);
    SwsContext* sws_ctx = sws_getContext(
        decoder_ctx->width, decoder_ctx->height, decoder_ctx->pix_fmt,
        decoder_ctx->width, decoder_ctx->height, AV_PIX_FMT_BGR24,
        SWS_BILINEAR, nullptr, nullptr, nullptr
    );

    AVPacket pkt;
    av_init_packet(&pkt);
    int frame_count = 0;

    while (is_running && av_read_frame(container, &pkt) >= 0)
    {
        if (pkt.stream_index != video_stream)
        {
            av_packet_unref(&pkt);
            continue;
        }

        AVFrame *frame = av_frame_alloc();
        int got_frame = 0;
        ret = avcodec_decode_video2(decoder_ctx, frame, &got_frame, &pkt);

        if (ret >= 0 && got_frame)
        {
            cv::Mat bgr_frame(frame->height, frame->width, CV_8UC3);
            uint8_t* dst_data[1] = {bgr_frame.data};
            int dst_linesize[1] = {bgr_frame.step};

            sws_scale(sws_ctx, frame->data, frame->linesize, 0, frame->height, dst_data, dst_linesize);

            std::lock_guard<std::mutex> lock(mtx);
            // 限制队列长度，避免旧帧积压
            if (frames.size() >= 2)
            {
                frames.pop();
            }
            frames.push(bgr_frame.clone());
            cv.notify_one();
        }

        av_frame_free(&frame);
        av_packet_unref(&pkt);

        frame_count++;
        if (frame_count > 10000)
        {
            is_running = false;
            break;
        }
    }

    // 通知显示线程退出
    {
        std::lock_guard<std::mutex> lock(mtx);
        is_running = false;
        cv.notify_one();
    }

    t.join();
    sws_freeContext(sws_ctx);
    avcodec_close(decoder_ctx);
    avformat_close_input(&container);
    avformat_network_deinit();

    return 0;
}

内容的提问来源于stack exchange，提问作者rayion