释放NVENC hwdevice_ctx时触发段错误的修复方案咨询

问题概述 #

使用h264_nvenc编码OpenGL renderbuffer输出时，程序清理阶段调用av_buffer_unref(&_hwDeviceRefCtx)和avcodec_free_context(&_pCodecCtx)触发段错误，Valgrind检测显示存在内存重复释放问题。

Valgrind错误输出 #

Invalid read of size 8
   at 0x48AD987: UnknownInlinedFun (buffer.c:121)
   by 0x48AD987: UnknownInlinedFun (buffer.c:144)
   by 0x48AD987: av_buffer_unref (buffer.c:139)
   by 0x5D06D7A: avcodec_close (avcodec.c:486)
   by 0x628DD7D: avcodec_free_context (options.c:175)
   by 0x10A863: main (main.cpp:115)
 Address 0x17812700 is 0 bytes inside a block of size 24 free'd
   at 0x484488F: free (vg_replace_malloc.c:985)
   by 0x48AD98F: UnknownInlinedFun (buffer.c:127)
   by 0x48AD98F: UnknownInlinedFun (buffer.c:144)
   by 0x48AD98F: av_buffer_unref (buffer.c:139)
   by 0x48BE098: hwframe_ctx_free (hwcontext.c:240)
   by 0x48AD9A6: UnknownInlinedFun (buffer.c:133)
   by 0x48AD9A6: UnknownInlinedFun (buffer.c:144)
   by 0x48AD9A6: av_buffer_unref (buffer.c:139)
   by 0x5D06D0A: UnknownInlinedFun (decode.c:1261)
   by 0x5D06D0A: avcodec_close (avcodec.c:465)
   by 0x628DD7D: avcodec_free_context (options.c:175)
   by 0x10A863: main (main.cpp:115)
 Block was alloc'd at
   at 0x4849366: posix_memalign (vg_replace_malloc.c:2099)
   by 0x48D9BD5: av_malloc (mem.c:105)
   by 0x48D9DAD: av_mallocz (mem.c:256)
   by 0x48AD8DD: UnknownInlinedFun (buffer.c:44)
   by 0x48AD8DD: av_buffer_create (buffer.c:64)
   by 0x48BDDEB: av_hwdevice_ctx_alloc (hwcontext.c:179)
   by 0x48BDF29: av_hwdevice_ctx_create (hwcontext.c:622)
   by 0x10A482: main (main.cpp:43)

Invalid free() / delete / delete[] / realloc()
   at 0x484488F: free (vg_replace_malloc.c:985)
   by 0x48AD98F: UnknownInlinedFun (buffer.c:127)
   by 0x48AD98F: UnknownInlinedFun (buffer.c:144)
   by 0x48AD98F: av_buffer_unref (buffer.c:139)
   by 0x5D06D7A: avcodec_close (avcodec.c:486)
   by 0x628DD7D: avcodec_free_context (options.c:175)
   by 0x10A863: main (main.cpp:115)
 Address 0x17812700 is 0 bytes inside a block of size 24 free'd
   at 0x484488F: free (vg_replace_malloc.c:985)
   by 0x48AD98F: UnknownInlinedFun (buffer.c:127)
   by 0x48AD98F: UnknownInlinedFun (buffer.c:144)
   by 0x48AD98F: av_buffer_unref (buffer.c:139)
   by 0x48BE098: hwframe_ctx_free (hwcontext.c:240)
   by 0x48AD9A6: UnknownInlinedFun (buffer.c:133)
   by 0x48AD9A6: UnknownInlinedFun (buffer.c:144)
   by 0x48AD9A6: av_buffer_unref (buffer.c:139)
   by 0x5D06D0A: UnknownInlinedFun (decode.c:1261)
   by 0x5D06D0A: avcodec_close (avcodec.c:465)
   by 0x628DD7D: avcodec_free_context (options.c:175)
   by 0x10A863: main (main.cpp:115)
 Block was alloc'd at
   at 0x4849366: posix_memalign (vg_replace_malloc.c:2099)
   by 0x48D9BD5: av_malloc (mem.c:105)
   by 0x48D9DAD: av_mallocz (mem.c:256)
   by 0x48AD8DD: UnknownInlinedFun (buffer.c:44)
   by 0x48AD8DD: av_buffer_create (buffer.c:64)
   by 0x48BDDEB: av_hwdevice_ctx_alloc (hwcontext.c:179)
   by 0x48BDF29: av_hwdevice_ctx_create (hwcontext.c:622)
   by 0x10A482: main (main.cpp:43)

复现代码 #

#include <string>

extern "C" {
  #include <libavutil/opt.h>
  #include <libavcodec/avcodec.h>
  #include <libavformat/avformat.h>
  #include <libavutil/hwcontext.h>
  #include <libavutil/pixdesc.h>
  #include <libavutil/hwcontext_cuda.h>
}

int main() {
    const int _SrcImageWidth=640;
    const int _SrcImageHeight=480;
    
    const AVOutputFormat *_oFmt = nullptr;
    AVFormatContext *_oFmtCtx = nullptr;
    
    const AVCodec *_pCodec = nullptr;
    AVCodecContext *_pCodecCtx = nullptr;
    
    AVFrame* _frame;
    AVPacket* _packet;
    AVStream* _stream;
    
    AVBufferRef *_hwDeviceRefCtx = nullptr;
    const CUcontext* _cudaCtx;
    
    const std::string _OutFileName = "output.mkv";
    
    //constructor part
    int ret;

    //output format context      
    avformat_alloc_output_context2( &_oFmtCtx, nullptr, nullptr, _OutFileName.c_str() );
    _oFmt = _oFmtCtx->oformat;

    //hardware format context
    ret = av_hwdevice_ctx_create( &_hwDeviceRefCtx, AV_HWDEVICE_TYPE_CUDA, "NVIDIA GeForce RTX 4070", nullptr, 0 );

    //hardware frame context for device buffer allocation
    AVBufferRef* hwFrameRefCtx = av_hwframe_ctx_alloc( _hwDeviceRefCtx );
    AVHWFramesContext* hwFrameCtx = (AVHWFramesContext*) (hwFrameRefCtx->data);
    hwFrameCtx->width = _SrcImageWidth;
    hwFrameCtx->height = _SrcImageHeight;
    hwFrameCtx->sw_format = AV_PIX_FMT_0BGR32;
    hwFrameCtx->format = AV_PIX_FMT_CUDA;
    hwFrameCtx->device_ref = _hwDeviceRefCtx;
    hwFrameCtx->device_ctx = (AVHWDeviceContext*) _hwDeviceRefCtx->data;

    ret = av_hwframe_ctx_init( hwFrameRefCtx );

    //get cuda context
    const AVHWDeviceContext* hwDeviceCtx = (AVHWDeviceContext*)(_hwDeviceRefCtx->data);
    const AVCUDADeviceContext* cudaDeviceCtx = (AVCUDADeviceContext*)(hwDeviceCtx->hwctx);
    _cudaCtx = &(cudaDeviceCtx->cuda_ctx);

    //codec context
    _pCodec = avcodec_find_encoder_by_name( "h264_nvenc" );

    _packet = av_packet_alloc();

    _stream = avformat_new_stream( _oFmtCtx, nullptr );
    _stream->id = _oFmtCtx->nb_streams - 1;
    _pCodecCtx = avcodec_alloc_context3( _pCodec );

    _pCodecCtx->qmin = 18;
    _pCodecCtx->qmax = 20;
    _pCodecCtx->width = _SrcImageWidth;
    _pCodecCtx->height = _SrcImageHeight;
    _pCodecCtx->framerate = (AVRational) {25,1};
    _pCodecCtx->time_base = (AVRational) {1,25};
    _stream->time_base = _pCodecCtx->time_base;
    _pCodecCtx->gop_size = 12; //I-Frame every at most 12 frames
    _pCodecCtx->max_b_frames = 2;
    _pCodecCtx->pix_fmt = AV_PIX_FMT_CUDA; //required to use renderbuffer as src
    _pCodecCtx->codec_type = AVMEDIA_TYPE_VIDEO;
    _pCodecCtx->sw_pix_fmt = AV_PIX_FMT_0BGR32; 
    _pCodecCtx->hw_device_ctx = _hwDeviceRefCtx;
    _pCodecCtx->hw_frames_ctx = av_buffer_ref( hwFrameRefCtx );
    av_opt_set(_pCodecCtx->priv_data, "preset", "p7", 0);
    av_opt_set(_pCodecCtx->priv_data, "rc", "vbr", 0);
    if( _oFmtCtx->oformat->flags & AVFMT_GLOBALHEADER ) {
        _pCodecCtx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
    }

    ret = avcodec_open2( _pCodecCtx, _pCodec, nullptr );
    avcodec_parameters_from_context( _stream->codecpar, _pCodecCtx );

    if (!(_oFmtCtx->oformat->flags & AVFMT_NOFILE)) {
        ret = avio_open(&_oFmtCtx->pb, _OutFileName.c_str(), AVIO_FLAG_WRITE);
    }
    ret = avformat_write_header( _oFmtCtx, nullptr );

    //use hardware frame from above
    _frame = av_frame_alloc();
    ret = av_hwframe_get_buffer( _pCodecCtx->hw_frames_ctx, _frame, 0 );
    _frame->pts = 1;

    av_buffer_unref( &hwFrameRefCtx );

    //destructor part
    av_frame_free( &_frame );
    av_packet_free( &_packet );

    av_write_trailer( _oFmtCtx );
    avio_closep( &_oFmtCtx->pb );

    avformat_free_context( _oFmtCtx );

    avcodec_free_context( &_pCodecCtx );
    av_buffer_unref( &_hwDeviceRefCtx );

    return 0;
}

编译命令 #

g++ -lavutil -lavformat -lavcodec -lz -lavutil -lswscale -lswresample -lm -ggdb3 -I/opt/cuda/include main.cpp

问题根源与修复方案 #

问题分析 #

1. AVBufferRef引用计数错误：FFmpeg的AVBufferRef采用引用计数管理内存。直接将_hwDeviceRefCtx赋值给_pCodecCtx->hw_device_ctx时未增加引用计数，avcodec_free_context释放_pCodecCtx时会自动调用av_buffer_unref释放hw_device_ctx，此时_hwDeviceRefCtx指向的内存已被释放，后续手动调用av_buffer_unref(&_hwDeviceRefCtx)会触发重复释放。
2. 冗余手动赋值：创建hwFrameCtx时，手动设置device_ref和device_ctx完全多余，av_hwframe_ctx_init会自动从av_hwframe_ctx_alloc传入的_hwDeviceRefCtx创建正确引用。

修复步骤 #

1. 移除冗余设备上下文赋值：删除hwFrameCtx->device_ref = _hwDeviceRefCtx;和hwFrameCtx->device_ctx = (AVHWDeviceContext*) _hwDeviceRefCtx->data;两行代码。
2. 正确管理设备上下文引用计数：将_pCodecCtx->hw_device_ctx = _hwDeviceRefCtx;改为_pCodecCtx->hw_device_ctx = av_buffer_ref(_hwDeviceRefCtx);，通过av_buffer_ref增加引用计数，确保avcodec_free_context释放的是引用而非原buffer。

修改后的关键代码片段 #

//hardware frame context for device buffer allocation
AVBufferRef* hwFrameRefCtx = av_hwframe_ctx_alloc( _hwDeviceRefCtx );
AVHWFramesContext* hwFrameCtx = (AVHWFramesContext*) (hwFrameRefCtx->data);
hwFrameCtx->width = _SrcImageWidth;
hwFrameCtx->height = _SrcImageHeight;
hwFrameCtx->sw_format = AV_PIX_FMT_0BGR32;
hwFrameCtx->format = AV_PIX_FMT_CUDA;
// 移除以下两行冗余赋值
// hwFrameCtx->device_ref = _hwDeviceRefCtx;
// hwFrameCtx->device_ctx = (AVHWDeviceContext*) _hwDeviceRefCtx->data;

ret = av_hwframe_ctx_init( hwFrameRefCtx );

// ...

// codec context部分修改引用计数
_pCodecCtx->hw_device_ctx = av_buffer_ref(_hwDeviceRefCtx); // 增加引用计数
_pCodecCtx->hw_frames_ctx = av_buffer_ref( hwFrameRefCtx );

清理顺序说明 #

修复后，avcodec_free_context(&_pCodecCtx)会自动释放hw_device_ctx的引用（引用计数减1），之后调用av_buffer_unref(&_hwDeviceRefCtx)会正确将引用计数减至0并释放内存，不会出现重复释放问题。

内容的提问来源于stack exchange，提问作者camelCase