使用FFmpeg.AutoGen在.NET 8解码H264/JPEG时帧数据混合问题求助

在.NET 8.0应用中使用FFmpeg.AutoGen进行H264和JPEG解码，每个解码操作使用独立实例（无共享对象），且不会同时使用同一实例，但实例可被不同线程调用。然而解码结果出现多源数据混合的情况，该问题多发生于同编码器的场景，但不限于此。

基类代码 #

public abstract unsafe class Decoder : IDisposable
{
    protected readonly object sync = new();
    protected AVCodecContext* codecContext;
    protected AVFrame* frame;
    protected AVPacket* packet;
    protected SwsContext* swsContext;
    private bool disposed = false;

    protected Decoder(AVCodecID codecId, ChannelWriter<(ArraySegment<byte> decodedData, int width, int height)> writer)
    {
        Writer = writer;
        var codec = ffmpeg.avcodec_find_decoder(codecId);
        codecContext = ffmpeg.avcodec_alloc_context3(codec);
        ffmpeg.avcodec_open2(codecContext, codec, null);

        frame = ffmpeg.av_frame_alloc();
        packet = ffmpeg.av_packet_alloc();
    }
...
}

H264解码代码 #

public override void Decode(ReadOnlySpan<byte> data)
{
    CheckDisposed();

    lock (sync)
    {
        fixed (byte* pData = data)
        {
            ffmpeg.av_packet_unref(packet);

            packet->data = pData;
            packet->size = data.Length;

            var result = ffmpeg.avcodec_send_packet(codecContext, packet);
            if (result < 0 && result != ffmpeg.AVERROR(ffmpeg.EAGAIN))
            {
                EvalResult(result);
            }

            while (true)
            {
                ffmpeg.av_frame_unref(frame);

                result = ffmpeg.avcodec_receive_frame(codecContext, frame);
                if (result == ffmpeg.AVERROR(ffmpeg.EAGAIN))
                {
                    // 需要更多输入数据，继续发送数据包
                    return;
                }
                else if (result < 0)
                {
                    EvalResult(result);
                }

                var width = frame->width;
                var height = frame->height;

                if (swsContext == null)
                {
                    swsContext = ffmpeg.sws_getContext(
                        width, height, (AVPixelFormat)frame->format,
                        width, height, AVPixelFormat.AV_PIX_FMT_BGRA,
                        ffmpeg.SWS_BILINEAR, null, null, null);
                }

                var bgraFrame = ffmpeg.av_frame_alloc();
                bgraFrame->format = (int)AVPixelFormat.AV_PIX_FMT_BGRA;
                bgraFrame->width = width;
                bgraFrame->height = height;
                ffmpeg.av_frame_get_buffer(bgraFrame, 32);

                ffmpeg.sws_scale(
                    swsContext,
                    frame->data, frame->linesize, 0, height,
                    bgraFrame->data, bgraFrame->linesize);

                var bgraSize = width * height * 4;
                var bgraArray = ArrayPool<byte>.Shared.Rent(bgraSize);
                var bgraSegment = new ArraySegment<byte>(bgraArray, 0, bgraSize);

                fixed (byte* pBGRAData = bgraSegment.Array)
                {
                    var data4 = new byte_ptr4();
                    data4.UpdateFrom(bgraFrame->data.ToArray());

                    var linesize4 = new int4();
                    linesize4.UpdateFrom(bgraFrame->linesize.ToArray());

                    ffmpeg.av_image_copy_to_buffer(
                        pBGRAData, bgraSize,
                        data4, linesize4,
                        (AVPixelFormat)bgraFrame->format, width, height, 1);
                }

                ffmpeg.av_frame_free(&bgraFrame);

                Writer.TryWrite((bgraSegment, width, height));
            }
        }
    }
}

JPEG解码代码 #

public override void Decode(ReadOnlySpan<byte> data)
{
    CheckDisposed();

    lock (sync)
    {
        fixed (byte* pData = data)
        {
            ffmpeg.av_packet_unref(packet);

            packet->data = pData;
            packet->size = data.Length;

            var result = ffmpeg.avcodec_send_packet(codecContext, packet);
            if (result < 0 && result != ffmpeg.AVERROR(ffmpeg.EAGAIN))
            {
                EvalResult(result);
            }

            while (true)
            {
                ffmpeg.av_frame_unref(frame);

                result = ffmpeg.avcodec_receive_frame(codecContext, frame);

                if (result == ffmpeg.AVERROR(ffmpeg.EAGAIN))
                {
                    // 需要更多输入数据，继续发送数据包
                    return;
                }
                else if (result < 0)
                {
                    EvalResult(result);
                }

                var width = frame->width;
                var height = frame->height;

                if (swsContext == null)
                {
                    swsContext = ffmpeg.sws_getContext(
                        width, height, (AVPixelFormat)frame->format,
                        width, height, AVPixelFormat.AV_PIX_FMT_BGRA,
                        ffmpeg.SWS_BILINEAR, null, null, null);
                }

                var bgraFrame = ffmpeg.av_frame_alloc();
                bgraFrame->format = (int)AVPixelFormat.AV_PIX_FMT_BGRA;
                bgraFrame->width = frame->width;
                bgraFrame->height = frame->height;
                ffmpeg.av_frame_get_buffer(bgraFrame, 32);

                ffmpeg.sws_scale(
                    swsContext,
                    frame->data, frame->linesize, 0, frame->height,
                    bgraFrame->data, bgraFrame->linesize);

                int bgraDataSize = bgraFrame->linesize[0] * bgraFrame->height;
                byte[] bgraData = ArrayPool<byte>.Shared.Rent(bgraDataSize);
                var bgraSegment = new ArraySegment<byte>(bgraData, 0, bgraDataSize);
                Marshal.Copy((IntPtr)bgraFrame->data[0], bgraData, 0, bgraDataSize);

                ffmpeg.av_frame_free(&bgraFrame);

                Writer.TryWrite((bgraSegment, width, height));
            }
        }
    }
}

1. SwsContext复用未处理分辨率/格式变化 #

swsContext首次初始化后便永久复用，但如果后续解码的帧出现分辨率、像素格式变化（比如H264流切换分辨率，JPEG图片分辨率不一致），旧上下文会用原参数缩放，导致画面错乱、数据混合。

修复：
在基类新增参数记录字段，每次解码时检查当前帧参数是否匹配，不匹配则销毁旧上下文重建：

// 基类新增私有字段
private int _lastSrcWidth = -1;
private int _lastSrcHeight = -1;
private AVPixelFormat _lastSrcFormat = AVPixelFormat.AV_PIX_FMT_NONE;

// 替换原swsContext初始化逻辑
int currentSrcWidth = frame->width;
int currentSrcHeight = frame->height;
AVPixelFormat currentSrcFormat = (AVPixelFormat)frame->format;

if (swsContext == null 
    || _lastSrcWidth != currentSrcWidth 
    || _lastSrcHeight != currentSrcHeight 
    || _lastSrcFormat != currentSrcFormat)
{
    if (swsContext != null)
    {
        ffmpeg.sws_freeContext(swsContext);
        swsContext = null;
    }

    swsContext = ffmpeg.sws_getContext(
        currentSrcWidth, currentSrcHeight, currentSrcFormat,
        currentSrcWidth, currentSrcHeight, AVPixelFormat.AV_PIX_FMT_BGRA,
        ffmpeg.SWS_BILINEAR, null, null, null);

    _lastSrcWidth = currentSrcWidth;
    _lastSrcHeight = currentSrcHeight;
    _lastSrcFormat = currentSrcFormat;
}

2. AVPacket数据指针依赖.NET内存导致无效引用 #

直接将fixed后的.NET内存指针赋值给packet->data，但FFmpeg解码器可能会保留该指针用于后续解码（如H264帧间依赖）。fixed块结束后，.NET GC可能移动或释放内存，导致指针指向无效区域，混入其他线程数据。

修复：
让FFmpeg管理数据包内存，避免依赖.NET内存：

// 替换原packet赋值逻辑
ffmpeg.av_packet_unref(packet);

// 复制输入数据到FFmpeg分配的内存
byte* packetData = ffmpeg.av_malloc((ulong)data.Length);
Buffer.MemoryCopy(data, packetData, data.Length, data.Length);
ffmpeg.av_packet_from_data(packet, packetData, data.Length);

// 后续send_packet逻辑不变
var result = ffmpeg.avcodec_send_packet(codecContext, packet);

av_packet_from_data会接管packetData内存，后续av_packet_unref会自动释放，无需手动调用av_free。

3. ArrayPool内存块残留旧数据 #

使用ArrayPool<byte>.Shared.Rent获取的内存块可能残留之前使用者的数据，未清零会导致新解码结果与旧数据混合。

修复：
Rent内存后立即清零有效区域：

// H264解码中
var bgraArray = ArrayPool<byte>.Shared.Rent(bgraSize);
var bgraSegment = new ArraySegment<byte>(bgraArray, 0, bgraSize);
Array.Clear(bgraArray, 0, bgraSize);

// JPEG解码中
byte[] bgraData = ArrayPool<byte>.Shared.Rent(bgraDataSize);
var bgraSegment = new ArraySegment<byte>(bgraData, 0, bgraDataSize);
Array.Clear(bgraData, 0, bgraDataSize);

额外建议：切换输入源时重置解码器 #

若同一解码器实例切换不同输入源，需调用avcodec_flush_buffers清空解码器状态，避免残留帧数据干扰：

ffmpeg.avcodec_flush_buffers(codecContext);

内容的提问来源于stack exchange，提问作者Paulo Morgado