FFmpeg(libav)解封装HLS+MediaCodec Surface渲染播放过快问题排查

API 探路者

2026-7-8

问题描述

使用libav解封装m3u8，获取数据包后通过绑定Surface的MediaCodec直接渲染解码，功能可运行但视频播放速度极快。尝试调整PTS/DTS参数未解决，因设备资源有限必须使用直接渲染方案。

初始化与解析代码

int MediaParserFFmpeg::init(const std::string &filePath)
{
    m_formatCtx = avformat_alloc_context();
    int ret = 0;
    
    ret = avformat_open_input(&m_formatCtx, m_mediaPath.c_str(), iFormat, nullptr);
    if (ret < 0)
    {
        return MEDIA_ERROR_INIT_FAILED;
    }

    ret = avformat_find_stream_info(m_formatCtx, nullptr);
    if (ret < 0)
    {
        return MEDIA_ERROR_INIT_FAILED;
    }

    m_duration = m_formatCtx->duration;
    const AVCodec *vcodec = nullptr;
    m_vStreamIdx = av_find_best_stream(m_formatCtx,
                                        AVMEDIA_TYPE_VIDEO,
                                        -1,
                                        -1,
                                        &vcodec,
                                        0);


    AVCodecParameters *codecParam = m_formatCtx->streams[m_vStreamIdx]->codecpar;
    m_width = codecParam->width;
    m_height = codecParam->height;
    m_vCodec = codecParam->codec_id;
    AVRational fpsRatio = m_formatCtx->streams[m_vStreamIdx]->avg_frame_rate;
    m_frameRate = fpsRatio.num / (float)fpsRatio.den;

    if (codecParam->codec_id == AV_CODEC_ID_H264)
    {
        // if stored in ANNEXB type convert to NALU
        m_isNal = codecParam->extradata_size > 0 && *(codecParam->extradata) == 1;
        if (m_isNal)
        {
            const AVBitStreamFilter *bsfFilter = av_bsf_get_by_name("h264_mp4toannexb");
            if (!bsfFilter)
            {
                return MEDIA_ERROR_INIT_FAILED;
            }
            av_bsf_alloc(bsfFilter, &m_bsfcContext);
            m_bsfcContext->par_in = codecParam;
            av_bsf_init(m_bsfcContext);

            m_bsfPacket = av_packet_alloc();
        }
    }

    m_packet = av_packet_alloc();
}

int MediaParserFFmpeg::getNextFrame(MediaType *mediaType, AVEncodedFrame **outFrame)
{
    int ret = 0;
    ret = av_read_frame(m_formatCtx, m_packet);

    if (ret < 0 && ret != AVERROR_EOF)
    {
        av_packet_unref(m_packet);
        return MEDIA_ERROR_PARSER_PARSE_FAILED;
    }

    if (ret == AVERROR_EOF)
    {
        av_packet_unref(m_packet);
        return MEDIA_INFO_PARSER_END_OF_STREAM;
    }

    if (m_packet->data && m_packet->size)
    {
        if (m_vCodec == VIDEOCODEC_H264 && m_isNal)
        {
            int res = av_bsf_send_packet(m_bsfcContext, m_packet);
            if (res != 0)
            {
                MEDIA_LOG() << "ZError:" << "Error adding NAL to H264 stream";
            }

            if (res == 0)
            {
                *outFrame = new EncodedVideoFrame(m_bsfPacket->data,
                                                    m_bsfPacket->size,
                                                    getTimeStamp(m_bsfPacket, m_formatCtx->streams[m_vStreamIdx]),
                                                    m_width,
                                                    m_height,
                                                    true);
                *mediaType = MEDIA_TYPE_VIDEO;
            }
            av_packet_unref(m_bsfPacket);

            res = av_bsf_receive_packet(m_bsfcContext, m_bsfPacket);
        }
    }
    av_packet_unref(m_packet);
    return MEDIA_SUCCESS;
}

int64_t MediaParserFFmpeg::getTimeStamp(AVPacket *packet, AVStream *stream)
{
    if (packet->pts != (qint64)AV_NOPTS_VALUE)
    {
        return (1000000 * (packet->pts * ((float)stream->time_base.num / stream->time_base.den)));
    }
    else if (packet->dts != (qint64)AV_NOPTS_VALUE)
    {
        return (1000000 * (packet->dts * ((float)stream->time_base.num / stream->time_base.den)));
    }
    else
    {
        return 0;
    }
}

MediaCodec推送代码

AMediaCodec_queueInputBuffer(m_mediaCodec, index, 0, encodedFrame->m_frameSize, encodedFrame->m_timeStamp, 0);

解决方案

1. 修复BSF包处理逻辑

当前getNextFrame中BSF的处理顺序完全错误：应该先发送原始包到BSF，再循环接收所有输出包，而不是先创建EncodedVideoFrame再调用av_bsf_receive_packet。修正后的逻辑如下：

int MediaParserFFmpeg::getNextFrame(MediaType *mediaType, AVEncodedFrame **outFrame)
{
    int ret = 0;
    *outFrame = nullptr;
    *mediaType = MEDIA_TYPE_UNKNOWN;

    ret = av_read_frame(m_formatCtx, m_packet);
    if (ret < 0)
    {
        av_packet_unref(m_packet);
        return ret == AVERROR_EOF ? MEDIA_INFO_PARSER_END_OF_STREAM : MEDIA_ERROR_PARSER_PARSE_FAILED;
    }

    if (m_packet->stream_index != m_vStreamIdx)
    {
        av_packet_unref(m_packet);
        return MEDIA_SUCCESS; // 跳过非视频流
    }

    if (m_vCodec == AV_CODEC_ID_H264 && m_isNal)
    {
        int res = av_bsf_send_packet(m_bsfcContext, m_packet);
        if (res != 0)
        {
            MEDIA_LOG() << "ZError:" << "Error sending packet to BSF";
            av_packet_unref(m_packet);
            return MEDIA_ERROR_PARSER_PARSE_FAILED;
        }

        // 循环接收BSF输出的所有包
        while (av_bsf_receive_packet(m_bsfcContext, m_bsfPacket) == 0)
        {
            int64_t pts = getTimeStamp(m_bsfPacket, m_formatCtx->streams[m_vStreamIdx]);
            bool isKeyFrame = (m_bsfPacket->flags & AV_PKT_FLAG_KEY) != 0;
            *outFrame = new EncodedVideoFrame(m_bsfPacket->data,
                                              m_bsfPacket->size,
                                              pts,
                                              m_width,
                                              m_height,
                                              isKeyFrame);
            *mediaType = MEDIA_TYPE_VIDEO;
            av_packet_unref(m_bsfPacket);
            break; // 每次返回一个帧，如需批量可调整
        }
    }
    else
    {
        // 非H264或无需BSF处理的情况
        int64_t pts = getTimeStamp(m_packet, m_formatCtx->streams[m_vStreamIdx]);
        bool isKeyFrame = (m_packet->flags & AV_PKT_FLAG_KEY) != 0;
        *outFrame = new EncodedVideoFrame(m_packet->data,
                                          m_packet->size,
                                          pts,
                                          m_width,
                                          m_height,
                                          isKeyFrame);
        *mediaType = MEDIA_TYPE_VIDEO;
    }

    av_packet_unref(m_packet);
    return *outFrame != nullptr ? MEDIA_SUCCESS : MEDIA_ERROR_PARSER_PARSE_FAILED;
}

2. 修复PTS转换精度问题

当前用浮点计算时间戳会导致精度丢失，改用FFmpeg提供的整数缩放函数av_rescale_q，确保时间戳准确转换为微秒（MediaCodec要求的时间单位）：

int64_t MediaParserFFmpeg::getTimeStamp(AVPacket *packet, AVStream *stream)
{
    int64_t ts = AV_NOPTS_VALUE;
    if (packet->pts != AV_NOPTS_VALUE)
    {
        ts = packet->pts;
    }
    else if (packet->dts != AV_NOPTS_VALUE)
    {
        ts = packet->dts;
    }
    else
    {
        return 0;
    }

    // 转换为微秒：AV_TIME_BASE是1e6，对应微秒
    return av_rescale_q(ts, stream->time_base, AV_TIME_BASE_Q);
}

3. 正确设置MediaCodec输入Buffer的Flags

推送时需要给关键帧设置BUFFER_FLAG_KEY_FRAME，帮助解码器正确识别帧类型，保证时序处理：

uint32_t flags = encodedFrame->isKeyFrame ? AMEDIACODEC_BUFFER_FLAG_KEY_FRAME : 0;
AMediaCodec_queueInputBuffer(m_mediaCodec, index, 0, encodedFrame->m_frameSize, encodedFrame->m_timeStamp, flags);

4. 控制数据包推送节奏

即使使用Surface直接渲染，也不能无限制快速推送数据包，需要根据MediaCodec的状态或当前帧的PTS来控制推送速度：

记录上一帧的PTS，计算当前帧与上一帧的时间差，调用usleep等待对应时长后再推送
或者通过AMediaCodec_dequeueOutputBuffer获取输出帧的时间戳，同步控制输入推送的节奏（推荐，更准确）

示例代码（基于PTS的简单控制）：

static int64_t lastPts = -1;
int64_t currentPts = encodedFrame->m_timeStamp;
if (lastPts != -1)
{
    int64_t sleepUs = currentPts - lastPts;
    if (sleepUs > 0)
    {
        usleep(sleepUs);
    }
}
lastPts = currentPts;

// 推送Buffer
uint32_t flags = encodedFrame->isKeyFrame ? AMEDIACODEC_BUFFER_FLAG_KEY_FRAME : 0;
AMediaCodec_queueInputBuffer(m_mediaCodec, index, 0, encodedFrame->m_frameSize, currentPts, flags);

内容的提问来源于stack exchange，提问作者forlayo

FFmpeg(libav)解封装HLS+MediaCodec Surface渲染播放过快问题排查

初始化与解析代码 #

MediaCodec推送代码 #

1. 修复BSF包处理逻辑 #

2. 修复PTS转换精度问题 #

3. 正确设置MediaCodec输入Buffer的Flags #