如何用Media Foundation解码MFVideoFormat_H264字节流并转RGB？

一、H.264裸流输入解码流程 #

1. 分割NALU单元 #

Socket接收的字节流是连续的H.264裸数据，需按0x00 0x00 0x00 0x01起始码分割出单个NALU（网络抽象层单元），这是Media Foundation解码器的最小输入单元。

2. 封装为IMFSample并输入解码器 #

将分割后的NALU数据封装为IMFMediaBuffer，再关联到IMFSample，最后输入解码器：

// 伪代码：处理单段NALU数据
std::vector<BYTE> naluData = ...; // 分割后的NALU字节数组
IMFMediaBuffer* pBuffer = nullptr;
HRESULT hr = MFCreateMemoryBuffer(naluData.size(), &pBuffer);
if (SUCCEEDED(hr)) {
    BYTE* pData = nullptr;
    DWORD cbCurrent = 0;
    hr = pBuffer->Lock(&pData, nullptr, &cbCurrent);
    if (SUCCEEDED(hr)) {
        memcpy(pData, naluData.data(), naluData.size());
        pBuffer->Unlock();
        pBuffer->SetCurrentLength(naluData.size());

        // 创建样本并设置时间戳（按30fps推算，单位100纳秒）
        IMFSample* pSample = nullptr;
        hr = MFCreateSample(&pSample);
        if (SUCCEEDED(hr)) {
            pSample->AddBuffer(pBuffer);
            LONGLONG pts = (frameIndex * 10000000LL) / 30;
            pSample->SetSampleTime(pts);
            pSample->SetSampleDuration(10000000LL / 30);

            // 输入解码器，处理"不接受输入"的情况
            hr = pDecoderTransform->ProcessInput(0, pSample, 0);
            if (hr == MF_E_NOTACCEPTING) {
                ProcessPendingOutput(pDecoderTransform);
                hr = pDecoderTransform->ProcessInput(0, pSample, 0);
            }
            pSample->Release();
        }
        pBuffer->Release();
    }
}

3. 获取解码输出 #

循环调用ProcessOutput获取解码后的NV12格式样本：

void ProcessPendingOutput(IMFTransform* pDecoder) {
    HRESULT hr = S_OK;
    while (true) {
        MFT_OUTPUT_DATA_BUFFER outputData = {};
        DWORD dwStatus = 0;
        hr = pDecoder->ProcessOutput(0, 1, &outputData, &dwStatus);
        
        if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT) break; // 无更多输出
        if (FAILED(hr)) break;

        // 处理NV12格式的输出样本
        HandleNV12Sample(outputData.pSample);
        outputData.pSample->Release();
    }
}

二、NV12转RGB的GPU加速方案 #

1. Media Foundation颜色转换MFT #

使用系统内置的颜色转换MFT实现硬件加速格式转换，支持NV12转ARGB32/RGB24等格式：

IMFTransform* pColorConvert = nullptr;
HRESULT hr = CoCreateInstance(CLSID_ColorConvertMFT, nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&pColorConvert));
if (SUCCEEDED(hr)) {
    // 设置输入类型（NV12，需和解码输出参数一致）
    IMFMediaType* pInputType = nullptr;
    hr = MFCreateMediaType(&pInputType);
    if (SUCCEEDED(hr)) {
        pInputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
        pInputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12);
        pInputType->SetUINT32(MF_MT_FRAME_WIDTH, width);
        pInputType->SetUINT32(MF_MT_FRAME_HEIGHT, height);
        pColorConvert->SetInputType(0, pInputType, 0);
        pInputType->Release();
    }

    // 设置输出类型（ARGB32）
    IMFMediaType* pOutputType = nullptr;
    hr = MFCreateMediaType(&pOutputType);
    if (SUCCEEDED(hr)) {
        pOutputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
        pOutputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_ARGB32);
        pOutputType->SetUINT32(MF_MT_FRAME_WIDTH, width);
        pOutputType->SetUINT32(MF_MT_FRAME_HEIGHT, height);
        pColorConvert->SetOutputType(0, pOutputType, 0);
        pOutputType->Release();
    }

    // 输入NV12样本，获取ARGB32输出
    hr = pColorConvert->ProcessInput(0, pNV12Sample, 0);
    if (SUCCEEDED(hr)) {
        MFT_OUTPUT_DATA_BUFFER outputData = {};
        DWORD dwStatus = 0;
        hr = pColorConvert->ProcessOutput(0, 1, &outputData, &dwStatus);
        if (SUCCEEDED(hr)) {
            HandleARGBSample(outputData.pSample);
            outputData.pSample->Release();
        }
    }
    pColorConvert->Release();
}

2. Direct2D结合DX11纹理渲染 #

直接将NV12纹理通过Direct2D渲染到窗口，避免CPU拷贝：

// 将IMFMediaBuffer转换为DX11 NV12纹理
ID3D11Texture2D* pNV12Texture = nullptr;
IMF2DBuffer* p2DBuffer = nullptr;
hr = pNV12Buffer->QueryInterface(IID_PPV_ARGS(&p2DBuffer));
if (SUCCEEDED(hr)) {
    IDXGISurface* pDXGISurface = nullptr;
    p2DBuffer->GetSurface(&pDXGISurface);
    pDXGISurface->QueryInterface(IID_PPV_ARGS(&pNV12Texture));
    pDXGISurface->Release();
    p2DBuffer->Release();
} else {
    // 系统内存buffer需复制到GPU纹理
    D3D11_TEXTURE2D_DESC desc = {};
    desc.Width = width; desc.Height = height;
    desc.Format = DXGI_FORMAT_NV12;
    desc.Usage = D3D11_USAGE_DEFAULT;
    desc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
    pD3D11Device->CreateTexture2D(&desc, nullptr, &pNV12Texture);
    
    BYTE* pData = nullptr;
    pNV12Buffer->Lock(&pData, nullptr, nullptr);
    pD3D11DeviceContext->UpdateSubresource(pNV12Texture, 0, nullptr, pData, width, 0);
    pNV12Buffer->Unlock();
}

// Direct2D渲染NV12纹理到窗口
ID2D1Bitmap1* pD2DBitmap = nullptr;
D2D1_BITMAP_PROPERTIES1 props = {};
props.pixelFormat.format = DXGI_FORMAT_NV12;
props.pixelFormat.alphaMode = D2D1_ALPHA_MODE_IGNORE;
pD2DDeviceContext->CreateBitmapFromDxgiSurface(pNV12Texture, &props, &pD2DBitmap);

pD2DRenderTarget->BeginDraw();
pD2DRenderTarget->DrawBitmap(pD2DBitmap, D2D1::RectF(0, 0, windowWidth, windowHeight));
pD2DRenderTarget->EndDraw();

pD2DBitmap->Release();
pNV12Texture->Release();

关键注意事项 #

• 时间戳设置：实时流若无PTS信息，需按帧率推算时间戳，避免解码器时序错误。
• 硬件加速优先：枚举解码器和颜色转换MFT时，指定MFT_ENUM_FLAG_HARDWARE优先选择硬件实现，提升性能。
• 错误处理：ProcessInput返回MF_E_NOTACCEPTING时，必须先处理完所有输出再继续输入。

内容的提问来源于stack exchange，提问作者user666