DXGI桌面复制转CUDA处理帧时，注册Direct3D纹理报错（参数无效）

算力阿峰

2026-6-14

DXGI桌面复制纹理注册CUDA失败：invalid argument

我用DXGI桌面复制捕获屏幕帧，想把Direct3D 11纹理注册到CUDA做GPU处理，但调用cudaGraphicsD3D11RegisterResource()时总是报错：Failed to register Direct3D texture with CUDA: invalid argument。

我的目标

使用DXGI桌面复制捕获屏幕帧
将帧存储在Direct3D 11 GPU纹理（ID3D11Texture2D）中
注册纹理到CUDA进行后续处理

实现代码

bool DesktopDuplicator::getFrameGPU(cudaGraphicsResource** cudaResource, int& outWidth, int& outHeight, UINT timeoutMs)
{
    Microsoft::WRL::ComPtr<ID3D11Texture2D> m_gpuTex;
    if (!m_duplication)
    {
        std::cerr << "Duplication not initialized.\n";
        return false;
    }

    // 1. Acquire the next frame
    DXGI_OUTDUPL_FRAME_INFO frameInfo = {};
    Microsoft::WRL::ComPtr<IDXGIResource> desktopResource;

    HRESULT hr = m_duplication->AcquireNextFrame(timeoutMs, &frameInfo, &desktopResource);
    if (hr == DXGI_ERROR_WAIT_TIMEOUT)
    {
        return false;
    }
    else if (FAILED(hr))
    {
        std::cerr << "AcquireNextFrame failed with HR=0x" << std::hex << hr << "\n";
        m_duplication->ReleaseFrame();
        return false;
    }

    // 2. Query for ID3D11Texture2D
    Microsoft::WRL::ComPtr<ID3D11Texture2D> d3dTex;
    hr = desktopResource.As(&d3dTex);
    if (FAILED(hr) || !d3dTex)
    {
        std::cerr << "Failed to get ID3D11Texture2D.\n";
        m_duplication->ReleaseFrame();
        return false;
    }

    // 3. Get texture description and create a CUDA-compatible GPU texture if needed
    D3D11_TEXTURE2D_DESC desc;
    d3dTex->GetDesc(&desc);
    outWidth = desc.Width;
    outHeight = desc.Height;

    D3D11_TEXTURE2D_DESC gpuDesc = {};
    gpuDesc.Width = desc.Width;
    gpuDesc.Height = desc.Height;
    gpuDesc.MipLevels = 1;
    gpuDesc.ArraySize = 1;
    gpuDesc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; // CUDA-compatible format
    gpuDesc.SampleDesc.Count = 1;
    gpuDesc.Usage = D3D11_USAGE_DEFAULT;
    gpuDesc.BindFlags = D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_RENDER_TARGET;
    gpuDesc.CPUAccessFlags = 0;
    gpuDesc.MiscFlags = D3D11_RESOURCE_MISC_SHARED; // Removed SHARED_KEYEDMUTEX

    hr = m_device->CreateTexture2D(&gpuDesc, nullptr, &m_gpuTex);
    if (FAILED(hr))
    {
        std::cerr << "Failed to create CUDA-compatible texture. HRESULT: " << std::hex << hr << "\n";
        m_duplication->ReleaseFrame();
        return false;
    }

    // 4. Copy (GPU->GPU) the DWM frame into our own texture
    m_context->CopyResource(m_gpuTex.Get(), d3dTex.Get());

    // 5. Obtain shared handle
    Microsoft::WRL::ComPtr<IDXGIResource> dxgiResource;
    hr = m_gpuTex.As(&dxgiResource);
    HANDLE sharedHandle = nullptr;
    if (SUCCEEDED(hr))
    {
        hr = dxgiResource->GetSharedHandle(&sharedHandle);
    }
    if (FAILED(hr) || !sharedHandle)
    {
        std::cerr << "Failed to obtain shared handle for Direct3D texture.\n";
        m_duplication->ReleaseFrame();
        return false;
    }

    // 6. Register the texture with CUDA using the shared handle
    cudaError_t cudaStatus = cudaGraphicsD3D11RegisterResource(cudaResource, d3dTex.Get(), cudaGraphicsRegisterFlagsNone);
    if (cudaStatus != cudaSuccess)
    {
        std::cerr << "Failed to register Direct3D texture with CUDA: " << cudaGetErrorString(cudaStatus) << "\n";
        m_duplication->ReleaseFrame();
        return false;
    }

    // Unmap and release the frame
    m_duplication->ReleaseFrame();
    return true;
}

已尝试的排查步骤

确保纹理使用D3D11_USAGE_DEFAULT创建
移除了D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX（CUDA不支持）
使用CUDA兼容格式DXGI_FORMAT_B8G8R8A8_UNORM
确认GetSharedHandle()返回有效句柄
尝试用cudaGraphicsRegisterFlagsNone作为注册标志
检查纹理创建HRESULT无失败

问题根源及修复方案

1. 错误注册了DXGI原始纹理

代码第6步错误地将DXGI返回的d3dTex（桌面复制原始纹理）传给CUDA注册函数，但该原始纹理的属性（如绑定标志、设备上下文归属）不满足CUDA注册要求。应该注册你自己创建的m_gpuTex。

2. 冗余的共享句柄操作

获取m_gpuTex的共享句柄但未使用，这一步完全多余，CUDA可直接通过D3D资源指针注册，无需共享句柄（除非跨设备/进程共享）。

3. 纹理绑定标志冗余

创建m_gpuTex时设置的D3D11_BIND_RENDER_TARGET是多余的，仅保留D3D11_BIND_SHADER_RESOURCE即可减少兼容性问题。

修复后的代码片段

bool DesktopDuplicator::getFrameGPU(cudaGraphicsResource** cudaResource, int& outWidth, int& outHeight, UINT timeoutMs)
{
    Microsoft::WRL::ComPtr<ID3D11Texture2D> m_gpuTex;
    if (!m_duplication)
    {
        std::cerr << "Duplication not initialized.\n";
        return false;
    }

    // 1. Acquire the next frame
    DXGI_OUTDUPL_FRAME_INFO frameInfo = {};
    Microsoft::WRL::ComPtr<IDXGIResource> desktopResource;

    HRESULT hr = m_duplication->AcquireNextFrame(timeoutMs, &frameInfo, &desktopResource);
    if (hr == DXGI_ERROR_WAIT_TIMEOUT)
    {
        return false;
    }
    else if (FAILED(hr))
    {
        std::cerr << "AcquireNextFrame failed with HR=0x" << std::hex << hr << "\n";
        m_duplication->ReleaseFrame();
        return false;
    }

    // 2. Query for ID3D11Texture2D
    Microsoft::WRL::ComPtr<ID3D11Texture2D> d3dTex;
    hr = desktopResource.As(&d3dTex);
    if (FAILED(hr) || !d3dTex)
    {
        std::cerr << "Failed to get ID3D11Texture2D.\n";
        m_duplication->ReleaseFrame();
        return false;
    }

    // 3. Get texture description and create a CUDA-compatible GPU texture
    D3D11_TEXTURE2D_DESC desc;
    d3dTex->GetDesc(&desc);
    outWidth = desc.Width;
    outHeight = desc.Height;

    D3D11_TEXTURE2D_DESC gpuDesc = {};
    gpuDesc.Width = desc.Width;
    gpuDesc.Height = desc.Height;
    gpuDesc.MipLevels = 1;
    gpuDesc.ArraySize = 1;
    gpuDesc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; // CUDA-compatible format
    gpuDesc.SampleDesc.Count = 1;
    gpuDesc.Usage = D3D11_USAGE_DEFAULT;
    gpuDesc.BindFlags = D3D11_BIND_SHADER_RESOURCE; // 仅保留必要绑定标志
    gpuDesc.CPUAccessFlags = 0;
    gpuDesc.MiscFlags = 0; // 无需SHARED标志

    hr = m_device->CreateTexture2D(&gpuDesc, nullptr, &m_gpuTex);
    if (FAILED(hr))
    {
        std::cerr << "Failed to create CUDA-compatible texture. HRESULT: " << std::hex << hr << "\n";
        m_duplication->ReleaseFrame();
        return false;
    }

    // 4. Copy (GPU->GPU) the DWM frame into our own texture
    m_context->CopyResource(m_gpuTex.Get(), d3dTex.Get());
    // 确保GPU完成复制操作
    m_context->Flush();

    // 5. 注册自己创建的兼容纹理到CUDA
    cudaError_t cudaStatus = cudaGraphicsD3D11RegisterResource(cudaResource, m_gpuTex.Get(), cudaGraphicsRegisterFlagsNone);
    if (cudaStatus != cudaSuccess)
    {
        std::cerr << "Failed to register Direct3D texture with CUDA: " << cudaGetErrorString(cudaStatus) << "\n";
        m_duplication->ReleaseFrame();
        return false;
    }

    // 释放帧资源
    m_duplication->ReleaseFrame();
    return true;
}

额外检查项

确保D3D11设备是CUDA兼容的：创建设备时添加D3D11_CREATE_DEVICE_BGRA_SUPPORT标志，且使用支持CUDA的GPU。
确认D3D和CUDA使用同一GPU：多GPU系统中需绑定到同一块显卡。
注册前同步GPU：调用m_context->Flush()确保复制操作完成后再注册纹理。

内容的提问来源于stack exchange，提问作者Jerry