CUDA11.7拦截内存管理API遇无效设备上下文(201)错误求助

DevOps 小满

2026-7-6

CUDA驱动API拦截导致"无效设备上下文"错误的问题与解决

问题描述

我实现了一个拦截库，用于拦截CUDA驱动API cuMemAlloc() 和 cuGetProcAddress() 并进行转发。将LD_LIBRARY_PATH设置为该拦截库路径后，运行基于CUDA Runtime API编写的矩阵乘法程序。当程序调用cudaMalloc()时，内部会调用cuGetProcAddress()获取cuMemAlloc()的函数地址来执行分配操作，因此我拦截cuGetProcAddress()并将其返回的cuMemAlloc地址替换为拦截库中的实现，此时虽能成功拦截，但通过dlsym调用原库函数时返回201（无效设备上下文）错误。请问我的拦截流程是否正确？该如何解决此问题？

相关代码

拦截库代码

#include <stdio.h>
#include <dlfcn.h>
#include <string.h>
#include <stdint.h>

typedef enum cudaError_enum {
    CUDA_SUCCESS                              = 0,
    //...
    //...Not copied completely from my code
    //...
    CUDA_ERROR_UNKNOWN                        = 999
} CUresult;

typedef unsigned long long CUdeviceptr_v2;
typedef CUdeviceptr_v2 CUdeviceptr;
typedef uint64_t cuuint64_t;
char *cuda_filename = "libcuda.so.515.65.01";


CUresult cuMemAlloc(CUdeviceptr *dptr, size_t bytesize){
        printf("hijacking cuMemAlloc!\n");
        CUresult (*hello)(CUdeviceptr *, size_t);
        CUresult ret;
        void *table = NULL;

        table = dlopen(cuda_filename, RTLD_NOW | RTLD_NODELETE);
        if (!table) {
                printf("Error can't find library %s", cuda_filename);
        }
        hello = (CUresult (*)(CUdeviceptr *, size_t))dlsym(table, "cuMemAlloc");
        if (!hello){
                printf("can't find function cuMemAlloc");
        }
        ret = hello(dptr, bytesize);


        return ret;
}


CUresult cuGetProcAddress(const char *symbol, void **pfn, int cudaVersion, cuuint64_t flags){
        //printf("hijacking cuGetProcAddress!\n");
        CUresult (*hello)(const char *, void **, int, cuuint64_t);
        CUresult ret;
        void *table = NULL;

        table = dlopen(cuda_filename, RTLD_NOW | RTLD_NODELETE);
        if (!table) {
                printf("Error can't find library %s", cuda_filename);
        }
        hello = (CUresult (*)(const char *, void **, int, cuuint64_t))dlsym(table, "cuGetProcAddress");
        if (!hello){
                printf("can't find function cuGetProcAddress");
        }
        ret = hello(symbol, pfn, cudaVersion, flags);
        if (!strcmp(symbol, "cuGetProcAddress"))
                *pfn = cuGetProcAddress;
        if (!strcmp(symbol, "cuMemAlloc"))
                *pfn = cuMemAlloc;

        return ret;
}

编译与环境设置命令

gcc hook.c -fPIC -shared -ldl -o libcuda.so.1
export LD_LIBRARY_PATH=$PWD

Runtime API程序代码

#include "cuda_runtime.h"
#include "device_launch_parameters.h"
#include <sys/time.h>
#include <stdio.h>
#include <math.h>

const int Row=2048;
const int Col=2048;

__global__
void matrix_mul_gpu(int *M, int* N, int* P, int width)
{
    int i = threadIdx.x + blockDim.x * blockIdx.x;
    int j = threadIdx.y + blockDim.y * blockIdx.y;

    int sum = 0;
    for(int k=0;k<width;k++)
    {
        int a = M[j*width+k];
        int b = N[k*width+i];
        sum += a*b;
    }
    P[j*width+i] = sum;
}

int main()
{
            cudaError_t cuda_err = cudaSuccess;
            printf("func start \n");
            int *A = (int *)malloc(sizeof(int) * Row * Col);
            int *B = (int *)malloc(sizeof(int) * Row * Col);
            int *C = (int *)malloc(sizeof(int) * Row * Col);
            //malloc device memory
            int *d_dataA, *d_dataB, *d_dataC;
            printf("before cudaMalloc()\n");
            cuda_err = cudaMalloc((void**)&d_dataA, sizeof(int) *Row*Col);
            //cuda_err = cudaGetLastError();
            if (cudaSuccess != cuda_err)
            {
                    fprintf(stderr, "(%s:%s:%d)",  __FILE__, __FUNCTION__, __LINE__);
                    fprintf(stderr, "%s\n", cudaGetErrorString(cuda_err));
                    printf("cuda_err is %d\n", cuda_err);
                    exit(1);
            }
            printf("after cudaMalloc()\n");
            cudaMalloc((void**)&d_dataB, sizeof(int) *Row*Col);
            cudaMalloc((void**)&d_dataC, sizeof(int) *Row*Col);
            //set value
            for (int i = 0; i < Row*Col; i++) {
                A[i] = 90;
                B[i] = 10;
            }

            cudaMemcpy(d_dataA, A, sizeof(int) * Row * Col, cudaMemcpyHostToDevice);
            cudaMemcpy(d_dataB, B, sizeof(int) * Row * Col, cudaMemcpyHostToDevice);
            dim3 threadPerBlock(16, 16);
            dim3 blockNumber((Col+threadPerBlock.x-1)/ threadPerBlock.x, (Row+threadPerBlock.y-1)/ threadPerBlock.y );
            matrix_mul_gpu <<<blockNumber, threadPerBlock>>> (d_dataA, d_dataB, d_dataC, Col);
            cudaDeviceSynchronize();
            cudaMemcpy(C, d_dataC, sizeof(int) * Row * Col, cudaMemcpyDeviceToHost);
            free(A);
            free(B);
            free(C);
            cudaFree(d_dataA);
            cudaFree(d_dataB);
            cudaFree(d_dataC);

    return 0;
}

错误输出

nvcc matri.cu -o matri.out
./matri.out

func start
before cudaMalloc()
hijacking cuMemAlloc!
(matri.cu:main:40)invalid device context
cuda_err is 201

问题分析与解决

错误原因

你的拦截流程存在核心问题：每次调用拦截的cuMemAlloc和cuGetProcAddress时，都会重新调用dlopen加载原libcuda.so库。这会导致创建原库的多个独立实例，而CUDA的设备上下文是和特定库实例绑定的——Runtime API已经在原库实例中初始化了上下文，但你调用的是新实例中的cuMemAlloc，这个新实例没有任何设备上下文，因此返回无效设备上下文错误。

修正方案

全局缓存原库句柄与原函数指针：在库初始化阶段（而非每次函数调用时）加载原libcuda.so，并一次性获取所有需要转发的原函数地址，避免重复加载库。
使用构造函数初始化：利用GCC的__attribute__((constructor))属性，在库加载时自动完成原库的打开和函数地址的获取。
避免递归拦截问题：确保拦截的函数只转发到原库的函数，而非再次调用拦截版本。

修正后的拦截库代码

#include <stdio.h>
#include <dlfcn.h>
#include <string.h>
#include <stdint.h>

typedef enum cudaError_enum {
    CUDA_SUCCESS                              = 0,
    //...
    //...保留你原有的错误码定义
    //...
    CUDA_ERROR_UNKNOWN                        = 999
} CUresult;

typedef unsigned long long CUdeviceptr_v2;
typedef CUdeviceptr_v2 CUdeviceptr;
typedef uint64_t cuuint64_t;

// 全局变量：缓存原库句柄和原函数指针
static void *g_cuda_handle = NULL;
static CUresult (*g_original_cuMemAlloc)(CUdeviceptr *, size_t) = NULL;
static CUresult (*g_original_cuGetProcAddress)(const char *, void **, int, cuuint64_t) = NULL;
const char *cuda_filename = "libcuda.so.515.65.01";

// 库初始化函数：加载原库并获取原函数地址
__attribute__((constructor))
void init_hook() {
    g_cuda_handle = dlopen(cuda_filename, RTLD_NOW | RTLD_NODELETE);
    if (!g_cuda_handle) {
        fprintf(stderr, "Error loading library %s: %s\n", cuda_filename, dlerror());
        return;
    }

    // 获取原cuMemAlloc地址
    g_original_cuMemAlloc = (CUresult (*)(CUdeviceptr *, size_t))dlsym(g_cuda_handle, "cuMemAlloc");
    if (!g_original_cuMemAlloc) {
        fprintf(stderr, "Error finding cuMemAlloc: %s\n", dlerror());
    }

    // 获取原cuGetProcAddress地址
    g_original_cuGetProcAddress = (CUresult (*)(const char *, void **, int, cuuint64_t))dlsym(g_cuda_handle, "cuGetProcAddress");
    if (!g_original_cuGetProcAddress) {
        fprintf(stderr, "Error finding cuGetProcAddress: %s\n", dlerror());
    }
}

// 拦截的cuMemAlloc：直接调用缓存的原函数
CUresult cuMemAlloc(CUdeviceptr *dptr, size_t bytesize){
    printf("hijacking cuMemAlloc!\n");
    if (!g_original_cuMemAlloc) {
        fprintf(stderr, "Original cuMemAlloc not available\n");
        return CUDA_ERROR_UNKNOWN;
    }
    return g_original_cuMemAlloc(dptr, bytesize);
}

// 拦截的cuGetProcAddress：调用原函数后替换指定符号
CUresult cuGetProcAddress(const char *symbol, void **pfn, int cudaVersion, cuuint64_t flags){
    if (!g_original_cuGetProcAddress) {
        fprintf(stderr, "Original cuGetProcAddress not available\n");
        return CUDA_ERROR_UNKNOWN;
    }
    CUresult ret = g_original_cuGetProcAddress(symbol, pfn, cudaVersion, flags);
    
    // 替换指定符号为拦截版本
    if (!strcmp(symbol, "cuMemAlloc")) {
        *pfn = cuMemAlloc;
    } else if (!strcmp(symbol, "cuGetProcAddress")) {
        *pfn = cuGetProcAddress;
    }
    
    return ret;
}

验证步骤

用修正后的代码重新编译拦截库：

gcc hook.c -fPIC -shared -ldl -o libcuda.so.1

设置环境变量并运行程序：

export LD_LIBRARY_PATH=$PWD
./matri.out

此时拦截逻辑会正常工作，且不会出现无效设备上下文错误，因为所有转发调用都使用同一个原库实例，共享Runtime API初始化的设备上下文。

内容的提问来源于stack exchange，提问作者yyyfish

CUDA11.7拦截内存管理API遇无效设备上下文(201)错误求助

问题描述 #

相关代码 #

拦截库代码 #

编译与环境设置命令 #

Runtime API程序代码 #

错误输出 #

问题分析与解决 #

错误原因 #

修正方案 #

修正后的拦截库代码 #

验证步骤 #