使用CuDNN实现3D卷积时遇CUDNN_STATUS_BAD_PARAM错误求助

问题描述 #

使用CuDNN 8.4 + CUDA 11.x在Ubuntu 18.04编写卷积程序，NVCC编译通过，但调用cudnnGetConvolutionForwardAlgorithm_v7时触发CUDNN_STATUS_BAD_PARAM错误，检查参数未发现明显问题，寻求调试与修复方案。

环境信息 #

• CuDNN 8.4
• CUDA 11.x
• Ubuntu 18.04

原始代码 #

#include <cudnn.h>
#include <cuda.h>
#include <iostream>
#include <cstdlib>
#include <time.h>
using namespace std;

#define checkCUDNN(expression)                               
  {                                                          
    cudnnStatus_t status = (expression);                     
    if (status != CUDNN_STATUS_SUCCESS) {                    
      std::cerr << "Error on line " << __LINE__ << ": "      
                << cudnnGetErrorString(status) << std::endl; 
      std::exit(EXIT_FAILURE);                               
    }                                                        
  }

int main(int argc, char* argv[]){
    int H = atoi(argv[1]);
    int W = atoi(argv[2]);
    int C = atoi(argv[3]);

    int FH = atoi(argv[4]);
    int FW = atoi(argv[5]);
    int K = atoi(argv[6]);

    double* input = new double[K*H*W];
    double* kernel = new double[K*C*FH*FW];

    for(int k=0; k<K; k++){
        for(int c=0; c<C; c++){
            for(int i=0; i<FH; i++){
                for(int j=0; j<FW; j++){
                    kernel[k*FH*FW*C+c*FH*FW+i*FW+j] = (c+k) * (i+j);
                }
            }
        }
    }

    for(int c=0; c<C; c++){
        for(int i=0; i<H; i++){
            for(int j=0; j<W; j++){
                input[c*H*W+i*W+j] = c * (i+j);
            }
        }
    }
    double* output = new double[K*H*W];

    cudnnHandle_t cudnn;
    checkCUDNN(cudnnCreate(&cudnn));

    cudnnTensorDescriptor_t input_descriptor;
    checkCUDNN(cudnnCreateTensorDescriptor(&input_descriptor));
    int dimA[3] = {C, H, W};
    int strideA[3] = {1, 1, 1};
    checkCUDNN(cudnnSetTensorNdDescriptor(input_descriptor, CUDNN_DATA_DOUBLE, 3, dimA, strideA));
    
    cudnnFilterDescriptor_t kernel_descriptor;
    checkCUDNN(cudnnCreateFilterDescriptor(&kernel_descriptor));
    checkCUDNN(cudnnSetFilter4dDescriptor(kernel_descriptor, CUDNN_DATA_DOUBLE, CUDNN_TENSOR_NCHW, K, C, FH, FW));

    cudnnTensorDescriptor_t output_descriptor;
    checkCUDNN(cudnnCreateTensorDescriptor(&output_descriptor));
    int dimA2[3] = {K, W, H};
    int strideA2[3] = {1, 1, 1};
    checkCUDNN(cudnnSetTensorNdDescriptor(output_descriptor, CUDNN_DATA_DOUBLE, 3, dimA2, strideA2));
    
    cudnnConvolutionDescriptor_t convolution_descriptor;
    checkCUDNN(cudnnCreateConvolutionDescriptor(&convolution_descriptor));
    int padding[3] = {0, 1, 1};
    int strideA3[3] = {1, 1, 1};
    int dilation[3] = {1, 1, 1};
    checkCUDNN(cudnnSetConvolutionNdDescriptor(convolution_descriptor, 3, padding, strideA3, dilation, CUDNN_CROSS_CORRELATION, CUDNN_DATA_DOUBLE));

    cudnnConvolutionFwdAlgoPerf_t convolution_algorithm;
    int perf_count;
    checkCUDNN(cudnnGetConvolutionForwardAlgorithm_v7(cudnn, input_descriptor, kernel_descriptor, convolution_descriptor, output_descriptor, CUDNN_CONVOLUTION_FWD_ALGO_COUNT , &perf_count, &convolution_algorithm));

    size_t workspace_bytes = 0;
    checkCUDNN(cudnnGetConvolutionForwardWorkspaceSize(cudnn,
                                            input_descriptor,
                                            kernel_descriptor,
                                            convolution_descriptor,
                                            output_descriptor,
                                            convolution_algorithm.algo,
                                            &workspace_bytes));
    
    double* input_device;
    double* kernel_device;
    double* output_device;
    void* d_workspace{nullptr};
    cudaMalloc(&d_workspace, workspace_bytes);

    cudaMalloc((void **)& input_device, C*H*W*sizeof(double));
    cudaMalloc((void **)& kernel_device, K*C*FH*FW*sizeof(double));
    cudaMalloc((void **)& output_device, K*H*W*sizeof(double));

    cudaMemcpy(input_device, input, C*H*W*sizeof(double), cudaMemcpyHostToDevice);
    cudaMemcpy(kernel_device, kernel, K*C*FH*FW*sizeof(double), cudaMemcpyHostToDevice);
    cudaMemcpy(output_device, output, K*H*W*sizeof(double), cudaMemcpyHostToDevice);

    const double alpha = 1, beta = 0;

    struct timespec time_start = {0, 0};
    struct timespec time_end = {0, 0};

    clock_gettime(CLOCK_MONOTONIC, &time_start);

    cudnnConvolutionForward(cudnn,
                            &alpha,
                            input_descriptor,
                            input_device,
                            kernel_descriptor,
                            kernel_device,
                            convolution_descriptor,
                            convolution_algorithm.algo,
                            d_workspace,
                            workspace_bytes,
                            &beta,
                            output_descriptor,
                            output_device);
    

    clock_gettime(CLOCK_MONOTONIC, &time_end);

    double exec_time = (time_end.tv_nsec - time_start.tv_nsec) / 1000000000.0;

    cudaMemcpy(output, output_device, K*H*W*sizeof(double), cudaMemcpyDeviceToHost);

    double checksum = 0.0;
    for(int i=0; i<K*H*W; i++)
        checksum += output[i];
    
    cout<<"C3 checksum: "<<checksum<<"\t";
    cout<<"C3 execution time: "<<exec_time<<endl;
    
    cudaFree(input_device);
    cudaFree(output_device);
    cudaFree(kernel_device);
    cudaFree(d_workspace);

    cudnnDestroyTensorDescriptor(input_descriptor);
    cudnnDestroyTensorDescriptor(output_descriptor);
    cudnnDestroyFilterDescriptor(kernel_descriptor);
    cudnnDestroyConvolutionDescriptor(convolution_descriptor);

    cudnnDestroy(cudnn);
    
    free(input);
    free(output);
    free(kernel);

    return 0;
}

问题排查与修复方案 #

1. 核心错误：维度与描述符不匹配 #

CuDNN对张量、滤波器、卷积的维度一致性要求严格，代码中多处违反了这一规则：

• 输入/输出张量应为4D（NCHW格式）：CuDNN的NCHW格式要求张量维度为[N, C, H, W]（批量数N、通道数C、高H、宽W），原代码使用3D描述符且步幅设置错误，导致参数不合法：

  // 修正输入张量描述符
  cudnnTensorDescriptor_t input_descriptor;
  checkCUDNN(cudnnCreateTensorDescriptor(&input_descriptor));
  int input_dims[4] = {1, C, H, W}; // N=1（单样本）
  int input_strides[4] = {C*H*W, H*W, W, 1}; // 按NCHW布局计算步幅
  checkCUDNN(cudnnSetTensorNdDescriptor(input_descriptor, CUDNN_DATA_DOUBLE, 4, input_dims, input_strides));
  

• 输出张量维度需匹配卷积计算结果：先根据卷积参数计算输出高宽，再设置正确的4D描述符：

  // 计算输出高宽（标准卷积公式）
  int OH = (H + 2*padding[0] - dilation[0]*(FH-1) - 1)/strideA3[0] + 1;
  int OW = (W + 2*padding[1] - dilation[1]*(FW-1) - 1)/strideA3[1] + 1;
  
  // 修正输出张量描述符
  cudnnTensorDescriptor_t output_descriptor;
  checkCUDNN(cudnnCreateTensorDescriptor(&output_descriptor));
  int output_dims[4] = {1, K, OH, OW};
  int output_strides[4] = {K*OH*OW, OH*OW, OW, 1};
  checkCUDNN(cudnnSetTensorNdDescriptor(output_descriptor, CUDNN_DATA_DOUBLE, 4, output_dims, output_strides));
  

• 卷积描述符的空间维度数错误：原代码实现的是2D卷积（仅H/W两个空间维度），cudnnSetConvolutionNdDescriptor的第一个参数应传2，而非3，同时padding/stride/dilation数组仅需2个元素：

  cudnnConvolutionDescriptor_t convolution_descriptor;
  checkCUDNN(cudnnCreateConvolutionDescriptor(&convolution_descriptor));
  int padding[2] = {1, 1}; // 对应H/W维度的padding
  int strideA3[2] = {1, 1}; // 对应H/W维度的步幅
  int dilation[2] = {1, 1}; // 对应H/W维度的膨胀系数
  checkCUDNN(cudnnSetConvolutionNdDescriptor(convolution_descriptor, 2, padding, strideA3, dilation, CUDNN_CROSS_CORRELATION, CUDNN_DATA_DOUBLE));
  

2. 数据分配修正 #

输入数据的内存分配与实际需求不符，应改为：

// 原代码错误：分配了K*H*W，实际需要C*H*W（单样本N=1）
double* input = new double[C*H*W];
// 输出数据应匹配计算出的OH/OW
double* output = new double[K*OH*OW];

3. 调试技巧 #

• 启用CuDNN日志：运行程序前设置环境变量，获取详细错误信息：

  export CUDNN_LOGINFO_DBG=1
  export CUDNN_LOGDEST_DBG=stdout
  ./your_program
  

• 打印描述符信息：使用cudnnPrintTensorDescriptor、cudnnPrintFilterDescriptor等函数（需包含cudnn_adv_infer.h），验证描述符的维度、步幅是否符合预期。
• 验证维度合法性：手动计算卷积输出尺寸，确保输入、滤波器、输出的维度满足卷积数学关系，CuDNN会严格校验这一点。

内容的提问来源于stack exchange，提问作者Mingzhe