立体视觉图像校正失败，无法生成视差图与深度信息求助

正在编写双目图像校正代码，目标是计算深度信息并生成点云，但目前因图像校正失败，无法生成视差图，进而无法完成深度计算。

实现的相机标定与立体图像校正函数 #

def CamIntr(image_paths, chessboard_params, square_size):
    # Calibration chessboard parameters
    #chessboard_params = [6, 9]  # Number of corners in y, x
    chessboard_params.append(square_size)  # Square size in m
    num_corners = chessboard_params[0] * chessboard_params[1]

    # Termination criteria for corner refinement
    criteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 30, 0.001)

    # Prepare object points (world coordinates)
    objp = np.zeros((chessboard_params[0] * chessboard_params[1], 3), np.float32)
    objp[:, :2] = np.mgrid[0:chessboard_params[0], 0:chessboard_params[1]].T.reshape(-1, 2)*chessboard_params[2]

    # Arrays to store object points and image points from all images
    objpoints = []  # 3D points in real-world space
    imgpoints = []  # 2D points in image plane

    for path in image_paths:
        # Read the image
        image = cv.imread(path)
        gray = cv.cvtColor(image, cv.COLOR_BGR2GRAY)  # Convert to grayscale

        # Find chessboard corners
        ret, corners = cv.findChessboardCorners(gray, (chessboard_params[0], chessboard_params[1]), None)

        
        #print(f"Chessboard found in {path}")
        objpoints.append(objp)
        corners2 = cv.cornerSubPix(gray, corners, (11, 11), (-1, -1), criteria)
        imgpoints.append(corners2)
        # Draw and display the corners
        #cv.drawChessboardCorners(image, (chessboard_params[1], chessboard_params[0]), corners2, ret)
        #cv.imshow('img', image)
        #cv.imwrite("Chessboard_draw.jpg", image) 
        #cv.waitKey(500)
      
    
    ret, mtx, dist, rvecs, tvecs = cv.calibrateCamera(objpoints, imgpoints, gray.shape[::-1], None, None)
    
    
    return ret, mtx, dist, rvecs, tvecs, objpoints, imgpoints 



def RectifyStereoImages(img_left, img_right, chessboardL, chessboardR):
    # Load stereo calibration data (intrinsics, extrinsics, etc.)
    
    gray_left = cv.imread(img_left,0)
    gray_right = cv.imread(img_right,0)
    
    ##CALIBRATION
    
    _, mtx_left, dist_left, rvecs_left, tvecs_left, objpoints_left, imgpoints_left = CamIntr(chessboardL, [6,9], 0.025)


    _, mtx_right, dist_right, rvecs_right, tvecs_right, objpoints_right, imgpoints_right = CamIntr(chessboardR, [6,9], 0.025)

    
   ##STEREO VISION CALIBRATION
   
    _, _, _, _, _, R, T, E, F = cv.stereoCalibrate(objpoints_left, imgpoints_left, imgpoints_right,
                                                    mtx_left, dist_left, mtx_right, dist_right,
                                                    gray_left.shape[::-1], criteria=(cv.CALIB_FIX_INTRINSIC, 30, 0.001))
    
    flags = 0
    flags |= cv.CALIB_FIX_INTRINSIC
    # Here we fix the intrinsic camara matrixes so that only Rot, Trns, Emat and Fmat are calculated.
    # Hence intrinsic parameters are the same 
    
    criteria_stereo= (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER, 30, 0.001)
    
    # This step is performed to transformation between the two cameras and calculate Essential and Fundamenatl matrix
    retStereo, newCameraMatrixL, distL, newCameraMatrixR, distR, rot, trans, essentialMatrix, fundamentalMatrix = cv.stereoCalibrate(objpoints_left, imgpoints_left, imgpoints_right, mtx_left, dist_left, mtx_right, dist_right, gray_left.shape[::-1], criteria_stereo, flags)
    
    ##STEREO RECTIFICATION

    rectifyScale= 1
    rectL, rectR, projMatrixL, projMatrixR, Q, roi_L, roi_R= cv.stereoRectify(newCameraMatrixL, distL, newCameraMatrixR, distR, gray_left.shape[::-1], rot, trans, rectifyScale,(0,0), alpha = 0.5)
    
    stereoMapL = cv.initUndistortRectifyMap(mtx_left, distL, rectL, projMatrixL, gray_left.shape[::-1], cv.CV_16SC2)
    stereoMapR = cv.initUndistortRectifyMap(mtx_right, distR, rectR, projMatrixR, gray_right.shape[::-1], cv.CV_16SC2)
    
    map_left_x = stereoMapL[0]
    map_left_y = stereoMapL[1]
    map_right_x = stereoMapR[0]
    map_right_y = stereoMapR[1]
    
    # Rectify the images
    img_left_rectified = cv.remap(gray_left, map_left_x, map_left_y, cv.INTER_LANCZOS4, cv.BORDER_CONSTANT, 0)
    img_right_rectified = cv.remap(gray_right, map_right_x, map_right_y, cv.INTER_LANCZOS4, cv.BORDER_CONSTANT, 0)
    
    
    return img_left_rectified, img_right_rectified

函数调用代码 #

# Get all files in the directory
all_files = os.listdir(r"C:\Users\emmay\Desktop\Folder")
# Filter files starting with the prefix "Chessboard"
chessboard1 = [file for file in all_files if file.startswith("Chessboard1")]
chessboard2 = [file for file in all_files if file.startswith("Chessboard2")]
squaresize = 0.025

img1 = "Chessboard1_1.png"
img2 = "Chessboard2_1.png"


imgL = cv.imread(img1,0)
imgR = cv.imread(img2,0)

fig, ax = plt.subplots()
im = ax.imshow(imgL)
plt.show()
ax.set_title("Left original image")

fig, ax = plt.subplots()
im = ax.imshow(imgR)
plt.show()
ax.set_title("Right original image")

L_rect, R_rect = RectifyStereoImages(img1, img2, chessboard1, chessboard2)
#images after rectification
fig, ax = plt.subplots()
im = ax.imshow(L_rect)
plt.show()
ax.set_title("Left rectified image")

fig, ax = plt.subplots()
im = ax.imshow(R_rect)
plt.show()
ax.set_title("Right rectified image")

标定结果与问题 #

使用两台相同型号相机，同步拍摄10组棋盘格图像。单独标定两台相机时，得到的内参矩阵和畸变系数差异较大：

第一台相机内参矩阵 #

array([[553.76951265,   0.        , 359.52945636],
       [  0.        , 558.7809926 , 322.92191203],
       [  0.        ,   0.        ,   1.        ]])

第一台相机畸变系数 #

array([[-5.74144340e-01,  2.21330577e+00,  2.87945868e-03,
         9.47036694e-04, -3.28003833e+00]])

第二台相机内参矩阵 #

([[643.44291723,   0.        , 297.31281198],
       [  0.        , 639.47052736, 216.32413232],
       [  0.        ,   0.        ,   1.        ]])

第二台相机畸变系数 #

array([[ 0.16309873, -1.99119008, -0.02374205, -0.01189547, 10.39937883]])

此时校正后的图像效果极差，无法用于生成视差图。

尝试的修改 #

尝试合并所有棋盘格图像强制两台相机内参一致，修改代码如下：

_, mtx_left, dist_left, rvecs_left, tvecs_left, objpoints_left, imgpoints_left = CamIntr(chessboardL + chessboardR, [6,9], 0.025)


_, mtx_right, dist_right, rvecs_right, tvecs_right, objpoints_right, imgpoints_right = CamIntr(chessboardR, chessboardL, [6,9], 0.025)

校正结果虽略有改善，但两台相机的校正图像差异仍过大，无法生成视差图。

已附上待校正的原始图像、单独标定后的校正结果图、合并标定后的校正结果图，寻求技术帮助解决图像校正失败的问题。

内容的提问来源于stack exchange，提问作者Emma