使用ChAruco板进行立体相机标定的内参不一致问题

为短焦距相机搭建立体系统，因无法同时捕获完整传统棋盘格，选用ChAruco板进行标定。但遇到内参不一致问题：两台同型号、基线平行（间距约10cm）的相机，立体联合标定得到的内参差异显著，但单独标定结果符合预期基准。更换图像加载顺序、测试不同规模数据集（10/20/100/400张图）后，问题仍存在。

设备搭建示意图：

标定代码 #

import numpy as np
import cv2 
import glob
from matplotlib import pyplot as plt
from cv2 import aruco
import os

################ FIND CHESSBOARD CORNERS - OBJECT POINTS AND IMAGE POINTS #############################

frameSize = (1440,1080)
aruco_dict = aruco.getPredefinedDictionary(aruco.DICT_5X5_1000)
charuco_board = aruco.CharucoBoard((20, 30), 17, 8.5, aruco_dict) 
charuco_detector = aruco.CharucoDetector(charuco_board)

# Arrays to store object points and image points from all the images.
objpoints = [] # 3d point in real world space
imgpointsL = [] # 2d points in image plane.
imgpointsR = [] # 2d points in image plane.


imagesLeft = sorted(glob.glob('images/StereoLeft/images_1/*.png'))
imagesRight = sorted(glob.glob('images/StereoRight/images_1/*.png'))

for imgLeft, imgRight in zip(imagesLeft, imagesRight):
    
    imgL = cv2.imread(imgLeft)
    imgR = cv2.imread(imgRight)
    grayL = cv2.cvtColor(imgL, cv2.COLOR_BGR2GRAY)
    grayR = cv2.cvtColor(imgR, cv2.COLOR_BGR2GRAY)

    cornersL, idsL, _ = aruco.detectMarkers(grayL, aruco_dict)
    cornersR, idsR, _ = aruco.detectMarkers(grayR, aruco_dict)

    aruco.drawDetectedMarkers(imgL, cornersL, idsL)
    aruco.drawDetectedMarkers(imgR, cornersR, idsR)

    # If found, add object points, image points (after refining them)
    if idsL is not None and idsR is not None:  # Ensure markers are detected in both images
        retL, charucoCornersL, charucoIdsL = aruco.interpolateCornersCharuco(cornersL, idsL, grayL, charuco_board)
        retR, charucoCornersR, charucoIdsR = aruco.interpolateCornersCharuco(cornersR, idsR, grayR, charuco_board)
        if retL > 0 and retR > 0:  # If ChArUco corners are interpolated

            if charucoIdsL is None:
                charucoIdsL = np.array([])
            if charucoIdsR is None:
                charucoIdsR = np.array([])

            commonIds = np.intersect1d(charucoIdsL, charucoIdsR, assume_unique=True)
            
            if commonIds.size > 0:
                # Filter corners with matching ids
                indicesL = np.isin(charucoIdsL, commonIds).flatten()
                indicesR = np.isin(charucoIdsR, commonIds).flatten()

                matchingCornersL = charucoCornersL[indicesL]
                matchingCornersR = charucoCornersR[indicesR]
                matchingIds = charucoIdsL[indicesL]  # charucoIdsR[indicesR] would be the same
                # print(charucoIdsL[indicesL].flatten(),charucoIdsR[indicesR].flatten())
                # Generate object points
                
                objp, imgPointsL = aruco.getBoardObjectAndImagePoints(charuco_board, matchingCornersL, matchingIds)
                _, imgPointsR = aruco.getBoardObjectAndImagePoints(charuco_board, matchingCornersR, matchingIds)
                
                min_points_required = 6
                if len(imgPointsL) >= min_points_required and len(imgPointsR) >= min_points_required:
                    objpoints.append(objp)
                    imgpointsL.append(imgPointsL)
                    imgpointsR.append(imgPointsR)

        cv2.imshow('img left', imgL)
        cv2.imshow('img right', imgR)
        cv2.waitKey(500)


cv2.destroyAllWindows()

############## CALIBRATION #######################################################

retL, cameraMatrixL, distL, rvecsL, tvecsL = cv2.calibrateCamera(objpoints, imgpointsL, frameSize, None, None)
heightL, widthL, channelsL = imgL.shape
newCameraMatrixL, roi_L = cv2.getOptimalNewCameraMatrix(cameraMatrixL, distL, (widthL, heightL), 1, (widthL, heightL))

retR, cameraMatrixR, distR, rvecsR, tvecsR = cv2.calibrateCamera(objpoints, imgpointsR, frameSize, None, None)
heightR, widthR, channelsR = imgR.shape
newCameraMatrixR, roi_R = cv2.getOptimalNewCameraMatrix(cameraMatrixR, distR, (widthR, heightR), 1, (widthR, heightR))

print(newCameraMatrixL)
print()
print(newCameraMatrixR)

directory_path = os.getcwd()

# Specify the file name
file_name = "intrinsics.txt"

# Combine the directory path and file name to form the full file path
file_path = os.path.join(directory_path, file_name)

# Writing to the file
with open(file_path, 'w') as file:
    # Writing Camera 1 matrix
    file.write('Matrix L\nCamera 1\n')
    for row in newCameraMatrixL:
        file.write(' '.join(map(str, row)) + '\n')
    file.write('\n')
    
    # Writing Camera 2 matrix
    file.write('Matrix R\nCamera 2\n')
    for row in newCameraMatrixR:
        file.write(' '.join(map(str, row)) + '\n')


########## Stereo Vision Calibration #############################################

flags = 0
flags |= cv2.CALIB_FIX_INTRINSIC
# Here we fix the intrinsic camara matrixes so that only Rot, Trns, Emat and Fmat are calculated.
# Hence intrinsic parameters are the same 

criteria_stereo = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)

# This step is performed to transformation between the two cameras and calculate Essential and Fundamental matrix
retStereo, newCameraMatrixL, distL, newCameraMatrixR, distR, rot, trans, essentialMatrix, fundamentalMatrix = cv2.stereoCalibrate(objpoints, imgpointsL, imgpointsR, newCameraMatrixL, distL, newCameraMatrixR, distR, grayL.shape[::-1], criteria_stereo, flags)


# Reprojection Error
mean_error = 0

for i in range(len(objpoints)):
    imgpoints2, _ = cv2.projectPoints(objpoints[i], rvecsL[i], tvecsL[i], newCameraMatrixL, distL)
    error = cv2.norm(imgpointsL[i], imgpoints2, cv2.NORM_L2)/len(imgpoints2)
    mean_error += error


print("Total error: {}".format(mean_error/len(objpoints)))


########## Stereo Rectification #################################################

rectifyScale= 1
rectL, rectR, projMatrixL, projMatrixR, Q, roi_L, roi_R= cv2.stereoRectify(newCameraMatrixL, distL, newCameraMatrixR, distR, grayL.shape[::-1], rot, trans, rectifyScale,(0,0))
print(Q)

stereoMapL = cv2.initUndistortRectifyMap(newCameraMatrixL, distL, rectL, projMatrixL, grayL.shape[::-1], cv2.CV_16SC2)
stereoMapR = cv2.initUndistortRectifyMap(newCameraMatrixR, distR, rectR, projMatrixR, grayR.shape[::-1], cv2.CV_16SC2)

print("Saving parameters!")
cv_file = cv2.FileStorage('stereoMap.xml', cv2.FILE_STORAGE_WRITE)

cv_file.write('stereoMapL_x',stereoMapL[0])
cv_file.write('stereoMapL_y',stereoMapL[1])
cv_file.write('stereoMapR_x',stereoMapR[0])
cv_file.write('stereoMapR_y',stereoMapR[1])
cv_file.write('q', Q)

cv_file.release()

标定结果 #

[[5.51660251e+03 0.00000000e+00 6.16936188e+02]
 [0.00000000e+00 1.42359236e+04 5.30500368e+02]
 [0.00000000e+00 0.00000000e+00 1.00000000e+00]]

[[2.22078760e+03 0.00000000e+00 7.22814921e+02]
 [0.00000000e+00 2.22879614e+03 5.06648447e+02]
 [0.00000000e+00 0.00000000e+00 1.00000000e+00]]

两台相机的内参矩阵差异显著，其中一个结果不符合单独标定的基准值。

内容的提问来源于stack exchange，提问作者CupidONO