OpenCV4.5.5鱼眼立体标定stereoCalibrate调用崩溃问题

问题背景 #

我近期一直在调试两台视场角210°的鱼眼相机构成的立体相机系统，测试开源鱼眼立体标定代码时，无论使用项目自带的示例图像，还是我自行采集的标定图像，代码都无法正常运行。

错误表现 #

无论怎么调整配置，程序总会在调用cv::fisheye::stereoCalibrate时崩溃，抛出错误如下：

OpenCV(4.5.5) Error: Assertion failed (abs_max < threshold) in cv::fisheye::stereoCalibrate, file ...\opencv-4.5.5\modules\calib3d\src\fisheye.cpp, line 1051

在fisheye.cpp第1051行加断点调试发现：优化迭代的前2轮循环中abs_max < threshold判定条件成立，第3轮循环后该条件不再满足，随即触发断言导致程序崩溃，暂时没找到具体诱因。

已尝试的排查方案 #

• 在多台PC上使用不同编译配置的OpenCV 4.5.5环境复现测试，问题稳定出现
• 尝试导入MATLAB生成的立体标定参数作为初始值
• 分别调用鱼眼模型接口cv::fisheye::stereoCalibrate和普通针孔相机接口cv::stereoCalibrate测试，两种调用方式均会触发崩溃

完整实现代码 #

#include <opencv2/core/core.hpp>
#include <opencv2/calib3d/calib3d.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <stdio.h>
#include <filesystem>
#include <string>
#include <vector>
#include <string>
#include <algorithm>
#include <iostream>
#include <iterator>
#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>

using namespace std;
using namespace cv;


static int print_help(char** argv)
{
    cout <<
        " Given a list of chessboard images, the number of corners (nx, ny)\n"
        " on the chessboards, and a flag: useCalibrated for \n"
        "   calibrated (0) or\n"
        "   uncalibrated \n"
        "     (1: use stereoCalibrate(), 2: compute fundamental\n"
        "         matrix separately) stereo. \n"
        " Calibrate the cameras and display the\n"
        " rectified results along with the computed disparity images.   \n" << endl;
    cout << "Usage:\n " << argv[0] << " -w=<board_width default=9> -h=<board_height default=6> -s=<square_size default=1.0> <image list XML/YML file default=stereo_calib.xml>\n" << endl;
    return 0;
}




static void StereoCalib(const vector<string>& imagelist, Size boardSize, float squareSize, bool displayCorners = false, bool useCalibrated = true, bool showRectified = true)
{
    if (imagelist.size() % 2 != 0)
    {
        cout << "Error: the image list contains odd (non-even) number of elements\n";
        return;
    }

    const int maxScale = 2;
    // ARRAY AND VECTOR STORAGE:

    vector<vector<Vec2f> > imagePoints[2];
    vector<vector<Vec3f> > objectPoints;
    Size imageSize;

    int i, j, k, nimages = (int)imagelist.size() / 2;

    imagePoints[0].resize(nimages);
    imagePoints[1].resize(nimages);
    vector<string> goodImageList;

    for (i = j = 0; i < nimages; i++)
    {
        for (k = 0; k < 2; k++)
        {
            const string& filename = imagelist[i * 2 + k];
            Mat img = imread(filename, 0);
            if (img.empty())
                break;
            if (imageSize == Size())
                imageSize = img.size();
            else if (img.size() != imageSize)
            {
                cout << "The image " << filename << " has the size different from the first image size. Skipping the pair\n";
                break;
            }
            bool found = false;
            vector<Vec2f>& corners = imagePoints[k][j];
            for (int scale = 1; scale <= maxScale; scale++)
            {
                Mat timg;
                if (scale == 1)
                    timg = img;
                    
                else
                    resize(img, timg, Size(), scale, scale, INTER_LINEAR_EXACT);
                    
                found = findChessboardCorners(timg, boardSize, corners,
                    CALIB_CB_ADAPTIVE_THRESH | CALIB_CB_NORMALIZE_IMAGE);
                if (found)
                {
                    if (scale > 1)
                    {
                        Mat cornersMat(corners);
                        cornersMat *= 1. / scale;
                    }
                    break;
                }
            }
            if (displayCorners)
            {
                cout << filename << endl;
                Mat cimg, cimg1;
                cvtColor(img, cimg, COLOR_GRAY2BGR);
                drawChessboardCorners(cimg, boardSize, corners, found);
                double sf = 640. / MAX(img.rows, img.cols);
                resize(cimg, cimg1, Size(), sf, sf, INTER_LINEAR_EXACT);
                imshow("corners", cimg1);
                char c = (char)waitKey(500);
                if (c == 27 || c == 'q' || c == 'Q') //Allow ESC to quit
                    exit(-1);
            }
            else
                putchar('.');
            if (!found)
                break;
            cornerSubPix(img, corners, Size(11, 11), Size(-1, -1),
                TermCriteria(TermCriteria::COUNT + TermCriteria::EPS,
                    30, 0.01));
        }
        if (k == 2)
        {
            goodImageList.push_back(imagelist[i * 2]);
            goodImageList.push_back(imagelist[i * 2 + 1]);
            j++;
        }
    }
    cout << j << " pairs have been successfully detected.\n";
    nimages = j;
    if (nimages < 2)
    {
        cout << "Error: too little pairs to run the calibration\n";
        return;
    }

    imagePoints[0].resize(nimages);
    imagePoints[1].resize(nimages);
    objectPoints.resize(nimages);

    for (i = 0; i < nimages; i++)
    {
        for (j = 0; j < boardSize.height; j++)
            for (k = 0; k < boardSize.width; k++)
                objectPoints[i].push_back(Point3f(k * squareSize, j * squareSize, 0));
    }



    /*TEST mit richtigem Format an Vector Punkte*/
    cv::FileStorage fs("omni_calib_data.xml", cv::FileStorage::READ);
    std::vector<cv::Mat> objectPoints1;
    std::vector<cv::Mat>    imagePoints1;
    cv::Size imgSize;
    fs["objectPoints"] >> objectPoints1;
    fs["imagePoints"] >> imagePoints1;
    fs["imageSize"] >> imgSize;






    cout << "Running stereo calibration ...\n";
    Mat testL = imread("C:/Users/Moritz/Documents/Flexybot/Code/fisheyeStereoCalibration/imgs/left2.jpg"),
        testR = imread("C:/Users/Moritz/Documents/Flexybot/Code/fisheyeStereoCalibration/imgs/right2.jpg");

    printf("Starting Calibration\n");
    cv::Matx33d K1, K2, R;
    cv::Vec3d T;
    cv::Vec4d D1, D2;
    
    int flag = 0;
    flag |= cv::fisheye::CALIB_RECOMPUTE_EXTRINSIC;
    flag |= cv::fisheye::CALIB_CHECK_COND;
    flag |= cv::fisheye::CALIB_FIX_SKEW;
    //flag |= cv::fisheye::CALIB_FIX_K2;
    //flag |= cv::fisheye::CALIB_FIX_K3;
    //flag |= cv::fisheye::CALIB_FIX_K4;

    
    
    cv::fisheye::stereoCalibrate(objectPoints, imagePoints[0], imagePoints[1],
        K1, D1, K2, D2, testR.size(), R, T,0,
    cv::TermCriteria(TermCriteria::COUNT + TermCriteria::EPS, 100, DBL_EPSILON));

    cv::FileStorage fs1("stereo_params.xml", cv::FileStorage::WRITE);
    fs1 << "K1" << Mat(K1);
    fs1 << "K2" << Mat(K2);
    fs1 << "D1" << D1;
    fs1 << "D2" << D2;
    fs1 << "R" << Mat(R);
    fs1 << "T" << T;
    printf("Done Calibration\n");

    printf("Starting Rectification\n");

    cv::Mat R1, R2, P1, P2, Q;
    cv::fisheye::stereoRectify(K1, D1, K2, D2, testR.size(), R, T, R1, R2, P1, P2,
        Q, CALIB_ZERO_DISPARITY, testR.size(), 0.0, 1.1);

    fs1 << "R1" << R1;
    fs1 << "R2" << R2;
    fs1 << "P1" << P1;
    fs1 << "P2" << P2;
    fs1 << "Q" << Q;

    printf("Done Rectification\n");


 
    Mat rL, rR, tL, tR, kL, dL, kR, dR,pL,pR;

    fisheye::calibrate(objectPoints, imagePoints[0], testL.size(), kL, dL, rL, tL, fisheye::CALIB_ZERO_DISPARITY);
    fisheye::calibrate(objectPoints, imagePoints[1], testR.size(), kR, dR, rR, tR, fisheye::CALIB_ZERO_DISPARITY);
    fisheye::estimateNewCameraMatrixForUndistortRectify(kL, dL, testL.size(), Matx33d::eye(), pL, 1);
    fisheye::estimateNewCameraMatrixForUndistortRectify(kR, dR, testR.size(), Matx33d::eye(), pR, 1);
    cv::fisheye::undistortImage(testL, testL, kL, dL,pL);
    cv::fisheye::undistortImage(testR, testR, kR, dR,pR);

    imshow("Links", testL);
    imshow("Rechts", testR);
    waitKey(0);
    return;

}

static bool readStringList(const string& filename, vector<string>& l)
{
    l.resize(0);
    FileStorage fs(filename, FileStorage::READ);
    if (!fs.isOpened())
        return false;
    FileNode n = fs.getFirstTopLevelNode();
    if (n.type() != FileNode::SEQ)
        return false;
    FileNodeIterator it = n.begin(), it_end = n.end();
    for (; it != it_end; ++it)
        l.push_back((string)*it);
    return true;
}

int main(int argc, char** argv)
{
    Size boardSize;
    string imagelistfn;
    bool showRectified;
    cv::CommandLineParser parser(argc, argv, "{w|9|}{h|6|}{s|20.0|}{nr||}{help||}{@input|stereo_calib.xml|}");
    if (parser.has("help"))
        return print_help(argv);
    showRectified = !parser.has("nr");
    imagelistfn = samples::findFile(parser.get<string>("@input"));
    boardSize.width = parser.get<int>("w");
    boardSize.height = parser.get<int>("h");
    float squareSize = parser.get<float>("s");
    if (!parser.check())
    {
        parser.printErrors();
        return 1;
    }
    vector<string> imagelist;
    bool ok = readStringList(imagelistfn, imagelist);
    if (!ok || imagelist.empty())
    {
        cout << "can not open " << imagelistfn << " or the string list is empty" << endl;
        return print_help(argv);
    }

    StereoCalib(imagelist, boardSize, squareSize, true, true, showRectified);
    return 0;
}

内容的提问来源于stack exchange，提问作者embarto