倾斜相机时立体标定失败，三维重建出现曲率问题

我搭建了一套由两台1080p相机组成的立体视觉系统，尝试计算三维坐标。平行布置相机时结果精度正常，但相机倾斜旋转后精度丢失，三维重建图出现曲率。以下是我使用的代码：

import numpy as np
import cv2
import json
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import numpy as np


# Load JSON from file
with open("stereo_1080_z_2.json", "r") as f:
    calib_data = json.load(f)

# 1. Intrinsic Camera Matrices
K1 = np.array(calib_data["LeftCameraMatrix"], dtype=np.float64)
K2 = np.array(calib_data["RightCameraMatrix"], dtype=np.float64)

# 2. Distortion Coefficients (flatten to 1D arrays)
D1 = np.array(calib_data["LeftDistCoeffs"], dtype=np.float64).flatten()
D2 = np.array(calib_data["RightDistCoeffs"], dtype=np.float64).flatten()

# 3. Rotation Matrix
R = np.array(calib_data["RotationMatrix"], dtype=np.float64)

# 4. Translation Vector
t = np.array(calib_data["TranslationVector"], dtype=np.float64)

# 3. Projection matrices
P1 = K2 @ np.hstack((np.eye(3), np.zeros((3, 1))))    # Left camera: [I | 0]
P2 = K1 @ np.hstack((R, t))                           # Right camera: [R | t]

# 4. 2D Points from Left and Right Images

raw_data2 = """824.5896,678.58014
766.8919,681.0476
709.4761,683.5503
652.5051,686.06177
596.3213,688.4066
540.6041,690.7059
485.71304,692.977
431.00085,695.2359
821.6526,620.22534
763.9344,622.83386
706.6069,625.45044
649.6632,628.121
593.4257,630.69147
537.8756,633.37933
482.72275,635.97375
428.3594,638.2883
818.653,561.6016
761.0327,564.5353
703.5856,567.5059
646.78455,570.32477
590.42865,573.20764
534.9492,576.0714
479.97192,578.65967
425.41275,581.6496
815.5818,503.18857
757.8619,506.39215
700.558,509.42053
643.72906,512.47064
587.4173,515.57184
531.7483,518.6343
476.75543,521.7598
422.51413,524.6003
812.42694,444.53024
754.7694,447.83057
697.3678,451.2303
640.50305,454.55603
584.2679,457.84747
528.8557,461.2194
473.7063,464.53302
419.2249,467.8827


"""


raw_data1 = """1183.8037,599.787
1118.8151,601.8794
1053.872,604.07806
988.91534,606.05383
923.9008,608.01843
859.0262,610.3032
794.07983,612.30756
728.7919,614.3955
1181.8151,535.18256
1116.7349,537.2038
1051.8402,539.15515
986.89935,541.2557
921.9206,543.3922
856.9487,545.53503
791.7619,547.5786
726.94025,549.5983
1180.0876,470.3838
1114.8981,472.3482
1049.9664,474.43146
985.07214,476.45203
919.9059,478.42847
854.7946,480.6491
789.81506,482.7439
724.7983,484.81985
1178.1721,405.50168
1112.9385,407.45044
1048.0436,409.50235
982.94086,411.5525
917.8295,413.63214
852.71783,415.81735
787.67737,417.95038
722.6456,419.97787
1176.3236,340.35297
1111.2251,342.51514
1045.9926,344.61606
980.9125,346.48035
915.85345,348.6267
850.59906,350.7703
785.52454,353.04633
720.4761,355.10464

 """
 # Convert to NumPy array
left_points = np.array([
    list(map(float, line.strip().split(',')))
    for line in raw_data1.strip().splitlines()
])

right_points =np.array([
    list(map(float, line.strip().split(',')))
    for line in raw_data2.strip().splitlines()
])# Combined plot
plt.figure(figsize=(12, 8))

# Left points
plt.scatter(left_points[:, 0], left_points[:, 1], color='blue', label='Left Points')
for i, (x, y) in enumerate(left_points, start=1):
    label = f'L{i} ({x:.1f}, {y:.1f})'
    plt.text(x + 2, y, label, fontsize=8, color='blue')

# Right points
plt.scatter(right_points[:, 0], right_points[:, 1], color='green', label='Right Points')
for i, (x, y) in enumerate(right_points, start=1):
    label = f'R{i} ({x:.1f}, {y:.1f})'
    plt.text(x + 2, y, label, fontsize=8, color='green')

plt.title('Left and Right Image Points with Labels and Coordinates')
plt.xlabel('X')
plt.ylabel('Y')
plt.gca().invert_yaxis()  # Keep image-like Y axis
plt.grid(True)
plt.legend()
plt.axis('equal')
plt.tight_layout()
plt.show()

# 5. Prepare for triangulation
pts1 = left_points.T  # shape (2, N)
pts2 = right_points.T # shape (2, N)

# 6. Triangulate
points_4d_hom = cv2.triangulatePoints(P1, P2, pts1, pts2)
points_3d = points_4d_hom[:3] / points_4d_hom[3]

# 7. Output results
points_3d = points_3d.T  # shape (N, 3)
for i, p in enumerate(points_3d):
    print(f"Point {i+1}: X = {p[0]:.2f}, Y = {p[1]:.2f}, Z = {p[2]:.2f}")

points = points_3d

X, Y, Z = points[:, 0], points[:, 1], points[:, 2]

# Separate X, Y, Z
X = points[:, 0]
Y = points[:, 1]
Z = points[:, 2]
point_indices = np.arange(1, len(points) + 1)

# Plot
plt.figure(figsize=(12, 6))
plt.plot(point_indices, X, color='red', marker='o', label='X')
plt.plot(point_indices, Y, color='green', marker='^', label='Y')
plt.plot(point_indices, Z, color='blue', marker='s', label='Z')

plt.xlabel('Point Number')
plt.ylabel('Value')
plt.title('Point Index vs X, Y, Z')
plt.legend()
plt.grid(True)
plt.tight_layout()
plt.show()
# 3D Plot
fig = plt.figure(figsize=(10, 7))
ax = fig.add_subplot(111, projection='3d')

# Plot points
ax.scatter(X, Y, Z, c='blue', marker='o', label='Triangulated Points')
ax.plot(X, Y, Z, linestyle='--', color='gray', alpha=0.3)

# Labels and title
ax.set_title("Stereo Triangulated 3D Points")
ax.set_xlabel("X (mm)")
ax.set_ylabel("Y (mm)")
ax.set_zlabel("Z (mm)")
ax.legend()

# Aspect ratio adjustment
ax.set_box_aspect([np.ptp(X), np.ptp(Y), np.ptp(Z)])  # Equalize scale visually

plt.tight_layout()
plt.show()

我使用cv2进行三角化计算三维点，同时有深度计算公式：z= b*fx/(ul-ur)，其中b为基线长度，fx为焦距，ul为左图像y坐标，ur为右图像y坐标。

我尝试进行立体标定，但结果也不正确，实际硬件布置：两相机间距755mm，右相机比左相机高14cm，z方向偏移可忽略。

内容的提问来源于stack exchange，提问作者PARTHSINGH Rajput