如何从RGB-D全景图获取有效3D模型？投影畸变问题求助

尝试将RGB-D全景图投影回3D空间，但重投影结果存在畸变，3D场景失真，相关效果图与全景图如下：
畸变效果图
原始全景图

以下是实现代码：

import argparse
import os
import cv2
import numpy as np
import open3d

class PointCloudReader():

    def __init__(self, resolution="full", random_level=0, generate_color=False, generate_normal=False):
        self.random_level = random_level
        self.resolution = resolution
        self.generate_color = generate_color
        self.point_cloud = self.generate_point_cloud(self.random_level, color=self.generate_color)

    def generate_point_cloud(self, random_level=0, color=False, normal=False):
        coords = []
        colors = []

        # Load and resize depth image
        depth_image_path = 'DPT/output_monodepth/basel_stapfelberg_panorama.png'
        depth_img = cv2.imread(depth_image_path, cv2.IMREAD_ANYDEPTH)
        depth_img = cv2.resize(depth_img, (depth_img.shape[1] // 2, depth_img.shape[0] // 2))

        # Load and resize RGB image
        equirectangular_image = 'DPT/input/basel_stapfelberg_panorama.png'
        rgb_img = cv2.imread(equirectangular_image)
        rgb_img = cv2.resize(rgb_img, (depth_img.shape[1], depth_img.shape[0]))

        # Define parameters for conversion
        focal_length = depth_img.shape[1] / 2
        sensor_width = 36
        sensor_height = 24
        y_ticks = np.deg2rad(np.arange(0, 360, 360 / depth_img.shape[1]))
        x_ticks = np.deg2rad(np.arange(-90, 90, 180 / depth_img.shape[0]))

        # Compute spherical coordinates
        theta, phi = np.meshgrid(y_ticks, x_ticks)
        depth = depth_img + np.random.random(depth_img.shape) * random_level
        x_sphere = depth * np.cos(phi) * np.sin(theta)
        y_sphere = depth * np.sin(phi)
        z_sphere = depth * np.cos(phi) * np.cos(theta)

        # Convert spherical coordinates to camera coordinates
        x_cam = x_sphere.flatten()
        y_cam = -y_sphere.flatten()
        z_cam = z_sphere.flatten()

        coords = np.stack((x_cam, y_cam, z_cam), axis=-1)

        if color:
            colors = rgb_img.reshape(-1, 3) / 255.0

        points = {'coords': coords}

        if color:
            points['colors'] = colors

        return points

    def visualize(self):
        pcd = open3d.geometry.PointCloud()
        pcd.points = open3d.utility.Vector3dVector(self.point_cloud['coords'])
        if self.generate_color:
            pcd.colors = open3d.utility.Vector3dVector(self.point_cloud['colors'])
        open3d.visualization.draw_geometries([pcd])

def main(args):
    reader = PointCloudReader(random_level=10, generate_color=True, generate_normal=False)
    reader.visualize()

if __name__ == "__main__":
    main(None)

以下是修复畸变、生成有效3D点云的关键调整：

• 修正坐标转换与轴系对齐
  代码中对y_cam的无差别取反可能导致上下轴颠倒，需根据全景图的像素方向调整。如果全景图的行从上到下对应俯仰角从90°到-90°，则保留取反；否则移除取反，确保相机坐标系（右手系）与像素坐标一致：

  # 修正后的坐标转换（根据实际图像方向调整）
  x_cam = x_sphere.flatten()
  y_cam = y_sphere.flatten()  # 移除不必要的取反，或根据图像方向决定是否保留
  z_cam = z_sphere.flatten()
  

• 修复俯仰角采样范围
  当前x_ticks的生成方式未覆盖90°仰角，导致顶部区域采样缺失。改用linspace均匀覆盖-90°到90°的完整范围：

  x_ticks = np.deg2rad(np.linspace(-90, 90, depth_img.shape[0]))
  

  若全景图顶部对应-90°、底部对应90°，则反转顺序：np.linspace(90, -90, depth_img.shape[0])

• 移除随机噪声干扰
  代码中给深度值添加的随机噪声会直接造成点云毛刺和失真，应删除或仅在调试时启用：

  depth = depth_img  # 移除随机噪声，使用原始深度数据
  

• 确认深度图的物理单位
  检查深度图是否为归一化值，若为0-255的8位图像，需转换为实际物理深度（如米）：

  # 示例：假设深度范围为0-10米，归一化到0-255
  depth = depth_img / 255.0 * 10.0
  

• 对齐RGB与深度图的采样方式
  缩放图像时，深度图使用最近邻插值避免深度值被平滑，RGB图使用线性插值保证颜色过渡自然：

  depth_img = cv2.resize(depth_img, (depth_img.shape[1]//2, depth_img.shape[0]//2), interpolation=cv2.INTER_NEAREST)
  rgb_img = cv2.resize(rgb_img, (depth_img.shape[1], depth_img.shape[0]), interpolation=cv2.INTER_LINEAR)
  

• 修正球面坐标生成逻辑
  确保meshgrid生成的theta（方位角）和phi（俯仰角）与图像的行列完全对应，theta对应全景图的列（0-360°），phi对应行（-90°到90°），当前代码的meshgrid顺序正确，但需确认图像的行列方向是否匹配。

修正后的完整代码示例 #

import argparse
import os
import cv2
import numpy as np
import open3d

class PointCloudReader():

    def __init__(self, resolution="full", random_level=0, generate_color=False, generate_normal=False):
        self.random_level = random_level
        self.resolution = resolution
        self.generate_color = generate_color
        self.point_cloud = self.generate_point_cloud(self.random_level, color=self.generate_color)

    def generate_point_cloud(self, random_level=0, color=False, normal=False):
        coords = []
        colors = []

        # Load and resize depth image
        depth_image_path = 'DPT/output_monodepth/basel_stapfelberg_panorama.png'
        depth_img = cv2.imread(depth_image_path, cv2.IMREAD_ANYDEPTH)
        # 深度图用最近邻插值
        depth_img = cv2.resize(depth_img, (depth_img.shape[1] // 2, depth_img.shape[0] // 2), interpolation=cv2.INTER_NEAREST)

        # Load and resize RGB image
        equirectangular_image = 'DPT/input/basel_stapfelberg_panorama.png'
        rgb_img = cv2.imread(equirectangular_image)
        # RGB图用线性插值
        rgb_img = cv2.resize(rgb_img, (depth_img.shape[1], depth_img.shape[0]), interpolation=cv2.INTER_LINEAR)

        # Define parameters for conversion
        y_ticks = np.deg2rad(np.linspace(0, 360, depth_img.shape[1], endpoint=False))
        # 完整覆盖-90到90度俯仰角
        x_ticks = np.deg2rad(np.linspace(-90, 90, depth_img.shape[0]))

        # Compute spherical coordinates
        theta, phi = np.meshgrid(y_ticks, x_ticks)
        # 移除随机噪声，使用原始深度
        depth = depth_img
        # 转换为实际深度（根据你的深度图格式调整）
        # depth = depth_img / 255.0 * 10.0

        x_sphere = depth * np.cos(phi) * np.sin(theta)
        y_sphere = depth * np.sin(phi)
        z_sphere = depth * np.cos(phi) * np.cos(theta)

        # Convert spherical coordinates to camera coordinates（根据图像方向调整y轴）
        x_cam = x_sphere.flatten()
        y_cam = -y_sphere.flatten()  # 若图像上下颠倒则保留取反，否则删除负号
        z_cam = z_sphere.flatten()

        coords = np.stack((x_cam, y_cam, z_cam), axis=-1)

        if color:
            colors = rgb_img.reshape(-1, 3) / 255.0

        points = {'coords': coords}

        if color:
            points['colors'] = colors

        return points

    def visualize(self):
        pcd = open3d.geometry.PointCloud()
        pcd.points = open3d.utility.Vector3dVector(self.point_cloud['coords'])
        if self.generate_color:
            pcd.colors = open3d.utility.Vector3dVector(self.point_cloud['colors'])
        open3d.visualization.draw_geometries([pcd])

def main(args):
    reader = PointCloudReader(generate_color=True)
    reader.visualize()

if __name__ == "__main__":
    main(None)

内容的提问来源于stack exchange，提问作者Hannah Stark