TrueDepth相机像素距离测量精度异常问题求助

使用前置TrueDepth相机结合Vision框架识别图像中的点并进行3D距离测量。已知Vision输出为归一化坐标，已将其转换为对应View的CGPoint，再通过dataOutputSynchronizer同步的depthData获取Z值，尝试用相机内参计算空间两点距离。

目前已成功识别点并转换为屏幕CGPoint，但测试时（手部基本与相机共面、仅小幅移动），转换后的CGPoint变化不大，深度值却波动极大：点2的深度值接近实际距离（手部距相机约1英尺）但不准确且波动明显，点1深度值异常偏高（3.6+，远大于实际）。

控制台测试输出：

there are 2 points found
recognized points
[(499.08930909633636, 634.0807711283367), (543.7462849617004, 1061.8824380238852)]
DEPTH POINT 1 =  3.6312041
DEPTH POINT 2 =  0.2998223

there are 2 points found
recognized points
[(498.33644700050354, 681.3769372304281), (602.3667773008347, 1130.4955183664956)]
DEPTH POINT 1 =  3.6276162
DEPTH POINT 2 =  0.560331

同步数据处理代码（dataOutputSynchronizer） #

func dataOutputSynchronizer(_ synchronizer: AVCaptureDataOutputSynchronizer,
                            didOutput synchronizedDataCollection: AVCaptureSynchronizedDataCollection) {
        
        var handPoints: [CGPoint] = []
        
        // Read all outputs
        guard renderingEnabled,
            let syncedDepthData: AVCaptureSynchronizedDepthData =
            synchronizedDataCollection.synchronizedData(for: depthDataOutput) as? AVCaptureSynchronizedDepthData,
            let syncedVideoData: AVCaptureSynchronizedSampleBufferData =
            synchronizedDataCollection.synchronizedData(for: videoDataOutput) as? AVCaptureSynchronizedSampleBufferData else {
                // only work on synced pairs
                return
        }
        
        if syncedDepthData.depthDataWasDropped || syncedVideoData.sampleBufferWasDropped {
            return
        }
        
        let depthPixelBuffer = syncedDepthData.depthData.depthDataMap
        guard let videoPixelBuffer = CMSampleBufferGetImageBuffer(syncedVideoData.sampleBuffer) else {
            return
        }
        
        // Get the cameraIntrinsics
        guard let  cameraIntrinsics = syncedDepthData.depthData.cameraCalibrationData?.intrinsicMatrix else {
            return
        }
        
        let image = CIImage(cvPixelBuffer: videoPixelBuffer)
        
        let handler = VNImageRequestHandler(
           cmSampleBuffer: syncedVideoData.sampleBuffer,
           orientation: .up,
           options: [:]
         )
        
         do {
           try handler.perform([handPoseRequest])
           guard
             let results = handPoseRequest.results?.prefix(2),
             !results.isEmpty
           else {
             return
           }

            var recognizedPoints: [VNRecognizedPoint] = []

             try results.forEach { observation in
               let fingers = try observation.recognizedPoints(.all)

               if let middleTipPoint = fingers[.middleDIP] {
                 recognizedPoints.append(middleTipPoint)
               }

               if let wristPoint = fingers[.wrist] {
                 recognizedPoints.append(wristPoint)
               }
             }

             // Store the Points in handPoints if they are confident points
             handPoints = recognizedPoints.filter {
               $0.confidence > 0.90
             }
             .map {
               // Adjust the Y
               CGPoint(x: $0.location.x, y: 1 - $0.location.y)
             }
             
             // Process the Points Found
             DispatchQueue.main.sync {
              self.processPoints(handPoints,depthPixelBuffer,videoPixelBuffer,cameraIntrinsics)
             }
         } catch {
             // Be more graceful here 
         }
    }

点处理代码（Process Points） #

func processPoints(_ handPoints: [CGPoint],_ depthPixelBuffer: CVImageBuffer,_ videoPixelBuffer: CVImageBuffer,_ cameraIntrinsics: simd_float3x3) {

        // This converts the normalized point to screen points
        // cameraView.previewLayer is a AVCaptureVideoPreviewLayer inside a UIView
        let convertedPoints = handPoints.map {
            cameraView.previewLayer.layerPointConverted(fromCaptureDevicePoint: $0)
        }
       
        // We need 2 hand points to get the distance 
        if handPoints.count == 2 {
            print("there are 2 points found");
            print("recognized points")
            print(convertedPoints)
            
            let handVisionPoint1 = convertedPoints[0]
        
            let handVisionPoint2 = convertedPoints[1]
            
            let scaleFactor = CGFloat(CVPixelBufferGetWidth(depthPixelBuffer)) / CGFloat(CVPixelBufferGetWidth(videoPixelBuffer))
            
            CVPixelBufferLockBaseAddress(depthPixelBuffer, .readOnly)
            let floatBuffer = unsafeBitCast(CVPixelBufferGetBaseAddress(depthPixelBuffer), to: UnsafeMutablePointer<Float32>.self)
            
            let width = CVPixelBufferGetWidth(depthPixelBuffer)
            let height = CVPixelBufferGetHeight(depthPixelBuffer)
            
            let handVisionPixelX = Int((handVisionPoint1.x * scaleFactor).rounded())
            let handVisionPixelY = Int((handVisionPoint1.y * scaleFactor).rounded())
            
            let handVisionPixe2X = Int((handVisionPoint2.x * scaleFactor).rounded())
            let handVisionPixe2Y = Int((handVisionPoint2.y * scaleFactor).rounded())
            
            CVPixelBufferLockBaseAddress(depthPixelBuffer, .readOnly)
            
            let rowDataPoint1 = CVPixelBufferGetBaseAddress(depthPixelBuffer)! + handVisionPixelY * CVPixelBufferGetBytesPerRow(depthPixelBuffer)
            let handVisionPoint1Depth = rowDataPoint1.assumingMemoryBound(to: Float32.self)[handVisionPixelX]
            
            print("DEPTH POINT 1 = ", handVisionPoint1Depth)
            
            let rowDataPoint2 = CVPixelBufferGetBaseAddress(depthPixelBuffer)! + handVisionPixe2Y * CVPixelBufferGetBytesPerRow(depthPixelBuffer)
            let handVisionPoint2Depth = rowDataPoint2.assumingMemoryBound(to: Float32.self)[handVisionPixe2X]
            
            print("DEPTH POINT 2 = ", handVisionPoint2Depth)
            //Int((width - touchPoint.x) * (height - touchPoint.y))
}

1. 深度图像素匹配逻辑错误 #

坐标转换方向错误 #

当前将屏幕CGPoint直接乘以缩放系数得到深度图像素坐标，但layerPointConverted(fromCaptureDevicePoint:)是把设备归一化坐标转成了预览层的屏幕坐标，这个坐标的Y轴方向和深度图的Y轴方向可能不一致，且预览层可能存在拉伸/缩放，直接用屏幕点计算会导致坐标偏移。

正确做法：

• 跳过屏幕转换，直接用Vision输出的归一化坐标（调整Y轴后）转换为深度图的像素坐标：

  // 用原始的handPoints（归一化坐标）直接计算深度图像素
  let depthWidth = CGFloat(CVPixelBufferGetWidth(depthPixelBuffer))
  let depthHeight = CGFloat(CVPixelBufferGetHeight(depthPixelBuffer))
  let point1DepthX = Int((handPoints[0].x * depthWidth).rounded())
  let point1DepthY = Int((handPoints[0].y * depthHeight).rounded())
  

• Vision的归一化坐标和深度图的坐标体系基于相机输出帧尺寸对齐，跳过屏幕转换可避免预览层拉伸干扰。

重复锁定像素Buffer #

代码中连续调用两次CVPixelBufferLockBaseAddress(depthPixelBuffer, .readOnly)，会导致锁定计数叠加，可能引发内存访问异常，应只调用一次，处理完后解锁：

CVPixelBufferLockBaseAddress(depthPixelBuffer, .readOnly)
defer {
    CVPixelBufferUnlockBaseAddress(depthPixelBuffer, .readOnly)
}
// 后续深度值读取操作

2. 数据流同步问题 #

虽然使用了AVCaptureDataOutputSynchronizer，仍需确认：

• 视频输出和深度输出的分辨率格式是否匹配，比如视频用1080p、深度用低分辨率格式，会导致缩放系数计算错误；
• 检查syncedDepthData.isDepthDataSyncedToVideoData是否为true，确保深度数据和视频帧严格同步；
• 避免在DispatchQueue.main.sync中处理耗时操作，同步调用可能导致数据堆积，改为async：

  DispatchQueue.main.async {
      self.processPoints(handPoints, depthPixelBuffer, videoPixelBuffer, cameraIntrinsics)
  }
  

3. 深度值有效性判断 #

TrueDepth的深度图中，无效深度值会返回极大值（如你看到的3.6+），说明点1对应的深度像素无效，需要过滤：

let depthValue = rowDataPoint1.assumingMemoryBound(to: Float32.self)[handVisionPixelX]
// 过滤无效深度（TrueDepth有效深度范围通常在0.2m到2m之间）
guard depthValue > 0.1 && depthValue < 2.5 else {
    print("Invalid depth for point 1")
    return
}

内容的提问来源于stack exchange，提问作者K_C