iOS SceneKit中如何获取面部3D Landmarks的正确世界坐标？

我正尝试在SceneKit中将面部Landmarks转换为3D点。即使深度值不正确，面部仍能正常跟踪，但无法作为遮挡物使用——因为世界坐标错误，面部网格始终处于所有场景对象的前方或后方。我曾尝试使用MediaPipe的faceNode.transform = matrix，该方法跟踪效果出色，但无法更新Landmarks。目前我通过这些Landmarks更新几何顶点，并使用MediaPipe提供的uv.json数据更新几何UV坐标。我尝试过将MediaPipe的Z值转换为SceneKit的Z值，但未成功。请问如何在SceneKit中获取Landmarks的正确世界坐标？

Landmarks转换代码 #

if let normalizedLandmarkArray = firstFaceNormalizedLandmarks {
    
    let vertexPositionArray = normalizedLandmarkArray.enumerated().map { index, landmark -> simd_float3 in

        let screenX = CGFloat(landmark.x) * imageWidth
        let screenY = CGFloat(landmark.y) * imageHeight
        
        let screenPoint = SCNVector3(screenX, screenY, 1)
        
        let worldPoint = sceneView.unprojectPoint(screenPoint)

        return simd_float3(Float(worldPoint.x), Float(worldPoint.y), Float(worldPoint.z))
        }

几何更新代码 #

struct MyVertex {
    var x: Float  // Position X
    var y: Float  // Position Y
    var z: Float  // Position Z
    
    var nx: Float // Normal X
    var ny: Float // Normal Y
    var nz: Float // Normal Z
    
    var s: Float  // Texture Coordinate S (U)
    var t: Float  // Texture Coordinate T (V)
}


func updateFaceNodeGeometry(landmarks: [simd_float3], geometry: SCNGeometry) -> SCNGeometry? {
    guard !uValues.isEmpty, !vValues.isEmpty else {
        print("❌ UV values are not loaded.")
        return nil
    }
    
    let limitedLandmarks = landmarks.prefix(468)
    
    var vertices: [MyVertex] = []
    
    for (index, landmark) in limitedLandmarks.enumerated() {
        let vertex = MyVertex(
            x: landmark.x,
            y: landmark.y,
            z: landmark.z,
            nx: 0.0, ny: 0.0, nz: 1.0, // 默认设置为表面法线
            s: uValues[index],
            t: vValues[index]
        )
        vertices.append(vertex)
    }
    
    let data = NSData(bytes: vertices, length: vertices.count * MemoryLayout<MyVertex>.size)

    
    let vertexSource = SCNGeometrySource(
        data: data as Data,
        semantic: .vertex,
        vectorCount: vertices.count,
        usesFloatComponents: true,
        componentsPerVector: 3, // x, y, z
        bytesPerComponent: MemoryLayout<Float>.size,
        dataOffset: 0,
        dataStride: MemoryLayout<MyVertex>.size
    )
    
    let normalSource = SCNGeometrySource(
        data: data as Data,
        semantic: .normal,
        vectorCount: vertices.count,
        usesFloatComponents: true,
        componentsPerVector: 3, // nx, ny, nz
        bytesPerComponent: MemoryLayout<Float>.size,
        dataOffset: MemoryLayout.offset(of: \MyVertex.nx)!,
        dataStride: MemoryLayout<MyVertex>.size
    )
    
    let texcoordSource = SCNGeometrySource(
        data: data as Data,
        semantic: .texcoord,
        vectorCount: vertices.count,
        usesFloatComponents: true,
        componentsPerVector: 2, // s, t
        bytesPerComponent: MemoryLayout<Float>.size,
        dataOffset: MemoryLayout.offset(of: \MyVertex.s)!,
        dataStride: MemoryLayout<MyVertex>.size
    )
    
    let updatedElement = geometry.elements.first ?? SCNGeometryElement(
        data: Data(),
        primitiveType: .triangles,
        primitiveCount: limitedLandmarks.count / 3,
        bytesPerIndex: MemoryLayout<UInt16>.size
    )
    
    guard let oldVertexSource = geometry.sources(for: .vertex).first else {
        print("❌ 几何中未找到顶点源!")
        return nil
    }
    // ✅ 创建新几何
    
    let updatedGeometry = SCNGeometry(
        sources: [vertexSource, normalSource, texcoordSource],
        elements: [updatedElement]
    )
    
    if let material = updatedGeometry.firstMaterial {
        material.isDoubleSided = true
        material.writesToDepthBuffer = true
        material.readsFromDepthBuffer = true
        material.transparency = 1.0
        material.transparencyMode = .dualLayer
        material.fillMode = .fill
//            material.diffuse.contents = UIColor.green
    }
    
    
//        DispatchQueue.main.async {
//            self.faceNode?.geometry = updatedGeometry
//            self.sceneView.scene?.isPaused = true
//            self.sceneView.scene?.isPaused = false
//            self.sceneView.setNeedsDisplay()
//        }
//        
    
//        print("✅ 使用C结构体样式成功更新几何!")
    return updatedGeometry
}

• 使用MediaPipe的3D Landmarks而非2D：确保获取NormalizedLandmark3D类型标记点，它包含真实Z轴深度值（相对于面部中心的米级深度），不要仅用2D坐标反推3D位置。
• 结合面部变换矩阵转换坐标：MediaPipe提供的faceTransform矩阵可将Landmarks的局部空间坐标转换为SceneKit世界空间坐标，比unprojectPoint更准确——后者依赖屏幕坐标，无法反映真实3D深度。
• 修正坐标转换逻辑：将每个3D Landmark坐标通过变换矩阵转换后，再应用到几何顶点上。

修改后的Landmarks转换代码 #

// 确保获取3D Landmarks和面部变换矩阵
if let face3DLandmarks = firstFace3dLandmarks, let faceTransform = firstFaceTransform {
    // 将MediaPipe变换矩阵转为simd格式
    let simdTransform = simd_float4x4(faceTransform)
    
    let vertexPositionArray = face3DLandmarks.enumerated().map { index, landmark -> simd_float3 in
        // MediaPipe的Landmark是归一化坐标，需缩放为真实尺寸（示例用0.15米对应面部宽度，可按需调整）
        let scaleFactor: Float = 0.15
        let localPoint = simd_float3(landmark.x, landmark.y, landmark.z) * scaleFactor
        
        // 将局部点转换为世界空间坐标
        let worldPoint = simdTransform * simd_float4(localPoint, 1.0)
        
        // 适配SceneKit坐标系统（翻转Y轴，因MediaPipe与SceneKit的Y轴方向相反）
        return simd_float3(worldPoint.x, -worldPoint.y, worldPoint.z)
    }
    
    // 用转换后的坐标更新面部几何
    if let currentGeometry = faceNode.geometry, 
       let updatedGeometry = updateFaceNodeGeometry(landmarks: vertexPositionArray, geometry: currentGeometry) {
        faceNode.geometry = updatedGeometry
    }
}

• 确保深度缓冲生效：除了材质的writesToDepthBuffer和readsFromDepthBuffer设置，还要开启SceneView的深度测试：sceneView.isDepthTestingEnabled = true，否则面部网格无法正确参与遮挡计算。
• 调整缩放因子：根据实际场景需求调整scaleFactor，确保面部网格在SceneKit中的尺寸与真实世界匹配。

内容的提问来源于stack exchange，提问作者Atila Güler