Apple Vision Pro集成Core ML模型实现实体手机检测技术求助

问题分析 #

原代码存在以下核心问题：

• 权限检查语法错误，无法正确判断相机授权状态
• 缺少相机帧捕获逻辑，仅判断授权但未获取实际图像数据
• 单次按钮触发的逻辑不适合实时检测场景

完整解决方案代码 #

1. 重构图像识别处理器（支持视频帧直接处理） #

优化处理器逻辑，新增直接处理视频帧的方法，提升识别效率：

import Foundation
import CoreML
import Vision
import SwiftUI
import AVFoundation

class ImageRecognitionHandler: ObservableObject {
    @Published var recognizedObjects: [String] = []
    @Published var isDetectingPhone = false
    private var model: VNCoreMLModel

    init() {
        do {
            let configuration = MLModelConfiguration()
            let phoneRecognitionModel = try PhoneRecognition1(configuration: configuration)
            model = try VNCoreMLModel(for: phoneRecognitionModel.model)
        } catch {
            fatalError("加载CoreML模型失败: \(error.localizedDescription)")
        }
    }

    // 兼容原UIImage识别逻辑
    func recognizeImage(_ image: UIImage) {
        guard let cgImage = image.cgImage else { return }
        processCGImage(cgImage)
    }

    // 直接处理视频流帧（性能更优）
    func recognizeSampleBuffer(_ sampleBuffer: CMSampleBuffer) {
        guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
        let request = VNCoreMLRequest(model: model) { [weak self] request, error in
            guard let results = request.results as? [VNRecognizedObjectObservation], error == nil else {
                print("图像识别失败: \(error?.localizedDescription ?? "未知错误")")
                return
            }

            let recognizedObjectIdentifiers = results.compactMap { $0.labels.first?.identifier }
            let detectedPhone = recognizedObjectIdentifiers.contains("phone")
            DispatchQueue.main.async {
                self?.isDetectingPhone = detectedPhone
                self?.recognizedObjects = recognizedObjectIdentifiers
                print(detectedPhone ? "检测到手机" : "未检测到手机")
            }
        }

        let handler = VNImageRequestHandler(cmSampleBuffer: sampleBuffer, options: [:])
        do {
            try handler.perform([request])
        } catch {
            print("执行识别请求失败: \(error.localizedDescription)")
        }
    }

    // 通用CGImage处理方法
    private func processCGImage(_ cgImage: CGImage) {
        let request = VNCoreMLRequest(model: model) { [weak self] request, error in
            guard let results = request.results as? [VNRecognizedObjectObservation], error == nil else {
                print("图像识别失败: \(error?.localizedDescription ?? "未知错误")")
                return
            }

            let recognizedObjectIdentifiers = results.compactMap { $0.labels.first?.identifier }
            let detectedPhone = recognizedObjectIdentifiers.contains("phone")
            DispatchQueue.main.async {
                self?.isDetectingPhone = detectedPhone
                self?.recognizedObjects = recognizedObjectIdentifiers
                print(detectedPhone ? "检测到手机" : "未检测到手机")
            }
        }

        let handler = VNImageRequestHandler(cgImage: cgImage)
        do {
            try handler.perform([request])
        } catch {
            print("执行识别请求失败: \(error.localizedDescription)")
        }
    }
}

2. 重构ContentView，实现相机捕获逻辑 #

添加相机会话管理、权限处理和实时帧捕获逻辑：

import SwiftUI
import RealityKit
import AVFoundation

struct ContentView: View {
    @StateObject private var imageRecognitionHandler = ImageRecognitionHandler()
    @State private var captureSession = AVCaptureSession()
    @State private var isSessionRunning = false
    @State private var showPermissionAlert = false

    var body: some View {
        VStack(spacing: 20) {
            // 实时显示检测状态
            Text(imageRecognitionHandler.isDetectingPhone ? "✅ 已检测到手机" : "❌ 未检测到手机")
                .font(.title)
                .bold()

            // 控制检测的按钮
            Button(isSessionRunning ? "停止检测" : "开始检测") {
                toggleDetection()
            }
            .padding()
            .background(Color.blue)
            .foregroundColor(.white)
            .cornerRadius(10)
        }
        .alert("需要相机权限", isPresented: $showPermissionAlert) {
            Button("设置") {
                guard let settingsUrl = URL(string: UIApplication.openSettingsURLString) else { return }
                UIApplication.shared.open(settingsUrl)
            }
            Button("取消", role: .cancel) {}
        }
        .onAppear {
            checkCameraPermission()
        }
        .onDisappear {
            stopCaptureSession()
        }
    }

    // 检查相机权限
    private func checkCameraPermission() {
        switch AVCaptureDevice.authorizationStatus(for: .video) {
        case .authorized:
            setupCaptureSession()
        case .notDetermined:
            AVCaptureDevice.requestAccess(for: .video) { granted in
                DispatchQueue.main.async {
                    if granted {
                        setupCaptureSession()
                    } else {
                        showPermissionAlert = true
                    }
                }
            }
        case .denied, .restricted:
            showPermissionAlert = true
        @unknown default:
            showPermissionAlert = true
        }
    }

    // 配置相机捕获会话
    private func setupCaptureSession() {
        captureSession.beginConfiguration()

        // 选择Vision Pro前置广角摄像头
        guard let videoDevice = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .front) else {
            print("无法获取相机设备")
            return
        }

        do {
            let videoInput = try AVCaptureDeviceInput(device: videoDevice)
            if captureSession.canAddInput(videoInput) {
                captureSession.addInput(videoInput)
            }
        } catch {
            print("创建相机输入失败: \(error.localizedDescription)")
            return
        }

        // 设置视频帧输出
        let videoOutput = AVCaptureVideoDataOutput()
        videoOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "cameraQueue"))
        if captureSession.canAddOutput(videoOutput) {
            captureSession.addOutput(videoOutput)
        }

        captureSession.commitConfiguration()
    }

    // 切换检测状态
    private func toggleDetection() {
        isSessionRunning ? stopCaptureSession() : startCaptureSession()
    }

    private func startCaptureSession() {
        if !captureSession.isRunning {
            captureSession.startRunning()
            isSessionRunning = true
        }
    }

    private func stopCaptureSession() {
        if captureSession.isRunning {
            captureSession.stopRunning()
            isSessionRunning = false
        }
    }
}

// 实现视频帧代理，处理实时捕获的帧
extension ContentView: AVCaptureVideoDataOutputSampleBufferDelegate {
    func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
        imageRecognitionHandler.recognizeSampleBuffer(sampleBuffer)
    }
}

#Preview(windowStyle: .automatic) {
    ContentView()
}

3. 主程序保持不变 #

import SwiftUI
import Vision

@main
struct TestVisionTrackerApp: App {
    var body: some Scene {
        WindowGroup {
            ContentView()
        }
    }
}

关键修改说明 #

• 权限修复：替换原错误的权限判断逻辑，添加完整的权限请求和引导流程
• 相机捕获：使用AVCaptureSession管理相机输入输出，实现实时视频帧捕获
• 性能优化：新增直接处理视频帧的方法，避免UIImage转换的性能损耗
• 状态反馈：添加UI文本实时显示检测结果，提升用户体验

注意事项 #

1. 必须在Info.plist中添加相机权限描述：NSCameraUsageDescription，值设为"需要访问相机以检测实体手机"，否则应用会崩溃
2. 确保PhoneRecognition1.mlmodel已正确添加到项目，且模型输入输出与Vision框架兼容
3. 可根据需求切换摄像头位置（.front/.back），适配不同检测场景
4. 实时检测会消耗一定性能，可通过调整捕获分辨率或添加帧率控制优化

内容的提问来源于stack exchange，提问作者kartbouni