iOS原生集成OpenCV时人脸检测功能异常问题求助

正在iOS原生应用中集成OpenCV，项目可正常构建，但无法正确检测人脸并绘制bounding box。克隆代码后需自行添加OpenCV iOS框架（体积过大未上传）。

Objective-C++ 代码 #

#import <opencv2/opencv.hpp>
#import <opencv2/imgcodecs/ios.h>
#import "OpenCVWrapper.h"

@interface UIImage (OpenCVWrapper)
- (void)convertToMat: (cv::Mat *)pMat: (bool)alphaExists;
@end

@implementation UIImage (OpenCVWrapper)

- (void)convertToMat: (cv::Mat *)pMat: (bool)alphaExists {
    UIImageOrientation orientation = self.imageOrientation;
    cv::Mat mat;
    UIImageToMat(self, mat, alphaExists);
    
    switch (orientation) {
        case UIImageOrientationRight:
            cv::rotate(mat, *pMat, cv::ROTATE_90_CLOCKWISE);
            break;
        case UIImageOrientationLeft:
            cv::rotate(mat, *pMat, cv::ROTATE_90_COUNTERCLOCKWISE);
            break;
        case UIImageOrientationDown:
            cv::rotate(mat, *pMat, cv::ROTATE_180);
            break;
        case UIImageOrientationUp:
        default:
            *pMat = mat;
            break;
    }
}
@end

@implementation OpenCVWrapper

static cv::CascadeClassifier faceCascade;

+ (void)initialize {
    if (self == [OpenCVWrapper self]) {
        NSString *faceCascadePath = [[NSBundle mainBundle] pathForResource:@"haarcascade_frontalface_default" ofType:@"xml"];
        if (!faceCascade.load([faceCascadePath UTF8String])) {
            NSLog(@"Error loading face detection model");
        }
    }
}

+ (NSArray<NSValue *> *)detectFaceRectsInUIImage:(UIImage *)image {
    // Convert UIImage to cv::Mat
    cv::Mat mat;
    [image convertToMat:&mat :false];

    // Convert the image to grayscale
    cv::Mat gray;
    cv::cvtColor(mat, gray, cv::COLOR_BGR2GRAY);
    cv::equalizeHist(gray, gray);

    // Detect faces
    std::vector<cv::Rect> faces;
    faceCascade.detectMultiScale(gray, faces, 1.1, 2, 0 | cv::CASCADE_SCALE_IMAGE, cv::Size(30, 30));

    // Convert cv::Rect to CGRect and wrap in NSValue
    NSMutableArray<NSValue *> *faceRects = [NSMutableArray arrayWithCapacity:faces.size()];
    for (const auto &face : faces) {
        CGRect faceRect = CGRectMake(face.x, face.y, face.width, face.height);
        [faceRects addObject:[NSValue valueWithCGRect:faceRect]];
    }

    return [faceRects copy];
}

@end

Swift 代码 #

import UIKit
import AVFoundation
import VideoToolbox

class ViewController: UIViewController,AVCaptureVideoDataOutputSampleBufferDelegate {
    
    var previewView : UIView!
    var boxView:UIView!
    
    //Camera Capture requiered properties
    var videoDataOutput: AVCaptureVideoDataOutput!
    var videoDataOutputQueue: DispatchQueue!
    var previewLayer:AVCaptureVideoPreviewLayer!
    var captureDevice : AVCaptureDevice!
    let session = AVCaptureSession()
    private var faceOverlayView: FaceOverlayView!
    
    override func viewDidLoad() {
        super.viewDidLoad()
            previewView = UIView(frame: CGRect(x: 0,
                                               y: 0,
                                               width: UIScreen.main.bounds.size.width,
                                               height: UIScreen.main.bounds.size.height))
            previewView.contentMode = UIView.ContentMode.scaleAspectFit
            view.addSubview(previewView)
            
            boxView = UIView(frame: self.view.frame)
            view.addSubview(boxView)
            
            // Initialize face overlay view
            faceOverlayView = FaceOverlayView(frame: view.bounds)
            view.addSubview(faceOverlayView)
            
            
            setupAVCapture()
    }
    
    override var shouldAutorotate: Bool {
        if (UIDevice.current.orientation == UIDeviceOrientation.landscapeLeft ||
            UIDevice.current.orientation == UIDeviceOrientation.landscapeRight ||
            UIDevice.current.orientation == UIDeviceOrientation.unknown) {
            return false
        }
        else {
            return true
        }
    }
    
    func setupAVCapture(){
        session.sessionPreset = AVCaptureSession.Preset.vga640x480
        guard let device = AVCaptureDevice
            .default(AVCaptureDevice.DeviceType.builtInWideAngleCamera,
                     for: .video,
                     position: AVCaptureDevice.Position.back) else {
            return
        }
        captureDevice = device
        beginSession()
    }
    
    func beginSession(){
        var deviceInput: AVCaptureDeviceInput!
        
        do {
            deviceInput = try AVCaptureDeviceInput(device: captureDevice)
            guard deviceInput != nil else {
                print("error: cant get deviceInput")
                return
            }
            
            if self.session.canAddInput(deviceInput){
                self.session.addInput(deviceInput)
            }
            
            videoDataOutput = AVCaptureVideoDataOutput()
            videoDataOutput.alwaysDiscardsLateVideoFrames=true
            videoDataOutputQueue = DispatchQueue(label: "VideoDataOutputQueue")
            videoDataOutput.setSampleBufferDelegate(self, queue:self.videoDataOutputQueue)
            
            if session.canAddOutput(self.videoDataOutput){
                session.addOutput(self.videoDataOutput)
            }
            
            videoDataOutput.connection(with: .video)?.isEnabled = true
            
            previewLayer = AVCaptureVideoPreviewLayer(session: self.session)
            previewLayer.videoGravity = AVLayerVideoGravity.resizeAspect
            
            let rootLayer :CALayer = self.previewView.layer
            rootLayer.masksToBounds=true
            previewLayer.frame = rootLayer.bounds
            rootLayer.addSublayer(self.previewLayer)
            DispatchQueue.global(qos: .userInitiated).async {
                self.session.startRunning()
            }
        } catch let error as NSError {
            deviceInput = nil
            print("error: \(error.localizedDescription)")
        }
    }
    
    func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
        guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return  }
        
        guard let image = UIImage(pixelBuffer: imageBuffer) else {
            return
        }
        detectFaces(in: image)
        //stopCamera()
    }
    
    
    
    func stopCamera(){
        session.stopRunning()
    }
    
    
    private func detectFaces(in image: UIImage) {
        guard let faceRects = OpenCVWrapper.detectFaceRects(in: image) else { return }
        
        DispatchQueue.main.async {
            let viewWidth = self.faceOverlayView.bounds.width
            let viewHeight = self.faceOverlayView.bounds.height
            let imageWidth = image.size.width
            let imageHeight = image.size.height
            
            let scaleX = viewWidth / imageWidth
            let scaleY = viewHeight / imageHeight
            
            let scaleFactor = min(scaleX, scaleY)
            
            let offsetX = (viewWidth - imageWidth * scaleFactor) / 2
            let offsetY = (viewHeight - imageHeight * scaleFactor) / 2
            
            let transformedRects = faceRects.map { $0.cgRectValue }.map { face in
                return CGRect(
                    x: face.origin.x * scaleFactor + offsetX,
                    y: face.origin.y * scaleFactor + offsetY,
                    width: face.size.width * scaleFactor,
                    height: face.size.height * scaleFactor
                )
            }
            
            self.faceOverlayView.setFaces(transformedRects)
        }
    }
    
    
}


extension UIImage {
    public convenience init?(pixelBuffer: CVPixelBuffer) {
        var cgImage: CGImage?
        VTCreateCGImageFromCVPixelBuffer(pixelBuffer, options: nil, imageOut: &cgImage)
        
        guard let cgImage = cgImage else {
            return nil
        }
        
        self.init(cgImage: cgImage)
    }
}

问题排查与解决方案 #

1. 级联分类器加载验证 #

• 确认haarcascade_frontalface_default.xml已添加到项目的Copy Bundle Resources中，避免加载路径错误。
• 强化加载错误处理，避免空分类器导致无检测结果：

  + (void)initialize {
      if (self == [OpenCVWrapper self]) {
          NSString *faceCascadePath = [[NSBundle mainBundle] pathForResource:@"haarcascade_frontalface_default" ofType:@"xml"];
          NSAssert(faceCascadePath != nil, @"Face cascade file not found in bundle");
          if (!faceCascade.load([faceCascadePath UTF8String])) {
              NSLog(@"Error loading face detection model");
              NSAssert(NO, @"Failed to load face cascade classifier");
          }
      }
  }
  

2. 图像方向修正 #

AVCapture后置摄像头输出的图像默认是横向（UIImageOrientation.right），但当前UIImage初始化时未设置方向，导致convertToMat中的旋转逻辑失效，人脸检测时图像方向错误：

• 修改UIImage扩展，添加方向参数：

  extension UIImage {
      public convenience init?(pixelBuffer: CVPixelBuffer, orientation: UIImage.Orientation = .right) {
          var cgImage: CGImage?
          VTCreateCGImageFromCVPixelBuffer(pixelBuffer, options: nil, imageOut: &cgImage)
          
          guard let cgImage = cgImage else {
              return nil
          }
          
          self.init(cgImage: cgImage, scale: UIScreen.main.scale, orientation: orientation)
      }
  }
  

• 在captureOutput中使用正确方向初始化UIImage：

  guard let image = UIImage(pixelBuffer: imageBuffer, orientation: .right) else {
      return
  }
  

3. 人脸检测参数优化 #

调整detectMultiScale参数，提升检测准确率：

faceCascade.detectMultiScale(gray, faces, 1.3, 3, 0 | cv::CASCADE_SCALE_IMAGE, cv::Size(50, 50));

• scaleFactor:1.3：每次缩放的比例，值越大检测速度越快，精度稍降
• minNeighbors:3：过滤候选框的阈值，值越大误检越少
• minSize:Size(50,50)：最小检测人脸尺寸，避免检测过小区域

4. 坐标转换逻辑修正 #

手动计算缩放偏移易出错，改用AVCapture系统方法转换坐标：

private func detectFaces(in image: UIImage) {
    guard let faceRects = OpenCVWrapper.detectFaceRects(in: image) else { return }
    
    DispatchQueue.main.async {
        let imageSize = image.size
        let transformedRects = faceRects.map { value -> CGRect in
            let faceRect = value.cgRectValue
            // 转换为归一化坐标（适配AVCapture metadata输出格式）
            let normalizedRect = CGRect(
                x: faceRect.origin.x / imageSize.width,
                y: faceRect.origin.y / imageSize.height,
                width: faceRect.size.width / imageSize.width,
                height: faceRect.size.height / imageSize.height
            )
            // 转换为预览层的Layer坐标
            let layerRect = self.previewLayer.layerRectConverted(fromMetadataOutputRect: normalizedRect)
            // 转换为faceOverlayView的本地坐标
            return self.faceOverlayView.convert(layerRect, from: self.previewLayer)
        }
        
        self.faceOverlayView.setFaces(transformedRects)
    }
}

5. 调试辅助 #

在detectFaceRectsInUIImage中添加日志，确认是否检测到人脸：

NSLog(@"Detected %zu faces", faces.size());

内容的提问来源于stack exchange，提问作者BraveEvidence