使用Swift开发MacOS应用实现屏幕图像检测

实现步骤与代码示例 #

方案一：基于OpenCV的模板匹配（推荐） #

OpenCV的matchTemplate配合非极大值抑制，能高效查找屏幕上所有重复出现的目标图像，适合精确匹配场景。

依赖准备

在Xcode中通过Swift Package Manager引入OpenCV：

• 包地址：https://github.com/opencv/opencv.git，选择4.x版本

完整代码

import Cocoa
import OpenCV

// 捕获主屏幕截图并转为OpenCV格式
func captureScreen() -> Mat? {
    guard let screen = NSScreen.main else { return nil }
    let screenRect = screen.frame
    let imageRep = NSCaptureScreenRect(screenRect)
    guard let bitmapRep = imageRep as? NSBitmapImageRep else { return nil }
    
    let width = Int(bitmapRep.pixelsWide)
    let height = Int(bitmapRep.pixelsHigh)
    let bytesPerRow = Int(bitmapRep.bytesPerRow)
    let data = bitmapRep.bitmapData!
    
    return Mat(rows: height, cols: width, type: CV_8UC4, data: data, step: bytesPerRow)
}

// 加载本地参考图像
func loadReferenceImage(from path: String) -> Mat? {
    guard let image = NSImage(contentsOfFile: path) else { return nil }
    return OpenCV.Mat(image: image)
}

// 查找所有匹配位置并去重
func findAllMatches(source: Mat, template: Mat, threshold: Double = 0.8) -> [CGRect] {
    var result = Mat()
    OpenCV.matchTemplate(source, template, result, TM_CCOEFF_NORMED)
    
    var matches = [CGRect]()
    let templateWidth = template.cols
    let templateHeight = template.rows
    
    // 遍历结果矩阵，筛选超过阈值的匹配点
    for y in 0..<result.rows {
        for x in 0..<result.cols {
            let value = result[y, x, 0]
            if value >= threshold {
                let rect = CGRect(x: x, y: y, width: templateWidth, height: templateHeight)
                matches.append(rect)
            }
        }
    }
    
    // 非极大值抑制，去除重叠匹配框
    return nonMaxSuppression(rects: matches, threshold: 0.3)
}

// 非极大值抑制实现
func nonMaxSuppression(rects: [CGRect], threshold: Double) -> [CGRect] {
    guard !rects.isEmpty else { return [] }
    
    let sortedRects = rects.sorted { $0.origin.y < $1.origin.y }
    var keepRects = [CGRect]()
    
    for rect in sortedRects {
        var overlap = false
        for kept in keepRects {
            let intersection = rect.intersection(kept)
            let intersectionArea = intersection.width * intersection.height
            let rectArea = rect.width * rect.height
            if intersectionArea / rectArea > threshold {
                overlap = true
                break
            }
        }
        if !overlap {
            keepRects.append(rect)
        }
    }
    
    return keepRects
}

// 执行检测
func runDetection() {
    guard let screenMat = captureScreen(),
          let templateMat = loadReferenceImage(from: "/Users/xxx/Desktop/target.png") else { // 替换为你的参考图像路径
        print("图像加载失败")
        return
    }
    
    let matches = findAllMatches(source: screenMat, template: templateMat)
    print("找到\(matches.count)个匹配位置：")
    for (index, rect) in matches.enumerated() {
        print("第\(index+1)个：\(rect)")
    }
}

runDetection()

方案二：基于Core Image的特征匹配（无第三方依赖） #

无需引入外部库，利用Core Image的特征检测能力实现匹配，适合特征明显的目标图像。

完整代码

import Cocoa
import CoreImage

// 捕获屏幕截图转为CIImage
func captureScreenAsCIImage() -> CIImage? {
    guard let screen = NSScreen.main else { return nil }
    let screenRect = screen.frame
    let imageRep = NSCaptureScreenRect(screenRect)
    guard let bitmapRep = imageRep as? NSBitmapImageRep else { return nil }
    guard let data = bitmapRep.representation(using: .png, properties: [:]) else { return nil }
    return CIImage(data: data)
}

// 提取图像特征点
func extractKeyPoints(from image: CIImage) -> [CIKeyPointFeature]? {
    let detector = CIDetector(ofType: CIDetectorTypeFeature, context: nil, options: [
        CIDetectorAccuracy: CIDetectorAccuracyHigh
    ])
    return detector?.features(in: image) as? [CIKeyPointFeature]
}

// 匹配特征点并生成目标位置
func findMatchingPositions(sourceImage: CIImage, refKeyPoints: [CIKeyPointFeature], refSize: CGSize) -> [CGRect] {
    guard let sourceKeyPoints = extractKeyPoints(from: sourceImage) else { return [] }
    
    var matches = [CGRect]()
    // 匹配特征点（简化版，可根据需求替换为更精确的匹配算法）
    for refPoint in refKeyPoints {
        for sourcePoint in sourceKeyPoints {
            let distance = sqrt(pow(refPoint.x - sourcePoint.x, 2) + pow(refPoint.y - sourcePoint.y, 2))
            if distance < 10 { // 相似度阈值，根据图像大小调整
                let rect = CGRect(
                    x: sourcePoint.x - refSize.width/2,
                    y: sourcePoint.y - refSize.height/2,
                    width: refSize.width,
                    height: refSize.height
                )
                matches.append(rect)
            }
        }
    }
    
    return matches
}

// 执行检测
func runCoreImageDetection() {
    guard let screenImage = captureScreenAsCIImage(),
          let refImage = CIImage(contentsOf: URL(fileURLWithPath: "/Users/xxx/Desktop/target.png")) else { // 替换为你的参考图像路径
        print("图像加载失败")
        return
    }
    
    guard let refKeyPoints = extractKeyPoints(from: refImage) else {
        print("特征提取失败")
        return
    }
    let refSize = CGSize(width: 50, height: 50) // 替换为参考图像实际尺寸
    
    let matches = findMatchingPositions(sourceImage: screenImage, refKeyPoints: refKeyPoints, refSize: refSize)
    print("找到\(matches.count)个匹配位置：")
    for (index, rect) in matches.enumerated() {
        print("第\(index+1)个：\(rect)")
    }
}

runCoreImageDetection()

关键说明 #

• OpenCV方案精度更高，适合完全相同的重复图像，调整threshold参数可控制匹配严格程度；
• Core Image方案无需第三方依赖，但特征匹配稳定性略低，需根据目标图像调整特征阈值和尺寸；
• Mac屏幕坐标系原点在左下角，代码中直接使用原始捕获坐标，若需转换为UI常用的左上角原点，可通过screen.frame.height - rect.origin.y - rect.height计算。

内容的提问来源于stack exchange，提问作者RenegadeAndy