如何通过macOS辅助功能API获取任意应用的文本光标位置？

问题背景 #

• 正在开发一款基于macOS辅助功能API的跨应用自动补全工具
• 使用kAXSelectedTextRangeAttribute可在Xcode、VS Code等应用中正常获取光标位置（推测这类应用会自动选中光标对应的空范围）
• 在TextEdit、Slack等应用中，该属性返回的选中范围长度为0，无法通过原有逻辑获取光标坐标
• 应用已获得必要的辅助功能权限

当前有效代码（针对选中范围长度非0的场景） #

func getCursorPosition() {
    let systemWideElement = AXUIElementCreateSystemWide()
    var focusedElement : AnyObject?
    
    let error = AXUIElementCopyAttributeValue(systemWideElement, kAXFocusedUIElementAttribute as CFString, &focusedElement)
    if (error != .success){
        print("Couldn't get the focused element. Probably a webkit application")
    } else {
        var selectedRangeValue : AnyObject?
        let selectedRangeError = AXUIElementCopyAttributeValue(focusedElement as! AXUIElement, kAXSelectedTextRangeAttribute as CFString, &selectedRangeValue)
        if (selectedRangeError == .success){
            var selectedRange : CFRange?
            AXValueGetValue(selectedRangeValue as! AXValue, AXValueType(rawValue: kAXValueCFRangeType)!, &selectedRange)
            var selectRect = CGRect()
            var selectBounds : AnyObject?
            let selectedBoundsError = AXUIElementCopyParameterizedAttributeValue(focusedElement as! AXUIElement, kAXBoundsForRangeParameterizedAttribute as CFString, selectedRangeValue!, &selectBounds)
            if (selectedBoundsError == .success){
                AXValueGetValue(selectBounds as! AXValue, .cgRect, &selectRect)
                    print("CursorX \(selectRect.origin.x), CursorY: \(selectRect.origin.y)")
            }
        }
    }
}

无效的Hack方法（无法获取精准位置） #

通过硬编码行高估算光标位置的方案适配性差，坐标误差大：

func getCursorPositionHacky(element: AXUIElement) {
    if element.position != nil {
        if element.size != nil {
            if element.insertionPointLineNumber != nil {
                
                // HACK: Calculate the cursor's approximate screen coordinates
                
                let lineHeight: CGFloat = 14.0 // This is an estimation from Inspector view
               
                var cursorX = element.position!.x + 10.0
 
              
                let cursorY = element.position!.y + (CGFloat(element.insertionPointLineNumber! - 1) * lineHeight)
                print("Cursor position is approximately at: (\(cursorX), \(cursorY))")
            }
        }
    }
}

可行解决方案 #

针对返回空选中范围的场景，我们可以通过以下逻辑优化：

1. 当kAXSelectedTextRangeAttribute返回长度为0的范围时，尝试读取kAXInsertionPointAttribute获取插入点的字符位置，构造对应的空范围后调用kAXBoundsForRangeParameterizedAttribute获取精准坐标
2. 若无法直接获取插入点属性，则通过元素的kAXNumberOfLinesAttribute计算行高，替代硬编码值提升估算精度
3. 针对WebKit类应用（如Slack），可遍历焦点元素的子元素，定位到实际的文本输入子元素后再执行上述逻辑

优化后的完整代码 #

func getCursorPosition() {
    let systemWideElement = AXUIElementCreateSystemWide()
    var focusedElement: AnyObject?
    
    let error = AXUIElementCopyAttributeValue(systemWideElement, kAXFocusedUIElementAttribute as CFString, &focusedElement)
    guard error == .success, let element = focusedElement as? AXUIElement else {
        print("无法获取焦点元素，可能是WebKit应用")
        return
    }
    
    var selectedRangeValue: AnyObject?
    let selectedRangeError = AXUIElementCopyAttributeValue(element, kAXSelectedTextRangeAttribute as CFString, &selectedRangeValue)
    
    if selectedRangeError == .success, let rangeValue = selectedRangeValue as? AXValue {
        var selectedRange = CFRange(location: 0, length: 0)
        AXValueGetValue(rangeValue, AXValueType(rawValue: kAXValueCFRangeType)!, &selectedRange)
        
        var targetRangeValue: AXValue?
        // 处理空选中范围（光标为插入点的情况）
        if selectedRange.length == 0 {
            // 尝试读取插入点属性
            var insertionPoint: AnyObject?
            let insertionError = AXUIElementCopyAttributeValue(element, kAXInsertionPointAttribute as CFString, &insertionPoint)
            if insertionError == .success, let point = insertionPoint as? NSNumber {
                let insertionRange = CFRange(location: point.intValue, length: 0)
                targetRangeValue = AXValueCreate(AXValueType(rawValue: kAXValueCFRangeType)!, &insertionRange)
            } else {
                // fallback：直接使用原空范围
                targetRangeValue = rangeValue
            }
        } else {
            targetRangeValue = rangeValue
        }
        
        guard let targetRange = targetRangeValue else {
            print("无法构造目标范围")
            return
        }
        
        // 请求范围对应的屏幕坐标
        var selectBounds: AnyObject?
        let boundsError = AXUIElementCopyParameterizedAttributeValue(element, kAXBoundsForRangeParameterizedAttribute as CFString, targetRange, &selectBounds)
        if boundsError == .success, let boundsValue = selectBounds as? AXValue {
            var selectRect = CGRect()
            AXValueGetValue(boundsValue, .cgRect, &selectRect)
            print("光标坐标：X \(selectRect.origin.x), Y: \(selectRect.origin.y)")
        } else {
            // 调用备选估算方案
            fallbackCursorPosition(for: element)
        }
    } else {
        // 直接调用备选估算方案
        fallbackCursorPosition(for: element)
    }
}

func fallbackCursorPosition(for element: AXUIElement) {
    // 获取元素位置、尺寸、插入点行号
    var positionValue: AnyObject?
    let posError = AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &positionValue)
    
    var sizeValue: AnyObject?
    let sizeError = AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &sizeValue)
    
    var lineNumberValue: AnyObject?
    let lineError = AXUIElementCopyAttributeValue(element, kAXInsertionPointLineNumberAttribute as CFString, &lineNumberValue)
    
    guard posError == .success, let pos = positionValue as? NSValue,
          sizeError == .success, let size = sizeValue as? NSValue,
          lineError == .success, let lineNumber = lineNumberValue as? NSNumber else {
        print("无法获取备选参数，无法估算光标位置")
        return
    }
    
    let elementPosition = pos.cgPointValue
    let elementSize = size.cgSizeValue
    
    // 计算行高：优先用总行数计算，否则用默认值
    var lineCountValue: AnyObject?
    let lineCountError = AXUIElementCopyAttributeValue(element, kAXNumberOfLinesAttribute as CFString, &lineCountValue)
    let lineHeight: CGFloat
    if lineCountError == .success, let lineCount = lineCountValue as? NSNumber, lineCount.intValue > 0 {
        lineHeight = elementSize.height / CGFloat(lineCount.intValue)
    } else {
        lineHeight = 16.0 // 适配常见字体的行高
    }
    
    // 估算光标坐标（X取元素左偏移+基础缩进，Y取对应行的垂直居中位置）
    let cursorX = elementPosition.x + 8.0
    let cursorY = elementPosition.y + (CGFloat(lineNumber.intValue - 1) * lineHeight) + lineHeight / 2
    
    print("估算光标坐标：(\(cursorX), \(cursorY))")
}

关键说明 #

• 优先通过辅助功能API获取精准坐标，只有当API调用失败时才使用估算方案
• 针对WebKit应用，可额外添加子元素遍历逻辑：通过AXUIElementCopyAttributeValues获取焦点元素的子元素，筛选出带有kAXRoleAttribute为kAXTextAreaRole或kAXTextFieldRole的元素，再对该子元素执行上述光标获取逻辑
• 可根据不同应用的特性，调整估算逻辑中的缩进值、行高默认值等参数，进一步提升适配性

内容的提问来源于stack exchange，提问作者ziyasal