452 lines
20 KiB
Swift
452 lines
20 KiB
Swift
import AppKit
|
|
import ApplicationServices
|
|
import ScreenCaptureKit
|
|
|
|
final class AccessibilityService: @unchecked Sendable {
|
|
private let screenshotImageWidth: CGFloat = 768
|
|
private let maxElements = 250
|
|
private let maxDepth = 22
|
|
|
|
private let importantRoles: Set<String> = [
|
|
"AXButton", "AXCheckBox", "AXRadioButton", "AXPopUpButton", "AXMenuButton",
|
|
"AXComboBox", "AXTextField", "AXTextArea", "AXSearchField", "AXLink",
|
|
"AXSlider", "AXIncrementor", "AXScrollArea", "AXScrollBar", "AXTabGroup",
|
|
"AXTab", "AXMenuItem", "AXCell", "AXRow", "AXStaticText", "AXImage",
|
|
"AXOutline", "AXTable", "AXList", "AXGroup",
|
|
]
|
|
|
|
private let enhancedLock = NSLock()
|
|
private var enhancedPIDs = Set<pid_t>()
|
|
|
|
func resolveTarget(appName: String?, pid requestedPID: pid_t? = nil, windowTitle: String? = nil) throws -> WindowTarget {
|
|
guard AXIsProcessTrusted() else { throw ComputerUseError.accessibilityDenied }
|
|
|
|
let app = try resolveApp(appName: appName, pid: requestedPID)
|
|
let pid = app.processIdentifier
|
|
let axApp = AXUIElementCreateApplication(pid)
|
|
if enableEnhancedAccessibility(axApp: axApp, pid: pid) {
|
|
// Chromium and Electron build the renderer AX tree lazily; give it a moment.
|
|
Thread.sleep(forTimeInterval: 0.35)
|
|
}
|
|
let axWindow = firstAXWindow(axApp: axApp, title: windowTitle)
|
|
let title = axWindow.flatMap { axString($0, kAXTitleAttribute) }
|
|
let info = firstCGWindowInfo(pid: pid, title: title)
|
|
let bounds = axWindow.flatMap(axFrame) ?? info?.bounds
|
|
|
|
guard let bounds, bounds.width > 20, bounds.height > 20 else {
|
|
throw ComputerUseError.noWindow(app.localizedName ?? appName ?? "frontmost app")
|
|
}
|
|
|
|
return WindowTarget(
|
|
appName: app.localizedName ?? "Unknown",
|
|
pid: pid,
|
|
windowNumber: info?.number,
|
|
windowTitle: title ?? info?.title,
|
|
bounds: bounds,
|
|
isFrontmost: NSWorkspace.shared.frontmostApplication?.processIdentifier == pid,
|
|
axWindow: axWindow
|
|
)
|
|
}
|
|
|
|
func snapshot(target: WindowTarget, strictMode: Bool, backgroundActivated: Bool) async throws -> AppSnapshot {
|
|
let records = records(target: target)
|
|
let (data, meta) = try await captureScreenshot(target: target)
|
|
|
|
return AppSnapshot(
|
|
id: "",
|
|
observation: 0,
|
|
appName: target.appName,
|
|
pid: target.pid,
|
|
windowNumber: target.windowNumber,
|
|
windowTitle: target.windowTitle,
|
|
screenshotData: data,
|
|
screenshotMimeType: "image/jpeg",
|
|
screenshotMeta: meta,
|
|
records: records,
|
|
strictMode: strictMode,
|
|
backgroundActivated: backgroundActivated,
|
|
recentActions: [],
|
|
addedLabels: [],
|
|
removedLabels: []
|
|
)
|
|
}
|
|
|
|
func records(target: WindowTarget) -> [AXElementRecord] {
|
|
target.axWindow.map(semanticRecords(window:)) ?? []
|
|
}
|
|
|
|
func press(record: AXElementRecord) -> Bool {
|
|
AXUIElementPerformAction(record.element, kAXPressAction as CFString) == .success
|
|
}
|
|
|
|
func focus(record: AXElementRecord) -> Bool {
|
|
AXUIElementSetAttributeValue(record.element, kAXFocusedAttribute as CFString, true as CFBoolean) == .success
|
|
}
|
|
|
|
func setValue(record: AXElementRecord, value: String) -> Bool {
|
|
var settable = DarwinBoolean(false)
|
|
guard AXUIElementIsAttributeSettable(record.element, kAXValueAttribute as CFString, &settable) == .success,
|
|
settable.boolValue else {
|
|
return false
|
|
}
|
|
return AXUIElementSetAttributeValue(record.element, kAXValueAttribute as CFString, value as CFString) == .success
|
|
}
|
|
|
|
func performAction(record: AXElementRecord, action: String) -> Bool {
|
|
AXUIElementPerformAction(record.element, action as CFString) == .success
|
|
}
|
|
|
|
func value(record: AXElementRecord) -> String? {
|
|
axString(record.element, kAXValueAttribute)
|
|
}
|
|
|
|
/// Chromium browsers and Electron apps only expose web content to AX after an
|
|
/// assistive client announces itself. VoiceOver does this by setting
|
|
/// AXEnhancedUserInterface; Electron additionally supports AXManualAccessibility.
|
|
/// Returns true when the attribute was newly applied to this pid.
|
|
private func enableEnhancedAccessibility(axApp: AXUIElement, pid: pid_t) -> Bool {
|
|
enhancedLock.lock()
|
|
let isNew = enhancedPIDs.insert(pid).inserted
|
|
enhancedLock.unlock()
|
|
guard isNew else { return false }
|
|
// Chromium reacts to the set attempt even though it reports an AXError
|
|
// (.notImplemented / .attributeUnsupported), so the return codes are
|
|
// logged for diagnostics but intentionally not treated as failure.
|
|
let enhanced = AXUIElementSetAttributeValue(axApp, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue)
|
|
let manual = AXUIElementSetAttributeValue(axApp, "AXManualAccessibility" as CFString, kCFBooleanTrue)
|
|
fputs("[ComputerUse] enhanced-ax pid=\(pid) AXEnhancedUserInterface=\(enhanced.rawValue) AXManualAccessibility=\(manual.rawValue)\n", stderr)
|
|
return true
|
|
}
|
|
|
|
private func resolveApp(appName: String?, pid: pid_t?) throws -> NSRunningApplication {
|
|
if let pid {
|
|
guard let app = NSRunningApplication(processIdentifier: pid) else {
|
|
throw ComputerUseError.appNotFound("pid \(pid)")
|
|
}
|
|
return app
|
|
}
|
|
guard let rawName = appName?.trimmingCharacters(in: .whitespacesAndNewlines), !rawName.isEmpty else {
|
|
guard let frontmost = NSWorkspace.shared.frontmostApplication else {
|
|
throw ComputerUseError.noFrontmostApplication
|
|
}
|
|
return frontmost
|
|
}
|
|
|
|
let needle = rawName.lowercased()
|
|
let regularApps = NSWorkspace.shared.runningApplications.filter { $0.activationPolicy == .regular }
|
|
if let exact = regularApps.first(where: { $0.localizedName?.lowercased() == needle }) {
|
|
return exact
|
|
}
|
|
if let contains = regularApps.first(where: { $0.localizedName?.lowercased().contains(needle) == true }) {
|
|
return contains
|
|
}
|
|
throw ComputerUseError.appNotFound(rawName)
|
|
}
|
|
|
|
private func firstAXWindow(axApp: AXUIElement, title preferredTitle: String?) -> AXUIElement? {
|
|
var windowValue: CFTypeRef?
|
|
guard AXUIElementCopyAttributeValue(axApp, kAXWindowsAttribute as CFString, &windowValue) == .success,
|
|
let windows = windowValue as? [AXUIElement] else {
|
|
return nil
|
|
}
|
|
let usable = windows.filter { window in
|
|
guard let frame = axFrame(window) else { return false }
|
|
return frame.width > 20 && frame.height > 20
|
|
}
|
|
if let preferredTitle, let match = usable.first(where: { axString($0, kAXTitleAttribute) == preferredTitle }) {
|
|
return match
|
|
}
|
|
return usable.first
|
|
}
|
|
|
|
private func semanticRecords(window: AXUIElement) -> [AXElementRecord] {
|
|
var records: [AXElementRecord] = []
|
|
collect(element: window, depth: 0, records: &records)
|
|
return records
|
|
}
|
|
|
|
private func collect(element: AXUIElement, depth: Int, records: inout [AXElementRecord]) {
|
|
guard depth <= maxDepth, records.count < maxElements else { return }
|
|
|
|
let rawRole = axString(element, kAXRoleAttribute) ?? "AXUnknown"
|
|
let role = normalizedRole(rawRole)
|
|
let value = axString(element, kAXValueAttribute).map { String($0.prefix(120)) }
|
|
let label = semanticLabel(element: element, role: role, value: value)
|
|
let actions = axActions(element)
|
|
let frame = axFrame(element)
|
|
let capabilities = capabilitiesFor(element: element, rawRole: rawRole, actions: actions)
|
|
let shouldSurface = shouldSurfaceElement(rawRole: rawRole, label: label, value: value, frame: frame, capabilities: capabilities)
|
|
|
|
if shouldSurface, let frame {
|
|
let id = records.count + 1
|
|
let semantic = SemanticAXElement(
|
|
id: id,
|
|
ref: "{e\(id)}",
|
|
role: role,
|
|
label: label,
|
|
value: value,
|
|
frame: ElementFrame(
|
|
x: Int(frame.origin.x),
|
|
y: Int(frame.origin.y),
|
|
width: Int(frame.width),
|
|
height: Int(frame.height)
|
|
),
|
|
state: stateFor(element: element, rawRole: rawRole),
|
|
capabilities: capabilities
|
|
)
|
|
records.append(AXElementRecord(element: element, semantic: semantic))
|
|
}
|
|
|
|
for child in axChildren(element) {
|
|
collect(element: child, depth: depth + 1, records: &records)
|
|
if records.count >= maxElements { break }
|
|
}
|
|
}
|
|
|
|
private func shouldSurfaceElement(rawRole: String, label: String, value: String?, frame: CGRect?, capabilities: AXElementCapabilities) -> Bool {
|
|
guard let frame, frame.width > 1, frame.height > 1 else { return false }
|
|
let hasSemanticText = !label.isEmpty || value?.isEmpty == false
|
|
let interactive = capabilities.canPress || capabilities.canFocus || capabilities.canScroll || capabilities.canAdjust || capabilities.canSetValue
|
|
if interactive { return true }
|
|
if !importantRoles.contains(rawRole) { return false }
|
|
if rawRole == "AXGroup" { return hasSemanticText && frame.width < 900 && frame.height < 700 }
|
|
return hasSemanticText
|
|
}
|
|
|
|
private func semanticLabel(element: AXUIElement, role: String, value: String?) -> String {
|
|
let candidates = [
|
|
axString(element, kAXTitleAttribute),
|
|
axString(element, kAXDescriptionAttribute),
|
|
axString(element, kAXHelpAttribute),
|
|
axString(element, kAXIdentifierAttribute),
|
|
value,
|
|
]
|
|
for candidate in candidates {
|
|
if let trimmed = candidate?.trimmingCharacters(in: .whitespacesAndNewlines), !trimmed.isEmpty {
|
|
return String(trimmed.prefix(120))
|
|
}
|
|
}
|
|
return role
|
|
}
|
|
|
|
private func capabilitiesFor(element: AXUIElement, rawRole: String, actions: [String]) -> AXElementCapabilities {
|
|
var valueSettable = DarwinBoolean(false)
|
|
let canSetValue = AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &valueSettable) == .success && valueSettable.boolValue
|
|
|
|
var focusSettable = DarwinBoolean(false)
|
|
let canFocus = AXUIElementIsAttributeSettable(element, kAXFocusedAttribute as CFString, &focusSettable) == .success && focusSettable.boolValue
|
|
|
|
let canAdjust = actions.contains(kAXIncrementAction) || actions.contains(kAXDecrementAction) || rawRole == "AXSlider" || rawRole == "AXIncrementor"
|
|
let canScroll = actions.contains("AXScrollToVisible") || rawRole == "AXScrollArea" || rawRole == "AXScrollBar"
|
|
let canPress = actions.contains(kAXPressAction) || ["AXButton", "AXCheckBox", "AXRadioButton", "AXLink", "AXMenuItem", "AXPopUpButton", "AXMenuButton", "AXCell"].contains(rawRole)
|
|
|
|
return AXElementCapabilities(
|
|
canPress: canPress,
|
|
canFocus: canFocus,
|
|
canScroll: canScroll,
|
|
canAdjust: canAdjust,
|
|
canSetValue: canSetValue,
|
|
actions: actions
|
|
)
|
|
}
|
|
|
|
private func stateFor(element: AXUIElement, rawRole: String) -> AXElementState {
|
|
AXElementState(
|
|
enabled: axBool(element, kAXEnabledAttribute),
|
|
focused: axBool(element, kAXFocusedAttribute),
|
|
selected: axBool(element, kAXSelectedAttribute),
|
|
expanded: axBool(element, kAXExpandedAttribute),
|
|
checked: rawRole == "AXCheckBox" || rawRole == "AXRadioButton" ? axBool(element, kAXValueAttribute) : nil
|
|
)
|
|
}
|
|
|
|
private func axChildren(_ element: AXUIElement) -> [AXUIElement] {
|
|
var value: CFTypeRef?
|
|
guard AXUIElementCopyAttributeValue(element, kAXChildrenAttribute as CFString, &value) == .success,
|
|
let children = value as? [AXUIElement] else {
|
|
return []
|
|
}
|
|
return children
|
|
}
|
|
|
|
private func axActions(_ element: AXUIElement) -> [String] {
|
|
var actionNames: CFArray?
|
|
guard AXUIElementCopyActionNames(element, &actionNames) == .success,
|
|
let names = actionNames as? [String] else {
|
|
return []
|
|
}
|
|
return names
|
|
}
|
|
|
|
private func axString(_ element: AXUIElement, _ attribute: String) -> String? {
|
|
var value: CFTypeRef?
|
|
guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success,
|
|
let value else {
|
|
return nil
|
|
}
|
|
if let string = value as? String, !string.isEmpty { return string }
|
|
if let attributed = value as? NSAttributedString, !attributed.string.isEmpty { return attributed.string }
|
|
if let number = value as? NSNumber { return number.stringValue }
|
|
return nil
|
|
}
|
|
|
|
private func axBool(_ element: AXUIElement, _ attribute: String) -> Bool? {
|
|
var value: CFTypeRef?
|
|
guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success,
|
|
let value else {
|
|
return nil
|
|
}
|
|
if let bool = value as? Bool { return bool }
|
|
if let number = value as? NSNumber { return number.boolValue }
|
|
return nil
|
|
}
|
|
|
|
private func axFrame(_ element: AXUIElement) -> CGRect? {
|
|
guard let position = axPoint(element, kAXPositionAttribute), let size = axSize(element, kAXSizeAttribute) else {
|
|
return nil
|
|
}
|
|
return CGRect(origin: position, size: size)
|
|
}
|
|
|
|
private func axPoint(_ element: AXUIElement, _ attribute: String) -> CGPoint? {
|
|
var value: CFTypeRef?
|
|
guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success,
|
|
let value,
|
|
CFGetTypeID(value) == AXValueGetTypeID() else {
|
|
return nil
|
|
}
|
|
let axValue = value as! AXValue
|
|
guard AXValueGetType(axValue) == .cgPoint else { return nil }
|
|
var point = CGPoint.zero
|
|
guard AXValueGetValue(axValue, .cgPoint, &point) else { return nil }
|
|
return point
|
|
}
|
|
|
|
private func axSize(_ element: AXUIElement, _ attribute: String) -> CGSize? {
|
|
var value: CFTypeRef?
|
|
guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success,
|
|
let value,
|
|
CFGetTypeID(value) == AXValueGetTypeID() else {
|
|
return nil
|
|
}
|
|
let axValue = value as! AXValue
|
|
guard AXValueGetType(axValue) == .cgSize else { return nil }
|
|
var size = CGSize.zero
|
|
guard AXValueGetValue(axValue, .cgSize, &size) else { return nil }
|
|
return size
|
|
}
|
|
|
|
private func normalizedRole(_ rawRole: String) -> String {
|
|
rawRole.hasPrefix("AX") ? String(rawRole.dropFirst(2)) : rawRole
|
|
}
|
|
|
|
private func firstCGWindowInfo(pid: pid_t, title: String?) -> (number: Int, title: String?, bounds: CGRect)? {
|
|
guard let list = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else {
|
|
return nil
|
|
}
|
|
|
|
let candidates = list.compactMap { info -> (number: Int, title: String?, bounds: CGRect)? in
|
|
guard let ownerPID = info[kCGWindowOwnerPID as String] as? Int32,
|
|
ownerPID == pid,
|
|
let layer = info[kCGWindowLayer as String] as? Int,
|
|
layer == 0,
|
|
let number = info[kCGWindowNumber as String] as? Int,
|
|
let boundsDict = info[kCGWindowBounds as String] as? [String: Any] else {
|
|
return nil
|
|
}
|
|
let bounds = CGRect(
|
|
x: cgFloat(boundsDict["X"]),
|
|
y: cgFloat(boundsDict["Y"]),
|
|
width: cgFloat(boundsDict["Width"]),
|
|
height: cgFloat(boundsDict["Height"])
|
|
)
|
|
guard bounds.width > 20, bounds.height > 20 else { return nil }
|
|
return (number, info[kCGWindowName as String] as? String, bounds)
|
|
}
|
|
|
|
if let title, let exact = candidates.first(where: { $0.title == title }) {
|
|
return exact
|
|
}
|
|
return candidates.first
|
|
}
|
|
|
|
private func captureScreenshot(target: WindowTarget) async throws -> (Data, ScreenshotMetadata) {
|
|
let cgImage = await screenCaptureKitImage(target: target) ?? legacyCaptureImage(target: target)
|
|
guard let cgImage else { throw ComputerUseError.screenshotFailed }
|
|
|
|
return try encodeScreenshot(cgImage, capturedBounds: target.bounds)
|
|
}
|
|
|
|
private func legacyCaptureImage(target: WindowTarget) -> CGImage? {
|
|
if let windowNumber = target.windowNumber {
|
|
return CGWindowListCreateImage(
|
|
CGRect.null,
|
|
.optionIncludingWindow,
|
|
CGWindowID(windowNumber),
|
|
[.bestResolution, .boundsIgnoreFraming]
|
|
)
|
|
}
|
|
|
|
return CGWindowListCreateImage(target.bounds, .optionOnScreenOnly, kCGNullWindowID, [.bestResolution])
|
|
}
|
|
|
|
private func screenCaptureKitImage(target: WindowTarget) async -> CGImage? {
|
|
guard let windowNumber = target.windowNumber else { return nil }
|
|
do {
|
|
let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)
|
|
guard let window = content.windows.first(where: { Int($0.windowID) == windowNumber }) else { return nil }
|
|
let configuration = SCStreamConfiguration()
|
|
let scale = screenScale(for: target.bounds)
|
|
configuration.width = max(1, Int(target.bounds.width * scale))
|
|
configuration.height = max(1, Int(target.bounds.height * scale))
|
|
configuration.showsCursor = false
|
|
let filter = SCContentFilter(desktopIndependentWindow: window)
|
|
return try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: configuration)
|
|
} catch {
|
|
return nil
|
|
}
|
|
}
|
|
|
|
private func screenScale(for bounds: CGRect) -> CGFloat {
|
|
NSScreen.screens.first(where: { $0.frame.intersects(bounds) })?.backingScaleFactor
|
|
?? NSScreen.main?.backingScaleFactor
|
|
?? 2
|
|
}
|
|
|
|
private func encodeScreenshot(_ cgImage: CGImage, capturedBounds: CGRect) throws -> (Data, ScreenshotMetadata) {
|
|
let rawWidth = CGFloat(cgImage.width)
|
|
let rawHeight = CGFloat(cgImage.height)
|
|
let targetWidth = min(screenshotImageWidth, rawWidth)
|
|
let targetHeight = rawHeight * (targetWidth / rawWidth)
|
|
|
|
let source = NSImage(cgImage: cgImage, size: NSSize(width: rawWidth, height: rawHeight))
|
|
let resized = NSImage(size: NSSize(width: targetWidth, height: targetHeight))
|
|
resized.lockFocus()
|
|
source.draw(in: NSRect(x: 0, y: 0, width: targetWidth, height: targetHeight))
|
|
resized.unlockFocus()
|
|
|
|
guard let tiff = resized.tiffRepresentation,
|
|
let rep = NSBitmapImageRep(data: tiff),
|
|
let jpeg = rep.representation(using: .jpeg, properties: [.compressionFactor: 0.45]) else {
|
|
throw ComputerUseError.screenshotFailed
|
|
}
|
|
|
|
return (
|
|
jpeg,
|
|
ScreenshotMetadata(
|
|
imageWidth: Int(targetWidth),
|
|
imageHeight: Int(targetHeight),
|
|
capturedBounds: capturedBounds
|
|
)
|
|
)
|
|
}
|
|
|
|
private func cgFloat(_ value: Any?) -> CGFloat {
|
|
if let value = value as? CGFloat { return value }
|
|
if let value = value as? Double { return CGFloat(value) }
|
|
if let value = value as? Int { return CGFloat(value) }
|
|
if let value = value as? NSNumber { return CGFloat(truncating: value) }
|
|
return 0
|
|
}
|
|
}
|