384 lines
20 KiB
Swift
384 lines
20 KiB
Swift
import AppKit
|
|
import Foundation
|
|
|
|
actor ComputerUseRuntime {
|
|
private let accessibility = AccessibilityService()
|
|
private let foregroundInput = InputService()
|
|
private var lastSnapshot: AppSnapshot?
|
|
private var strictMode = true
|
|
private var activationSession: BackgroundActivationSession?
|
|
private var activationKey: String?
|
|
private var activatedWindowKey: String?
|
|
private var activationPreviousPID: pid_t?
|
|
private var activationTargetPID: pid_t?
|
|
private var frontmostMonitor: FrontmostApplicationMonitor?
|
|
private var snapshotSequence = 0
|
|
private var previousLabelsByWindow: [String: Set<String>] = [:]
|
|
private var recentActions: [String] = []
|
|
private var staleSnapshotReason: String?
|
|
|
|
func setStrictMode(_ enabled: Bool) -> ActionMetadata {
|
|
strictMode = enabled
|
|
if !enabled {
|
|
resetBackgroundActivation()
|
|
}
|
|
return ActionMetadata(
|
|
ok: true,
|
|
path: .none,
|
|
strictMode: enabled,
|
|
backgroundSafe: enabled,
|
|
fallbackUsed: false,
|
|
message: enabled ? "Strict background mode enabled." : "Strict background mode disabled. Foreground fallback is allowed."
|
|
)
|
|
}
|
|
|
|
func snapshot(appName: String?, pid: Int? = nil, windowTitle: String? = nil, strict requestedStrict: Bool?) async throws -> AppSnapshot {
|
|
let effectiveStrict = requestedStrict ?? strictMode
|
|
if !effectiveStrict {
|
|
resetBackgroundActivation()
|
|
}
|
|
|
|
let requestedPID = pid.map(pid_t.init)
|
|
var target = try accessibility.resolveTarget(appName: appName, pid: requestedPID, windowTitle: windowTitle)
|
|
let backgroundActivated: Bool
|
|
if effectiveStrict, !target.isFrontmost {
|
|
backgroundActivated = try await ensureBackgroundActivation(target: target)
|
|
target = try accessibility.resolveTarget(appName: appName, pid: requestedPID, windowTitle: windowTitle)
|
|
} else {
|
|
backgroundActivated = false
|
|
}
|
|
|
|
let snapshot = try await accessibility.snapshot(
|
|
target: target,
|
|
strictMode: effectiveStrict,
|
|
backgroundActivated: backgroundActivated
|
|
)
|
|
let annotatedSnapshot = annotate(snapshot: snapshot)
|
|
staleSnapshotReason = nil
|
|
lastSnapshot = annotatedSnapshot
|
|
return annotatedSnapshot
|
|
}
|
|
|
|
func click(snapshotID: String?, ref: String?, index: Int?, imageX: Double?, imageY: Double?, screenX: Double? = nil, screenY: Double? = nil, clickCount: Int, strict requestedStrict: Bool?) async throws -> ActionMetadata {
|
|
let snapshot = try requireSnapshot(snapshotID: snapshotID)
|
|
let effectiveStrict = requestedStrict ?? snapshot.strictMode
|
|
try validateSnapshotForAction(snapshot: snapshot, strict: effectiveStrict)
|
|
|
|
if let record = findRecord(ref: ref, index: index, in: snapshot) {
|
|
let fresh = freshRecord(matching: record, in: snapshot)
|
|
if let fresh {
|
|
AgentCursorOverlay.shared.show(at: fresh.semantic.frame.center)
|
|
if fresh.semantic.capabilities.canPress, accessibility.press(record: fresh) {
|
|
return recordAction(ActionMetadata(ok: true, path: .accessibility, strictMode: effectiveStrict, backgroundSafe: true, fallbackUsed: false, message: "Pressed \(fresh.semantic.ref) via AXPress."))
|
|
}
|
|
if fresh.semantic.capabilities.canFocus, accessibility.focus(record: fresh) {
|
|
return recordAction(ActionMetadata(ok: true, path: .accessibility, strictMode: effectiveStrict, backgroundSafe: true, fallbackUsed: false, message: "Focused \(fresh.semantic.ref) via AX."))
|
|
}
|
|
return try await clickPoint(fresh.semantic.frame.center, clickCount: clickCount, strict: effectiveStrict, fallbackUsed: true)
|
|
}
|
|
return try await clickPoint(record.semantic.frame.center, clickCount: clickCount, strict: effectiveStrict, fallbackUsed: true)
|
|
}
|
|
|
|
if let screenX, let screenY {
|
|
return try await clickPoint(CGPoint(x: screenX, y: screenY), clickCount: clickCount, strict: effectiveStrict, fallbackUsed: false)
|
|
}
|
|
|
|
if let imageX, let imageY {
|
|
let point = snapshot.screenshotMeta.toScreen(imageX: imageX, imageY: imageY)
|
|
return try await clickPoint(point, clickCount: clickCount, strict: effectiveStrict, fallbackUsed: false)
|
|
}
|
|
|
|
throw ComputerUseError.invalidElement(ref ?? index.map(String.init) ?? "<missing>")
|
|
}
|
|
|
|
func typeText(snapshotID: String?, text: String, strict requestedStrict: Bool?) throws -> ActionMetadata {
|
|
let snapshot = try requireSnapshot(snapshotID: snapshotID)
|
|
let effectiveStrict = requestedStrict ?? snapshot.strictMode
|
|
try validateSnapshotForAction(snapshot: snapshot, strict: effectiveStrict)
|
|
if effectiveStrict {
|
|
try BackgroundInputDispatcher.typeText(pid: snapshot.pid, text: text)
|
|
return recordAction(ActionMetadata(ok: true, path: .backgroundCGEvent, strictMode: true, backgroundSafe: true, fallbackUsed: false, message: "Typed text with postToPid."))
|
|
}
|
|
|
|
try foregroundInput.typeText(text)
|
|
return recordAction(ActionMetadata(ok: true, path: .foregroundCGEvent, strictMode: false, backgroundSafe: false, fallbackUsed: true, message: "Typed text with foreground HID fallback."))
|
|
}
|
|
|
|
func pressKey(snapshotID: String?, combo: String, strict requestedStrict: Bool?) throws -> ActionMetadata {
|
|
let snapshot = try requireSnapshot(snapshotID: snapshotID)
|
|
let effectiveStrict = requestedStrict ?? snapshot.strictMode
|
|
try validateSnapshotForAction(snapshot: snapshot, strict: effectiveStrict)
|
|
if effectiveStrict {
|
|
try BackgroundInputDispatcher.pressKey(pid: snapshot.pid, combo: combo)
|
|
return recordAction(ActionMetadata(ok: true, path: .backgroundCGEvent, strictMode: true, backgroundSafe: true, fallbackUsed: false, message: "Pressed key with postToPid."))
|
|
}
|
|
|
|
try foregroundInput.pressKey(combo)
|
|
return recordAction(ActionMetadata(ok: true, path: .foregroundCGEvent, strictMode: false, backgroundSafe: false, fallbackUsed: true, message: "Pressed key with foreground HID fallback."))
|
|
}
|
|
|
|
func scroll(snapshotID: String?, direction: String?, pages: Double, imageX: Double?, imageY: Double?, strict requestedStrict: Bool?) throws -> ActionMetadata {
|
|
let snapshot = try requireSnapshot(snapshotID: snapshotID)
|
|
let effectiveStrict = requestedStrict ?? snapshot.strictMode
|
|
try validateSnapshotForAction(snapshot: snapshot, strict: effectiveStrict)
|
|
let amount = max(1, Int32(pages * 5))
|
|
let deltas = scrollDeltas(direction: direction, amount: amount)
|
|
let point: CGPoint = {
|
|
if let imageX, let imageY {
|
|
return snapshot.screenshotMeta.toScreen(imageX: imageX, imageY: imageY)
|
|
}
|
|
return CGPoint(x: snapshot.screenshotMeta.capturedBounds.midX, y: snapshot.screenshotMeta.capturedBounds.midY)
|
|
}()
|
|
AgentCursorOverlay.shared.show(at: point)
|
|
|
|
if effectiveStrict {
|
|
guard let windowNumber = snapshot.windowNumber else {
|
|
throw ComputerUseError.strictModeViolation("background scroll requires a CG window number")
|
|
}
|
|
try BackgroundInputDispatcher.scroll(pid: snapshot.pid, windowNumber: windowNumber, point: point, deltaX: deltas.x, deltaY: deltas.y)
|
|
return recordAction(ActionMetadata(ok: true, path: .backgroundCGEvent, strictMode: true, backgroundSafe: true, fallbackUsed: false, message: "Scrolled with postToPid."))
|
|
}
|
|
|
|
try foregroundInput.scroll(point: point, deltaX: deltas.x, deltaY: deltas.y)
|
|
return recordAction(ActionMetadata(ok: true, path: .foregroundCGEvent, strictMode: false, backgroundSafe: false, fallbackUsed: true, message: "Scrolled with foreground HID fallback."))
|
|
}
|
|
|
|
func setValue(snapshotID: String?, ref: String?, index: Int?, value: String) throws -> ActionMetadata {
|
|
let snapshot = try requireSnapshot(snapshotID: snapshotID)
|
|
try validateSnapshotForAction(snapshot: snapshot, strict: snapshot.strictMode)
|
|
guard let record = findRecord(ref: ref, index: index, in: snapshot) else {
|
|
throw ComputerUseError.invalidElement(ref ?? index.map(String.init) ?? "<missing>")
|
|
}
|
|
guard let fresh = freshRecord(matching: record, in: snapshot) else {
|
|
throw ComputerUseError.staleSnapshot("The target element changed. Take a new snapshot before setting its value.")
|
|
}
|
|
AgentCursorOverlay.shared.show(at: fresh.semantic.frame.center)
|
|
let axWriteAccepted = accessibility.setValue(record: fresh, value: value)
|
|
if axWriteAccepted, valueMatches(record: fresh, expected: value) {
|
|
return recordAction(ActionMetadata(ok: true, path: .accessibility, strictMode: snapshot.strictMode, backgroundSafe: true, fallbackUsed: false, message: "Set \(fresh.semantic.ref) via AXValue."))
|
|
}
|
|
|
|
// Web inputs (Chromium/Electron) accept AXValue writes without applying
|
|
// them. Fall back to focus + select-all + retype, then verify.
|
|
_ = accessibility.focus(record: fresh)
|
|
try BackgroundInputDispatcher.pressKey(pid: snapshot.pid, combo: "command+a")
|
|
try BackgroundInputDispatcher.typeText(pid: snapshot.pid, text: value)
|
|
Thread.sleep(forTimeInterval: 0.08)
|
|
let verified = valueMatches(record: fresh, expected: value)
|
|
return recordAction(ActionMetadata(
|
|
ok: verified,
|
|
path: .backgroundCGEvent,
|
|
strictMode: snapshot.strictMode,
|
|
backgroundSafe: true,
|
|
fallbackUsed: true,
|
|
message: verified
|
|
? "AXValue write did not stick; set \(fresh.semantic.ref) via select-all + retype fallback."
|
|
: "Could not verify the value of \(fresh.semantic.ref) after AXValue and retype fallback."
|
|
))
|
|
}
|
|
|
|
private func valueMatches(record: AXElementRecord, expected: String) -> Bool {
|
|
accessibility.value(record: record) == expected
|
|
}
|
|
|
|
func performAction(snapshotID: String?, ref: String?, index: Int?, action: String) throws -> ActionMetadata {
|
|
let snapshot = try requireSnapshot(snapshotID: snapshotID)
|
|
try validateSnapshotForAction(snapshot: snapshot, strict: snapshot.strictMode)
|
|
guard let record = findRecord(ref: ref, index: index, in: snapshot) else {
|
|
throw ComputerUseError.invalidElement(ref ?? index.map(String.init) ?? "<missing>")
|
|
}
|
|
guard let fresh = freshRecord(matching: record, in: snapshot) else {
|
|
throw ComputerUseError.staleSnapshot("The target element changed. Take a new snapshot before performing another action.")
|
|
}
|
|
AgentCursorOverlay.shared.show(at: fresh.semantic.frame.center)
|
|
let ok = accessibility.performAction(record: fresh, action: action)
|
|
return recordAction(ActionMetadata(ok: ok, path: .accessibility, strictMode: snapshot.strictMode, backgroundSafe: true, fallbackUsed: false, message: ok ? "Performed \(action) on \(fresh.semantic.ref)." : "AX action \(action) failed."))
|
|
}
|
|
|
|
func wait(milliseconds: Int) async -> ActionMetadata {
|
|
let clamped = max(0, min(milliseconds, 10_000))
|
|
try? await Task.sleep(nanoseconds: UInt64(clamped) * 1_000_000)
|
|
return ActionMetadata(ok: true, path: .none, strictMode: strictMode, backgroundSafe: true, fallbackUsed: false, message: "Waited \(clamped)ms.")
|
|
}
|
|
|
|
private func clickPoint(_ point: CGPoint, clickCount: Int, strict: Bool, fallbackUsed: Bool) async throws -> ActionMetadata {
|
|
let snapshot = try requireSnapshot()
|
|
try validateSnapshotForAction(snapshot: snapshot, strict: strict)
|
|
AgentCursorOverlay.shared.show(at: point)
|
|
if strict {
|
|
guard let windowNumber = snapshot.windowNumber else {
|
|
throw ComputerUseError.strictModeViolation("background click requires a CG window number")
|
|
}
|
|
try await BackgroundInputDispatcher.click(pid: snapshot.pid, windowNumber: windowNumber, point: point, doubleClick: clickCount >= 2)
|
|
return recordAction(ActionMetadata(ok: true, path: .backgroundCGEvent, strictMode: true, backgroundSafe: true, fallbackUsed: fallbackUsed, message: "Clicked with postToPid at \(Int(point.x)),\(Int(point.y))."))
|
|
}
|
|
|
|
try await foregroundInput.click(point: point, doubleClick: clickCount >= 2)
|
|
return recordAction(ActionMetadata(ok: true, path: .foregroundCGEvent, strictMode: false, backgroundSafe: false, fallbackUsed: true, message: "Clicked with foreground HID fallback at \(Int(point.x)),\(Int(point.y))."))
|
|
}
|
|
|
|
private func ensureBackgroundActivation(target: WindowTarget) async throws -> Bool {
|
|
ensureFrontmostMonitor()
|
|
guard let previousPID = NSWorkspace.shared.frontmostApplication?.processIdentifier else {
|
|
throw ComputerUseError.noFrontmostApplication
|
|
}
|
|
let nextActivationKey = "\(previousPID):\(target.pid)"
|
|
if activationKey != nextActivationKey {
|
|
resetBackgroundActivation()
|
|
let next = BackgroundActivationSession(previousPID: previousPID, targetPID: target.pid)
|
|
try next.start()
|
|
activationSession = next
|
|
activationKey = nextActivationKey
|
|
activationPreviousPID = previousPID
|
|
activationTargetPID = target.pid
|
|
activatedWindowKey = nil
|
|
}
|
|
|
|
let nextWindowKey = "\(target.pid):\(target.windowNumber ?? -1)"
|
|
if activatedWindowKey != nextWindowKey {
|
|
guard let activationSession else {
|
|
throw ComputerUseError.strictModeViolation("background activation session was not created")
|
|
}
|
|
try await activationSession.activate(target: target)
|
|
activatedWindowKey = nextWindowKey
|
|
}
|
|
return true
|
|
}
|
|
|
|
private func ensureFrontmostMonitor() {
|
|
if frontmostMonitor != nil { return }
|
|
frontmostMonitor = FrontmostApplicationMonitor { [weak self] pid in
|
|
guard let runtime = self else { return }
|
|
Task { await runtime.frontmostApplicationChanged(pid: pid) }
|
|
}
|
|
}
|
|
|
|
private func frontmostApplicationChanged(pid: pid_t?) {
|
|
guard let pid, activationSession != nil else { return }
|
|
if pid == activationPreviousPID {
|
|
staleSnapshotReason = "The user returned focus to the previous app. Take a new snapshot before sending more input."
|
|
} else if pid == activationTargetPID {
|
|
staleSnapshotReason = "The target app became frontmost. Take a new snapshot before continuing."
|
|
} else {
|
|
staleSnapshotReason = "The user changed app focus. Take a new snapshot before sending more input."
|
|
}
|
|
resetBackgroundActivation()
|
|
}
|
|
|
|
private func resetBackgroundActivation() {
|
|
activationSession?.stop()
|
|
activationSession = nil
|
|
activationKey = nil
|
|
activatedWindowKey = nil
|
|
activationPreviousPID = nil
|
|
activationTargetPID = nil
|
|
}
|
|
|
|
private func requireSnapshot(snapshotID: String? = nil) throws -> AppSnapshot {
|
|
guard let lastSnapshot else { throw ComputerUseError.noSnapshot }
|
|
if let snapshotID, snapshotID != lastSnapshot.id {
|
|
throw ComputerUseError.staleSnapshot("Snapshot \(snapshotID) is no longer current. Take a new snapshot before retrying.")
|
|
}
|
|
return lastSnapshot
|
|
}
|
|
|
|
private func validateSnapshotForAction(snapshot: AppSnapshot, strict: Bool) throws {
|
|
if let staleSnapshotReason {
|
|
throw ComputerUseError.staleSnapshot(staleSnapshotReason)
|
|
}
|
|
guard strict else { return }
|
|
if NSWorkspace.shared.frontmostApplication?.processIdentifier == snapshot.pid { return }
|
|
guard activationSession != nil, activationTargetPID == snapshot.pid else {
|
|
throw ComputerUseError.staleSnapshot("The target app is no longer safely activated. Take a new snapshot before retrying.")
|
|
}
|
|
}
|
|
|
|
private func annotate(snapshot: AppSnapshot) -> AppSnapshot {
|
|
snapshotSequence += 1
|
|
let key = windowKey(snapshot: snapshot)
|
|
let labels = Set(snapshot.records.map { stableLabel(for: $0.semantic) }.filter { !$0.isEmpty })
|
|
let previous = previousLabelsByWindow[key] ?? []
|
|
previousLabelsByWindow[key] = labels
|
|
|
|
let added = labels.subtracting(previous).sorted()
|
|
let removed = previous.subtracting(labels).sorted()
|
|
return snapshot.withSessionMetadata(
|
|
id: UUID().uuidString.lowercased(),
|
|
observation: snapshotSequence,
|
|
recentActions: Array(recentActions.suffix(8)),
|
|
addedLabels: Array(added.prefix(16)),
|
|
removedLabels: Array(removed.prefix(16))
|
|
)
|
|
}
|
|
|
|
private func recordAction(_ metadata: ActionMetadata) -> ActionMetadata {
|
|
recentActions.append(metadata.message)
|
|
if recentActions.count > 12 {
|
|
recentActions.removeFirst(recentActions.count - 12)
|
|
}
|
|
return metadata
|
|
}
|
|
|
|
private func freshRecord(matching record: AXElementRecord, in snapshot: AppSnapshot) -> AXElementRecord? {
|
|
guard let target = try? accessibility.resolveTarget(appName: snapshot.appName, pid: snapshot.pid, windowTitle: snapshot.windowTitle) else {
|
|
return nil
|
|
}
|
|
let records = accessibility.records(target: target)
|
|
if let index = records.firstIndex(where: { $0.semantic.id == record.semantic.id }), stableMatch(records[index].semantic, record.semantic) {
|
|
return records[index]
|
|
}
|
|
return records.first { stableMatch($0.semantic, record.semantic) }
|
|
}
|
|
|
|
private func stableMatch(_ candidate: SemanticAXElement, _ target: SemanticAXElement) -> Bool {
|
|
candidate.role == target.role && stableLabel(for: candidate) == stableLabel(for: target)
|
|
}
|
|
|
|
private func stableLabel(for element: SemanticAXElement) -> String {
|
|
let value = element.value?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
|
let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
let display = value.isEmpty ? label : "\(label) \(value)"
|
|
return "\(element.role):\(display)"
|
|
.components(separatedBy: .whitespacesAndNewlines)
|
|
.filter { !$0.isEmpty }
|
|
.joined(separator: " ")
|
|
.lowercased()
|
|
}
|
|
|
|
private func windowKey(snapshot: AppSnapshot) -> String {
|
|
"\(snapshot.pid):\(snapshot.windowNumber ?? -1)"
|
|
}
|
|
|
|
private func findRecord(ref: String?, index: Int?, in snapshot: AppSnapshot) -> AXElementRecord? {
|
|
if let ref {
|
|
let normalized = normalizeRef(ref)
|
|
return snapshot.records.first { $0.semantic.ref == normalized }
|
|
}
|
|
if let index {
|
|
if let byID = snapshot.records.first(where: { $0.semantic.id == index }) {
|
|
return byID
|
|
}
|
|
if index >= 0 && index < snapshot.records.count {
|
|
return snapshot.records[index]
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
private func normalizeRef(_ raw: String) -> String {
|
|
let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
if trimmed.hasPrefix("{e"), trimmed.hasSuffix("}") { return trimmed }
|
|
if trimmed.hasPrefix("e") { return "{\(trimmed)}" }
|
|
return "{e\(trimmed)}"
|
|
}
|
|
|
|
private func scrollDeltas(direction: String?, amount: Int32) -> (x: Int32, y: Int32) {
|
|
switch direction?.lowercased() ?? "down" {
|
|
case "up": return (0, amount)
|
|
case "left": return (amount, 0)
|
|
case "right": return (-amount, 0)
|
|
default: return (0, -amount)
|
|
}
|
|
}
|
|
}
|