源码快照 @ 2026-09-02 (source-only, 剔除非源码素材)

packages/handsfree/native/HandsFree/.gitignore

@@ -0,0 +1,2 @@
+.build/
+.swiftpm/

packages/handsfree/native/HandsFree/Package.swift

@@ -0,0 +1,13 @@
+// swift-tools-version: 5.9
+import PackageDescription
+
+let package = Package(
+    name: "HandsFree",
+    platforms: [.macOS(.v14)],
+    targets: [
+        .executableTarget(
+            name: "HandsFreeComputerUse",
+            path: "Sources/ComputerUse"
+        ),
+    ]
+)

packages/handsfree/native/HandsFree/Sources/ComputerUse/AccessibilityService.swift

@@ -0,0 +1,451 @@
+import AppKit
+import ApplicationServices
+import ScreenCaptureKit
+
+final class AccessibilityService: @unchecked Sendable {
+    private let screenshotImageWidth: CGFloat = 768
+    private let maxElements = 250
+    private let maxDepth = 22
+
+    private let importantRoles: Set<String> = [
+        "AXButton", "AXCheckBox", "AXRadioButton", "AXPopUpButton", "AXMenuButton",
+        "AXComboBox", "AXTextField", "AXTextArea", "AXSearchField", "AXLink",
+        "AXSlider", "AXIncrementor", "AXScrollArea", "AXScrollBar", "AXTabGroup",
+        "AXTab", "AXMenuItem", "AXCell", "AXRow", "AXStaticText", "AXImage",
+        "AXOutline", "AXTable", "AXList", "AXGroup",
+    ]
+
+    private let enhancedLock = NSLock()
+    private var enhancedPIDs = Set<pid_t>()
+
+    func resolveTarget(appName: String?, pid requestedPID: pid_t? = nil, windowTitle: String? = nil) throws -> WindowTarget {
+        guard AXIsProcessTrusted() else { throw ComputerUseError.accessibilityDenied }
+
+        let app = try resolveApp(appName: appName, pid: requestedPID)
+        let pid = app.processIdentifier
+        let axApp = AXUIElementCreateApplication(pid)
+        if enableEnhancedAccessibility(axApp: axApp, pid: pid) {
+            // Chromium and Electron build the renderer AX tree lazily; give it a moment.
+            Thread.sleep(forTimeInterval: 0.35)
+        }
+        let axWindow = firstAXWindow(axApp: axApp, title: windowTitle)
+        let title = axWindow.flatMap { axString($0, kAXTitleAttribute) }
+        let info = firstCGWindowInfo(pid: pid, title: title)
+        let bounds = axWindow.flatMap(axFrame) ?? info?.bounds
+
+        guard let bounds, bounds.width > 20, bounds.height > 20 else {
+            throw ComputerUseError.noWindow(app.localizedName ?? appName ?? "frontmost app")
+        }
+
+        return WindowTarget(
+            appName: app.localizedName ?? "Unknown",
+            pid: pid,
+            windowNumber: info?.number,
+            windowTitle: title ?? info?.title,
+            bounds: bounds,
+            isFrontmost: NSWorkspace.shared.frontmostApplication?.processIdentifier == pid,
+            axWindow: axWindow
+        )
+    }
+
+    func snapshot(target: WindowTarget, strictMode: Bool, backgroundActivated: Bool) async throws -> AppSnapshot {
+        let records = records(target: target)
+        let (data, meta) = try await captureScreenshot(target: target)
+
+        return AppSnapshot(
+            id: "",
+            observation: 0,
+            appName: target.appName,
+            pid: target.pid,
+            windowNumber: target.windowNumber,
+            windowTitle: target.windowTitle,
+            screenshotData: data,
+            screenshotMimeType: "image/jpeg",
+            screenshotMeta: meta,
+            records: records,
+            strictMode: strictMode,
+            backgroundActivated: backgroundActivated,
+            recentActions: [],
+            addedLabels: [],
+            removedLabels: []
+        )
+    }
+
+    func records(target: WindowTarget) -> [AXElementRecord] {
+        target.axWindow.map(semanticRecords(window:)) ?? []
+    }
+
+    func press(record: AXElementRecord) -> Bool {
+        AXUIElementPerformAction(record.element, kAXPressAction as CFString) == .success
+    }
+
+    func focus(record: AXElementRecord) -> Bool {
+        AXUIElementSetAttributeValue(record.element, kAXFocusedAttribute as CFString, true as CFBoolean) == .success
+    }
+
+    func setValue(record: AXElementRecord, value: String) -> Bool {
+        var settable = DarwinBoolean(false)
+        guard AXUIElementIsAttributeSettable(record.element, kAXValueAttribute as CFString, &settable) == .success,
+              settable.boolValue else {
+            return false
+        }
+        return AXUIElementSetAttributeValue(record.element, kAXValueAttribute as CFString, value as CFString) == .success
+    }
+
+    func performAction(record: AXElementRecord, action: String) -> Bool {
+        AXUIElementPerformAction(record.element, action as CFString) == .success
+    }
+
+    func value(record: AXElementRecord) -> String? {
+        axString(record.element, kAXValueAttribute)
+    }
+
+    /// Chromium browsers and Electron apps only expose web content to AX after an
+    /// assistive client announces itself. VoiceOver does this by setting
+    /// AXEnhancedUserInterface; Electron additionally supports AXManualAccessibility.
+    /// Returns true when the attribute was newly applied to this pid.
+    private func enableEnhancedAccessibility(axApp: AXUIElement, pid: pid_t) -> Bool {
+        enhancedLock.lock()
+        let isNew = enhancedPIDs.insert(pid).inserted
+        enhancedLock.unlock()
+        guard isNew else { return false }
+        // Chromium reacts to the set attempt even though it reports an AXError
+        // (.notImplemented / .attributeUnsupported), so the return codes are
+        // logged for diagnostics but intentionally not treated as failure.
+        let enhanced = AXUIElementSetAttributeValue(axApp, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue)
+        let manual = AXUIElementSetAttributeValue(axApp, "AXManualAccessibility" as CFString, kCFBooleanTrue)
+        fputs("[ComputerUse] enhanced-ax pid=\(pid) AXEnhancedUserInterface=\(enhanced.rawValue) AXManualAccessibility=\(manual.rawValue)\n", stderr)
+        return true
+    }
+
+    private func resolveApp(appName: String?, pid: pid_t?) throws -> NSRunningApplication {
+        if let pid {
+            guard let app = NSRunningApplication(processIdentifier: pid) else {
+                throw ComputerUseError.appNotFound("pid \(pid)")
+            }
+            return app
+        }
+        guard let rawName = appName?.trimmingCharacters(in: .whitespacesAndNewlines), !rawName.isEmpty else {
+            guard let frontmost = NSWorkspace.shared.frontmostApplication else {
+                throw ComputerUseError.noFrontmostApplication
+            }
+            return frontmost
+        }
+
+        let needle = rawName.lowercased()
+        let regularApps = NSWorkspace.shared.runningApplications.filter { $0.activationPolicy == .regular }
+        if let exact = regularApps.first(where: { $0.localizedName?.lowercased() == needle }) {
+            return exact
+        }
+        if let contains = regularApps.first(where: { $0.localizedName?.lowercased().contains(needle) == true }) {
+            return contains
+        }
+        throw ComputerUseError.appNotFound(rawName)
+    }
+
+    private func firstAXWindow(axApp: AXUIElement, title preferredTitle: String?) -> AXUIElement? {
+        var windowValue: CFTypeRef?
+        guard AXUIElementCopyAttributeValue(axApp, kAXWindowsAttribute as CFString, &windowValue) == .success,
+              let windows = windowValue as? [AXUIElement] else {
+            return nil
+        }
+        let usable = windows.filter { window in
+            guard let frame = axFrame(window) else { return false }
+            return frame.width > 20 && frame.height > 20
+        }
+        if let preferredTitle, let match = usable.first(where: { axString($0, kAXTitleAttribute) == preferredTitle }) {
+            return match
+        }
+        return usable.first
+    }
+
+    private func semanticRecords(window: AXUIElement) -> [AXElementRecord] {
+        var records: [AXElementRecord] = []
+        collect(element: window, depth: 0, records: &records)
+        return records
+    }
+
+    private func collect(element: AXUIElement, depth: Int, records: inout [AXElementRecord]) {
+        guard depth <= maxDepth, records.count < maxElements else { return }
+
+        let rawRole = axString(element, kAXRoleAttribute) ?? "AXUnknown"
+        let role = normalizedRole(rawRole)
+        let value = axString(element, kAXValueAttribute).map { String($0.prefix(120)) }
+        let label = semanticLabel(element: element, role: role, value: value)
+        let actions = axActions(element)
+        let frame = axFrame(element)
+        let capabilities = capabilitiesFor(element: element, rawRole: rawRole, actions: actions)
+        let shouldSurface = shouldSurfaceElement(rawRole: rawRole, label: label, value: value, frame: frame, capabilities: capabilities)
+
+        if shouldSurface, let frame {
+            let id = records.count + 1
+            let semantic = SemanticAXElement(
+                id: id,
+                ref: "{e\(id)}",
+                role: role,
+                label: label,
+                value: value,
+                frame: ElementFrame(
+                    x: Int(frame.origin.x),
+                    y: Int(frame.origin.y),
+                    width: Int(frame.width),
+                    height: Int(frame.height)
+                ),
+                state: stateFor(element: element, rawRole: rawRole),
+                capabilities: capabilities
+            )
+            records.append(AXElementRecord(element: element, semantic: semantic))
+        }
+
+        for child in axChildren(element) {
+            collect(element: child, depth: depth + 1, records: &records)
+            if records.count >= maxElements { break }
+        }
+    }
+
+    private func shouldSurfaceElement(rawRole: String, label: String, value: String?, frame: CGRect?, capabilities: AXElementCapabilities) -> Bool {
+        guard let frame, frame.width > 1, frame.height > 1 else { return false }
+        let hasSemanticText = !label.isEmpty || value?.isEmpty == false
+        let interactive = capabilities.canPress || capabilities.canFocus || capabilities.canScroll || capabilities.canAdjust || capabilities.canSetValue
+        if interactive { return true }
+        if !importantRoles.contains(rawRole) { return false }
+        if rawRole == "AXGroup" { return hasSemanticText && frame.width < 900 && frame.height < 700 }
+        return hasSemanticText
+    }
+
+    private func semanticLabel(element: AXUIElement, role: String, value: String?) -> String {
+        let candidates = [
+            axString(element, kAXTitleAttribute),
+            axString(element, kAXDescriptionAttribute),
+            axString(element, kAXHelpAttribute),
+            axString(element, kAXIdentifierAttribute),
+            value,
+        ]
+        for candidate in candidates {
+            if let trimmed = candidate?.trimmingCharacters(in: .whitespacesAndNewlines), !trimmed.isEmpty {
+                return String(trimmed.prefix(120))
+            }
+        }
+        return role
+    }
+
+    private func capabilitiesFor(element: AXUIElement, rawRole: String, actions: [String]) -> AXElementCapabilities {
+        var valueSettable = DarwinBoolean(false)
+        let canSetValue = AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &valueSettable) == .success && valueSettable.boolValue
+
+        var focusSettable = DarwinBoolean(false)
+        let canFocus = AXUIElementIsAttributeSettable(element, kAXFocusedAttribute as CFString, &focusSettable) == .success && focusSettable.boolValue
+
+        let canAdjust = actions.contains(kAXIncrementAction) || actions.contains(kAXDecrementAction) || rawRole == "AXSlider" || rawRole == "AXIncrementor"
+        let canScroll = actions.contains("AXScrollToVisible") || rawRole == "AXScrollArea" || rawRole == "AXScrollBar"
+        let canPress = actions.contains(kAXPressAction) || ["AXButton", "AXCheckBox", "AXRadioButton", "AXLink", "AXMenuItem", "AXPopUpButton", "AXMenuButton", "AXCell"].contains(rawRole)
+
+        return AXElementCapabilities(
+            canPress: canPress,
+            canFocus: canFocus,
+            canScroll: canScroll,
+            canAdjust: canAdjust,
+            canSetValue: canSetValue,
+            actions: actions
+        )
+    }
+
+    private func stateFor(element: AXUIElement, rawRole: String) -> AXElementState {
+        AXElementState(
+            enabled: axBool(element, kAXEnabledAttribute),
+            focused: axBool(element, kAXFocusedAttribute),
+            selected: axBool(element, kAXSelectedAttribute),
+            expanded: axBool(element, kAXExpandedAttribute),
+            checked: rawRole == "AXCheckBox" || rawRole == "AXRadioButton" ? axBool(element, kAXValueAttribute) : nil
+        )
+    }
+
+    private func axChildren(_ element: AXUIElement) -> [AXUIElement] {
+        var value: CFTypeRef?
+        guard AXUIElementCopyAttributeValue(element, kAXChildrenAttribute as CFString, &value) == .success,
+              let children = value as? [AXUIElement] else {
+            return []
+        }
+        return children
+    }
+
+    private func axActions(_ element: AXUIElement) -> [String] {
+        var actionNames: CFArray?
+        guard AXUIElementCopyActionNames(element, &actionNames) == .success,
+              let names = actionNames as? [String] else {
+            return []
+        }
+        return names
+    }
+
+    private func axString(_ element: AXUIElement, _ attribute: String) -> String? {
+        var value: CFTypeRef?
+        guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success,
+              let value else {
+            return nil
+        }
+        if let string = value as? String, !string.isEmpty { return string }
+        if let attributed = value as? NSAttributedString, !attributed.string.isEmpty { return attributed.string }
+        if let number = value as? NSNumber { return number.stringValue }
+        return nil
+    }
+
+    private func axBool(_ element: AXUIElement, _ attribute: String) -> Bool? {
+        var value: CFTypeRef?
+        guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success,
+              let value else {
+            return nil
+        }
+        if let bool = value as? Bool { return bool }
+        if let number = value as? NSNumber { return number.boolValue }
+        return nil
+    }
+
+    private func axFrame(_ element: AXUIElement) -> CGRect? {
+        guard let position = axPoint(element, kAXPositionAttribute), let size = axSize(element, kAXSizeAttribute) else {
+            return nil
+        }
+        return CGRect(origin: position, size: size)
+    }
+
+    private func axPoint(_ element: AXUIElement, _ attribute: String) -> CGPoint? {
+        var value: CFTypeRef?
+        guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success,
+              let value,
+              CFGetTypeID(value) == AXValueGetTypeID() else {
+            return nil
+        }
+        let axValue = value as! AXValue
+        guard AXValueGetType(axValue) == .cgPoint else { return nil }
+        var point = CGPoint.zero
+        guard AXValueGetValue(axValue, .cgPoint, &point) else { return nil }
+        return point
+    }
+
+    private func axSize(_ element: AXUIElement, _ attribute: String) -> CGSize? {
+        var value: CFTypeRef?
+        guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success,
+              let value,
+              CFGetTypeID(value) == AXValueGetTypeID() else {
+            return nil
+        }
+        let axValue = value as! AXValue
+        guard AXValueGetType(axValue) == .cgSize else { return nil }
+        var size = CGSize.zero
+        guard AXValueGetValue(axValue, .cgSize, &size) else { return nil }
+        return size
+    }
+
+    private func normalizedRole(_ rawRole: String) -> String {
+        rawRole.hasPrefix("AX") ? String(rawRole.dropFirst(2)) : rawRole
+    }
+
+    private func firstCGWindowInfo(pid: pid_t, title: String?) -> (number: Int, title: String?, bounds: CGRect)? {
+        guard let list = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else {
+            return nil
+        }
+
+        let candidates = list.compactMap { info -> (number: Int, title: String?, bounds: CGRect)? in
+            guard let ownerPID = info[kCGWindowOwnerPID as String] as? Int32,
+                  ownerPID == pid,
+                  let layer = info[kCGWindowLayer as String] as? Int,
+                  layer == 0,
+                  let number = info[kCGWindowNumber as String] as? Int,
+                  let boundsDict = info[kCGWindowBounds as String] as? [String: Any] else {
+                return nil
+            }
+            let bounds = CGRect(
+                x: cgFloat(boundsDict["X"]),
+                y: cgFloat(boundsDict["Y"]),
+                width: cgFloat(boundsDict["Width"]),
+                height: cgFloat(boundsDict["Height"])
+            )
+            guard bounds.width > 20, bounds.height > 20 else { return nil }
+            return (number, info[kCGWindowName as String] as? String, bounds)
+        }
+
+        if let title, let exact = candidates.first(where: { $0.title == title }) {
+            return exact
+        }
+        return candidates.first
+    }
+
+    private func captureScreenshot(target: WindowTarget) async throws -> (Data, ScreenshotMetadata) {
+        let cgImage = await screenCaptureKitImage(target: target) ?? legacyCaptureImage(target: target)
+        guard let cgImage else { throw ComputerUseError.screenshotFailed }
+
+        return try encodeScreenshot(cgImage, capturedBounds: target.bounds)
+    }
+
+    private func legacyCaptureImage(target: WindowTarget) -> CGImage? {
+        if let windowNumber = target.windowNumber {
+            return CGWindowListCreateImage(
+                CGRect.null,
+                .optionIncludingWindow,
+                CGWindowID(windowNumber),
+                [.bestResolution, .boundsIgnoreFraming]
+            )
+        }
+
+        return CGWindowListCreateImage(target.bounds, .optionOnScreenOnly, kCGNullWindowID, [.bestResolution])
+    }
+
+    private func screenCaptureKitImage(target: WindowTarget) async -> CGImage? {
+        guard let windowNumber = target.windowNumber else { return nil }
+        do {
+            let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)
+            guard let window = content.windows.first(where: { Int($0.windowID) == windowNumber }) else { return nil }
+            let configuration = SCStreamConfiguration()
+            let scale = screenScale(for: target.bounds)
+            configuration.width = max(1, Int(target.bounds.width * scale))
+            configuration.height = max(1, Int(target.bounds.height * scale))
+            configuration.showsCursor = false
+            let filter = SCContentFilter(desktopIndependentWindow: window)
+            return try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: configuration)
+        } catch {
+            return nil
+        }
+    }
+
+    private func screenScale(for bounds: CGRect) -> CGFloat {
+        NSScreen.screens.first(where: { $0.frame.intersects(bounds) })?.backingScaleFactor
+            ?? NSScreen.main?.backingScaleFactor
+            ?? 2
+    }
+
+    private func encodeScreenshot(_ cgImage: CGImage, capturedBounds: CGRect) throws -> (Data, ScreenshotMetadata) {
+        let rawWidth = CGFloat(cgImage.width)
+        let rawHeight = CGFloat(cgImage.height)
+        let targetWidth = min(screenshotImageWidth, rawWidth)
+        let targetHeight = rawHeight * (targetWidth / rawWidth)
+
+        let source = NSImage(cgImage: cgImage, size: NSSize(width: rawWidth, height: rawHeight))
+        let resized = NSImage(size: NSSize(width: targetWidth, height: targetHeight))
+        resized.lockFocus()
+        source.draw(in: NSRect(x: 0, y: 0, width: targetWidth, height: targetHeight))
+        resized.unlockFocus()
+
+        guard let tiff = resized.tiffRepresentation,
+              let rep = NSBitmapImageRep(data: tiff),
+              let jpeg = rep.representation(using: .jpeg, properties: [.compressionFactor: 0.45]) else {
+            throw ComputerUseError.screenshotFailed
+        }
+
+        return (
+            jpeg,
+            ScreenshotMetadata(
+                imageWidth: Int(targetWidth),
+                imageHeight: Int(targetHeight),
+                capturedBounds: capturedBounds
+            )
+        )
+    }
+
+    private func cgFloat(_ value: Any?) -> CGFloat {
+        if let value = value as? CGFloat { return value }
+        if let value = value as? Double { return CGFloat(value) }
+        if let value = value as? Int { return CGFloat(value) }
+        if let value = value as? NSNumber { return CGFloat(truncating: value) }
+        return 0
+    }
+}

packages/handsfree/native/HandsFree/Sources/ComputerUse/AgentCursorOverlay.swift

@@ -0,0 +1,104 @@
+import AppKit
+
+final class AgentCursorOverlay: @unchecked Sendable {
+    static let shared = AgentCursorOverlay()
+
+    private var panel: AgentCursorPanel?
+    private var hideWorkItem: DispatchWorkItem?
+    private let size = CGSize(width: 30, height: 34)
+
+    func show(at point: CGPoint) {
+        DispatchQueue.main.async { [weak self] in
+            self?.showOnMain(at: point)
+        }
+    }
+
+    private func showOnMain(at point: CGPoint) {
+        let panel = ensurePanel()
+        panel.setFrame(frame(for: point), display: true)
+        panel.orderFrontRegardless()
+
+        hideWorkItem?.cancel()
+        let nextHide = DispatchWorkItem { [weak panel] in
+            panel?.orderOut(nil)
+        }
+        hideWorkItem = nextHide
+        DispatchQueue.main.asyncAfter(deadline: .now() + 1.5, execute: nextHide)
+    }
+
+    private func ensurePanel() -> AgentCursorPanel {
+        if let panel { return panel }
+
+        let panel = AgentCursorPanel(
+            contentRect: NSRect(origin: .zero, size: size),
+            styleMask: [.borderless, .nonactivatingPanel],
+            backing: .buffered,
+            defer: false
+        )
+        panel.backgroundColor = .clear
+        panel.isOpaque = false
+        panel.hasShadow = false
+        panel.ignoresMouseEvents = true
+        panel.level = .floating
+        panel.collectionBehavior = [.canJoinAllSpaces, .stationary, .ignoresCycle, .fullScreenAuxiliary]
+        panel.contentView = AgentCursorView(frame: NSRect(origin: .zero, size: size))
+        self.panel = panel
+        return panel
+    }
+
+    private func frame(for topLeftPoint: CGPoint) -> NSRect {
+        let screen = screenContaining(topLeftPoint) ?? NSScreen.main
+        let screenFrame = screen?.frame ?? NSRect(x: 0, y: 0, width: 1440, height: 900)
+        let x = screenFrame.minX + topLeftPoint.x - 2
+        let y = screenFrame.maxY - topLeftPoint.y - size.height + 2
+        return NSRect(x: x, y: y, width: size.width, height: size.height)
+    }
+
+    private func screenContaining(_ topLeftPoint: CGPoint) -> NSScreen? {
+        NSScreen.screens.first { screen in
+            let frame = screen.frame
+            return topLeftPoint.x >= frame.minX
+                && topLeftPoint.x <= frame.maxX
+                && topLeftPoint.y >= 0
+                && topLeftPoint.y <= frame.height
+        }
+    }
+}
+
+final class AgentCursorPanel: NSPanel {
+    override var canBecomeKey: Bool { false }
+    override var canBecomeMain: Bool { false }
+}
+
+final class AgentCursorView: NSView {
+    override var isOpaque: Bool { false }
+
+    override func draw(_ dirtyRect: NSRect) {
+        super.draw(dirtyRect)
+
+        let shadow = NSShadow()
+        shadow.shadowBlurRadius = 3
+        shadow.shadowOffset = NSSize(width: 0, height: -1)
+        shadow.shadowColor = NSColor.black.withAlphaComponent(0.35)
+        shadow.set()
+
+        let path = NSBezierPath()
+        path.move(to: NSPoint(x: 4, y: 30))
+        path.line(to: NSPoint(x: 4, y: 4))
+        path.line(to: NSPoint(x: 12, y: 11))
+        path.line(to: NSPoint(x: 17, y: 1))
+        path.line(to: NSPoint(x: 22, y: 4))
+        path.line(to: NSPoint(x: 17, y: 14))
+        path.line(to: NSPoint(x: 27, y: 14))
+        path.close()
+
+        NSColor.white.setFill()
+        path.fill()
+
+        shadow.shadowColor = .clear
+        shadow.set()
+        NSColor.systemBlue.setStroke()
+        path.lineWidth = 2
+        path.stroke()
+    }
+}

packages/handsfree/native/HandsFree/Sources/ComputerUse/BackgroundActivationSession.swift

@@ -0,0 +1,164 @@
+import AppKit
+import CoreGraphics
+
+final class BackgroundActivationSession: @unchecked Sendable {
+    private final class TapContext {
+        let suppressFocusMessages: Bool
+
+        init(suppressFocusMessages: Bool) {
+            self.suppressFocusMessages = suppressFocusMessages
+        }
+    }
+
+    private struct TapRef {
+        let tap: CFMachPort
+        let source: CFRunLoopSource
+        let context: TapContext
+    }
+
+    private let previousPID: pid_t
+    private let targetPID: pid_t
+    private var taps: [TapRef] = []
+    private var runLoop: CFRunLoop?
+    private var thread: Thread?
+    private var startupError: Error?
+    private var started = false
+    private var lastTarget: WindowTarget?
+
+    init(previousPID: pid_t, targetPID: pid_t) {
+        self.previousPID = previousPID
+        self.targetPID = targetPID
+    }
+
+    deinit {
+        stop()
+    }
+
+    func start() throws {
+        guard !started else { return }
+
+        let ready = DispatchSemaphore(value: 0)
+        let thread = Thread { [weak self] in
+            guard let self else {
+                ready.signal()
+                return
+            }
+            self.runLoop = CFRunLoopGetCurrent()
+            do {
+                try self.installTapsOnCurrentRunLoop()
+            } catch {
+                self.startupError = error
+            }
+            ready.signal()
+            if self.startupError == nil {
+                CFRunLoopRun()
+            }
+        }
+        thread.name = "OpenWorkBackgroundActivationSession"
+        self.thread = thread
+        thread.start()
+        ready.wait()
+
+        if let startupError {
+            throw startupError
+        }
+        started = true
+    }
+
+    func activate(target: WindowTarget) async throws {
+        guard let windowNumber = target.windowNumber else {
+            throw ComputerUseError.strictModeViolation("target window has no CG window number")
+        }
+
+        postAppKitDefined(subtype: 1, target: target, windowNumber: windowNumber)
+        try await Task.sleep(nanoseconds: 25_000_000)
+        try await BackgroundInputDispatcher.click(pid: target.pid, windowNumber: windowNumber, point: target.center)
+        try await Task.sleep(nanoseconds: 80_000_000)
+        lastTarget = target
+    }
+
+    func stop() {
+        if let target = lastTarget, let windowNumber = target.windowNumber {
+            postAppKitDefined(subtype: 2, target: target, windowNumber: windowNumber)
+        }
+        for tapRef in taps {
+            CFMachPortInvalidate(tapRef.tap)
+        }
+        taps.removeAll()
+        if let runLoop {
+            CFRunLoopStop(runLoop)
+        }
+        runLoop = nil
+        thread = nil
+        started = false
+    }
+
+    private func installTapsOnCurrentRunLoop() throws {
+        if previousPID != targetPID {
+            taps.append(try installTap(pid: previousPID, suppressFocusMessages: true))
+        }
+        taps.append(try installTap(pid: targetPID, suppressFocusMessages: false))
+    }
+
+    private func installTap(pid: pid_t, suppressFocusMessages: Bool) throws -> TapRef {
+        let context = TapContext(suppressFocusMessages: suppressFocusMessages)
+        let userInfo = Unmanaged.passUnretained(context).toOpaque()
+        guard let tap = CGEvent.tapCreateForPid(
+            pid: pid,
+            place: .headInsertEventTap,
+            options: .defaultTap,
+            eventsOfInterest: CGEventMask.max,
+            callback: BackgroundActivationSession.eventTapCallback,
+            userInfo: userInfo
+        ) else {
+            throw ComputerUseError.strictModeViolation("could not install per-process event tap for pid \(pid)")
+        }
+
+        guard let source = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, tap, 0) else {
+            CFMachPortInvalidate(tap)
+            throw ComputerUseError.strictModeViolation("could not create run loop source for pid \(pid) event tap")
+        }
+
+        CFRunLoopAddSource(CFRunLoopGetCurrent(), source, .commonModes)
+        CGEvent.tapEnable(tap: tap, enable: true)
+        return TapRef(tap: tap, source: source, context: context)
+    }
+
+    private func postAppKitDefined(subtype: Int16, target: WindowTarget, windowNumber: Int) {
+        guard let event = NSEvent.otherEvent(
+            with: .appKitDefined,
+            location: target.center,
+            modifierFlags: [],
+            timestamp: ProcessInfo.processInfo.systemUptime,
+            windowNumber: windowNumber,
+            context: nil,
+            subtype: subtype,
+            data1: 0,
+            data2: 0
+        )?.cgEvent else {
+            return
+        }
+
+        BackgroundInputDispatcher.address(event, pid: target.pid, windowNumber: windowNumber)
+        event.postToPid(target.pid)
+    }
+
+    private static let eventTapCallback: CGEventTapCallBack = { _, type, event, userInfo in
+        if type == .tapDisabledByTimeout || type == .tapDisabledByUserInput {
+            return Unmanaged.passUnretained(event)
+        }
+        guard let userInfo else {
+            return Unmanaged.passUnretained(event)
+        }
+        let context = Unmanaged<TapContext>.fromOpaque(userInfo).takeUnretainedValue()
+        if context.suppressFocusMessages && BackgroundActivationSession.isFocusMessage(type) {
+            return nil
+        }
+        return Unmanaged.passUnretained(event)
+    }
+
+    private static func isFocusMessage(_ type: CGEventType) -> Bool {
+        let raw = Int(type.rawValue)
+        return raw == 13 || raw == 19 || raw == 20
+    }
+}

packages/handsfree/native/HandsFree/Sources/ComputerUse/BackgroundInputDispatcher.swift

@@ -0,0 +1,163 @@
+import Foundation
+import CoreGraphics
+
+enum BackgroundInputDispatcher {
+    private static let privateWindowField = CGEventField(rawValue: 51)
+    private static let privateRouteField = CGEventField(rawValue: 58)
+
+    static func click(pid: pid_t, windowNumber: Int, point: CGPoint, doubleClick: Bool = false) async throws {
+        guard let source = CGEventSource(stateID: .combinedSessionState) else {
+            throw ComputerUseError.eventSourceFailed
+        }
+
+        let clickCount = doubleClick ? 2 : 1
+        for clickState in 1...clickCount {
+            guard let down = CGEvent(mouseEventSource: source, mouseType: .leftMouseDown, mouseCursorPosition: point, mouseButton: .left),
+                  let up = CGEvent(mouseEventSource: source, mouseType: .leftMouseUp, mouseCursorPosition: point, mouseButton: .left) else {
+                throw ComputerUseError.eventCreationFailed
+            }
+
+            address(down, pid: pid, windowNumber: windowNumber)
+            down.setIntegerValueField(.mouseEventClickState, value: Int64(clickState))
+            down.setDoubleValueField(.mouseEventPressure, value: 1)
+            down.postToPid(pid)
+
+            try await Task.sleep(nanoseconds: 30_000_000)
+
+            address(up, pid: pid, windowNumber: windowNumber)
+            up.setIntegerValueField(.mouseEventClickState, value: Int64(clickState))
+            up.setDoubleValueField(.mouseEventPressure, value: 0)
+            up.postToPid(pid)
+
+            if clickState < clickCount {
+                try await Task.sleep(nanoseconds: 50_000_000)
+            }
+        }
+    }
+
+    static func scroll(pid: pid_t, windowNumber: Int, point: CGPoint, deltaX: Int32, deltaY: Int32) throws {
+        guard let source = CGEventSource(stateID: .combinedSessionState) else {
+            throw ComputerUseError.eventSourceFailed
+        }
+        guard let event = CGEvent(scrollWheelEvent2Source: source, units: .line, wheelCount: 2, wheel1: deltaY, wheel2: deltaX, wheel3: 0) else {
+            throw ComputerUseError.eventCreationFailed
+        }
+        event.location = point
+        address(event, pid: pid, windowNumber: windowNumber)
+        event.postToPid(pid)
+    }
+
+    static func typeText(pid: pid_t, text: String) throws {
+        guard let source = CGEventSource(stateID: .combinedSessionState) else {
+            throw ComputerUseError.eventSourceFailed
+        }
+
+        let units = Array(text.utf16)
+        let chunkSize = 20
+        for start in stride(from: 0, to: units.count, by: chunkSize) {
+            let end = min(start + chunkSize, units.count)
+            let chunk = Array(units[start..<end])
+            guard let event = CGEvent(keyboardEventSource: source, virtualKey: 0, keyDown: true) else {
+                throw ComputerUseError.eventCreationFailed
+            }
+            event.setIntegerValueField(.eventTargetUnixProcessID, value: Int64(pid))
+            event.keyboardSetUnicodeString(stringLength: chunk.count, unicodeString: chunk)
+            event.postToPid(pid)
+            Thread.sleep(forTimeInterval: 0.01)
+        }
+    }
+
+    static func pressKey(pid: pid_t, combo: String) throws {
+        let parsed = try parseCombo(combo)
+        guard let source = CGEventSource(stateID: .combinedSessionState) else {
+            throw ComputerUseError.eventSourceFailed
+        }
+
+        // Post real modifier key events around the main key. Chromium and other
+        // apps ignore bare flag bits on background-posted events, so flags alone
+        // are not enough for combos like command+a.
+        let modifierKeys: [(flag: CGEventFlags, keyCode: CGKeyCode)] = [
+            (.maskCommand, 0x37), (.maskShift, 0x38), (.maskAlternate, 0x3A), (.maskControl, 0x3B),
+        ]
+        let activeModifiers = modifierKeys.filter { parsed.flags.contains($0.flag) }
+
+        var accumulated: CGEventFlags = []
+        for modifier in activeModifiers {
+            accumulated.insert(modifier.flag)
+            try postKey(source: source, pid: pid, keyCode: modifier.keyCode, keyDown: true, flags: accumulated)
+            Thread.sleep(forTimeInterval: 0.005)
+        }
+
+        try postKey(source: source, pid: pid, keyCode: parsed.keyCode, keyDown: true, flags: parsed.flags)
+        Thread.sleep(forTimeInterval: 0.01)
+        try postKey(source: source, pid: pid, keyCode: parsed.keyCode, keyDown: false, flags: parsed.flags)
+
+        for modifier in activeModifiers.reversed() {
+            accumulated.remove(modifier.flag)
+            Thread.sleep(forTimeInterval: 0.005)
+            try postKey(source: source, pid: pid, keyCode: modifier.keyCode, keyDown: false, flags: accumulated)
+        }
+    }
+
+    private static func postKey(source: CGEventSource, pid: pid_t, keyCode: CGKeyCode, keyDown: Bool, flags: CGEventFlags) throws {
+        guard let event = CGEvent(keyboardEventSource: source, virtualKey: keyCode, keyDown: keyDown) else {
+            throw ComputerUseError.eventCreationFailed
+        }
+        event.flags = flags
+        event.setIntegerValueField(.eventTargetUnixProcessID, value: Int64(pid))
+        event.postToPid(pid)
+    }
+
+    static func address(_ event: CGEvent, pid: pid_t, windowNumber: Int) {
+        event.setIntegerValueField(.eventTargetUnixProcessID, value: Int64(pid))
+        event.setIntegerValueField(.mouseEventWindowUnderMousePointer, value: Int64(windowNumber))
+        event.setIntegerValueField(.mouseEventWindowUnderMousePointerThatCanHandleThisEvent, value: Int64(windowNumber))
+        if let privateWindowField {
+            event.setIntegerValueField(privateWindowField, value: Int64(windowNumber))
+        }
+        if let privateRouteField {
+            event.setIntegerValueField(privateRouteField, value: 1)
+        }
+    }
+
+    static func parseCombo(_ combo: String) throws -> (flags: CGEventFlags, keyCode: CGKeyCode) {
+        let parts = combo.lowercased().split(separator: "+").map(String.init)
+        var flags: CGEventFlags = []
+        var keyName = ""
+
+        for part in parts {
+            switch part {
+            case "command", "cmd", "meta": flags.insert(.maskCommand)
+            case "shift": flags.insert(.maskShift)
+            case "control", "ctrl": flags.insert(.maskControl)
+            case "option", "alt": flags.insert(.maskAlternate)
+            default: keyName = part
+            }
+        }
+
+        guard let keyCode = keyCodes[keyName] else {
+            throw ComputerUseError.unknownKey(keyName)
+        }
+        return (flags, keyCode)
+    }
+
+    private static let keyCodes: [String: CGKeyCode] = [
+        "return": 0x24, "enter": 0x24, "tab": 0x30, "space": 0x31,
+        "delete": 0x33, "backspace": 0x33, "escape": 0x35, "esc": 0x35,
+        "up": 0x7E, "down": 0x7D, "left": 0x7B, "right": 0x7C,
+        "home": 0x73, "end": 0x77, "pageup": 0x74, "pagedown": 0x79,
+        "a": 0x00, "b": 0x0B, "c": 0x08, "d": 0x02, "e": 0x0E,
+        "f": 0x03, "g": 0x05, "h": 0x04, "i": 0x22, "j": 0x26,
+        "k": 0x28, "l": 0x25, "m": 0x2E, "n": 0x2D, "o": 0x1F,
+        "p": 0x23, "q": 0x0C, "r": 0x0F, "s": 0x01, "t": 0x11,
+        "u": 0x20, "v": 0x09, "w": 0x0D, "x": 0x07, "y": 0x10, "z": 0x06,
+        "0": 0x1D, "1": 0x12, "2": 0x13, "3": 0x14, "4": 0x15,
+        "5": 0x17, "6": 0x16, "7": 0x1A, "8": 0x1C, "9": 0x19,
+        "f1": 0x7A, "f2": 0x78, "f3": 0x63, "f4": 0x76,
+        "f5": 0x60, "f6": 0x61, "f7": 0x62, "f8": 0x64,
+        "f9": 0x65, "f10": 0x6D, "f11": 0x67, "f12": 0x6F,
+        "-": 0x1B, "=": 0x18, "[": 0x21, "]": 0x1E,
+        "\\": 0x2A, ";": 0x29, "'": 0x27, ",": 0x2B,
+        ".": 0x2F, "/": 0x2C, "`": 0x32,
+    ]
+}

packages/handsfree/native/HandsFree/Sources/ComputerUse/ComputerUseRuntime.swift

@@ -0,0 +1,383 @@
+import AppKit
+import Foundation
+
+actor ComputerUseRuntime {
+    private let accessibility = AccessibilityService()
+    private let foregroundInput = InputService()
+    private var lastSnapshot: AppSnapshot?
+    private var strictMode = true
+    private var activationSession: BackgroundActivationSession?
+    private var activationKey: String?
+    private var activatedWindowKey: String?
+    private var activationPreviousPID: pid_t?
+    private var activationTargetPID: pid_t?
+    private var frontmostMonitor: FrontmostApplicationMonitor?
+    private var snapshotSequence = 0
+    private var previousLabelsByWindow: [String: Set<String>] = [:]
+    private var recentActions: [String] = []
+    private var staleSnapshotReason: String?
+
+    func setStrictMode(_ enabled: Bool) -> ActionMetadata {
+        strictMode = enabled
+        if !enabled {
+            resetBackgroundActivation()
+        }
+        return ActionMetadata(
+            ok: true,
+            path: .none,
+            strictMode: enabled,
+            backgroundSafe: enabled,
+            fallbackUsed: false,
+            message: enabled ? "Strict background mode enabled." : "Strict background mode disabled. Foreground fallback is allowed."
+        )
+    }
+
+    func snapshot(appName: String?, pid: Int? = nil, windowTitle: String? = nil, strict requestedStrict: Bool?) async throws -> AppSnapshot {
+        let effectiveStrict = requestedStrict ?? strictMode
+        if !effectiveStrict {
+            resetBackgroundActivation()
+        }
+
+        let requestedPID = pid.map(pid_t.init)
+        var target = try accessibility.resolveTarget(appName: appName, pid: requestedPID, windowTitle: windowTitle)
+        let backgroundActivated: Bool
+        if effectiveStrict, !target.isFrontmost {
+            backgroundActivated = try await ensureBackgroundActivation(target: target)
+            target = try accessibility.resolveTarget(appName: appName, pid: requestedPID, windowTitle: windowTitle)
+        } else {
+            backgroundActivated = false
+        }
+
+        let snapshot = try await accessibility.snapshot(
+            target: target,
+            strictMode: effectiveStrict,
+            backgroundActivated: backgroundActivated
+        )
+        let annotatedSnapshot = annotate(snapshot: snapshot)
+        staleSnapshotReason = nil
+        lastSnapshot = annotatedSnapshot
+        return annotatedSnapshot
+    }
+
+    func click(snapshotID: String?, ref: String?, index: Int?, imageX: Double?, imageY: Double?, screenX: Double? = nil, screenY: Double? = nil, clickCount: Int, strict requestedStrict: Bool?) async throws -> ActionMetadata {
+        let snapshot = try requireSnapshot(snapshotID: snapshotID)
+        let effectiveStrict = requestedStrict ?? snapshot.strictMode
+        try validateSnapshotForAction(snapshot: snapshot, strict: effectiveStrict)
+
+        if let record = findRecord(ref: ref, index: index, in: snapshot) {
+            let fresh = freshRecord(matching: record, in: snapshot)
+            if let fresh {
+                AgentCursorOverlay.shared.show(at: fresh.semantic.frame.center)
+                if fresh.semantic.capabilities.canPress, accessibility.press(record: fresh) {
+                    return recordAction(ActionMetadata(ok: true, path: .accessibility, strictMode: effectiveStrict, backgroundSafe: true, fallbackUsed: false, message: "Pressed \(fresh.semantic.ref) via AXPress."))
+                }
+                if fresh.semantic.capabilities.canFocus, accessibility.focus(record: fresh) {
+                    return recordAction(ActionMetadata(ok: true, path: .accessibility, strictMode: effectiveStrict, backgroundSafe: true, fallbackUsed: false, message: "Focused \(fresh.semantic.ref) via AX."))
+                }
+                return try await clickPoint(fresh.semantic.frame.center, clickCount: clickCount, strict: effectiveStrict, fallbackUsed: true)
+            }
+            return try await clickPoint(record.semantic.frame.center, clickCount: clickCount, strict: effectiveStrict, fallbackUsed: true)
+        }
+
+        if let screenX, let screenY {
+            return try await clickPoint(CGPoint(x: screenX, y: screenY), clickCount: clickCount, strict: effectiveStrict, fallbackUsed: false)
+        }
+
+        if let imageX, let imageY {
+            let point = snapshot.screenshotMeta.toScreen(imageX: imageX, imageY: imageY)
+            return try await clickPoint(point, clickCount: clickCount, strict: effectiveStrict, fallbackUsed: false)
+        }
+
+        throw ComputerUseError.invalidElement(ref ?? index.map(String.init) ?? "<missing>")
+    }
+
+    func typeText(snapshotID: String?, text: String, strict requestedStrict: Bool?) throws -> ActionMetadata {
+        let snapshot = try requireSnapshot(snapshotID: snapshotID)
+        let effectiveStrict = requestedStrict ?? snapshot.strictMode
+        try validateSnapshotForAction(snapshot: snapshot, strict: effectiveStrict)
+        if effectiveStrict {
+            try BackgroundInputDispatcher.typeText(pid: snapshot.pid, text: text)
+            return recordAction(ActionMetadata(ok: true, path: .backgroundCGEvent, strictMode: true, backgroundSafe: true, fallbackUsed: false, message: "Typed text with postToPid."))
+        }
+
+        try foregroundInput.typeText(text)
+        return recordAction(ActionMetadata(ok: true, path: .foregroundCGEvent, strictMode: false, backgroundSafe: false, fallbackUsed: true, message: "Typed text with foreground HID fallback."))
+    }
+
+    func pressKey(snapshotID: String?, combo: String, strict requestedStrict: Bool?) throws -> ActionMetadata {
+        let snapshot = try requireSnapshot(snapshotID: snapshotID)
+        let effectiveStrict = requestedStrict ?? snapshot.strictMode
+        try validateSnapshotForAction(snapshot: snapshot, strict: effectiveStrict)
+        if effectiveStrict {
+            try BackgroundInputDispatcher.pressKey(pid: snapshot.pid, combo: combo)
+            return recordAction(ActionMetadata(ok: true, path: .backgroundCGEvent, strictMode: true, backgroundSafe: true, fallbackUsed: false, message: "Pressed key with postToPid."))
+        }
+
+        try foregroundInput.pressKey(combo)
+        return recordAction(ActionMetadata(ok: true, path: .foregroundCGEvent, strictMode: false, backgroundSafe: false, fallbackUsed: true, message: "Pressed key with foreground HID fallback."))
+    }
+
+    func scroll(snapshotID: String?, direction: String?, pages: Double, imageX: Double?, imageY: Double?, strict requestedStrict: Bool?) throws -> ActionMetadata {
+        let snapshot = try requireSnapshot(snapshotID: snapshotID)
+        let effectiveStrict = requestedStrict ?? snapshot.strictMode
+        try validateSnapshotForAction(snapshot: snapshot, strict: effectiveStrict)
+        let amount = max(1, Int32(pages * 5))
+        let deltas = scrollDeltas(direction: direction, amount: amount)
+        let point: CGPoint = {
+            if let imageX, let imageY {
+                return snapshot.screenshotMeta.toScreen(imageX: imageX, imageY: imageY)
+            }
+            return CGPoint(x: snapshot.screenshotMeta.capturedBounds.midX, y: snapshot.screenshotMeta.capturedBounds.midY)
+        }()
+        AgentCursorOverlay.shared.show(at: point)
+
+        if effectiveStrict {
+            guard let windowNumber = snapshot.windowNumber else {
+                throw ComputerUseError.strictModeViolation("background scroll requires a CG window number")
+            }
+            try BackgroundInputDispatcher.scroll(pid: snapshot.pid, windowNumber: windowNumber, point: point, deltaX: deltas.x, deltaY: deltas.y)
+            return recordAction(ActionMetadata(ok: true, path: .backgroundCGEvent, strictMode: true, backgroundSafe: true, fallbackUsed: false, message: "Scrolled with postToPid."))
+        }
+
+        try foregroundInput.scroll(point: point, deltaX: deltas.x, deltaY: deltas.y)
+        return recordAction(ActionMetadata(ok: true, path: .foregroundCGEvent, strictMode: false, backgroundSafe: false, fallbackUsed: true, message: "Scrolled with foreground HID fallback."))
+    }
+
+    func setValue(snapshotID: String?, ref: String?, index: Int?, value: String) throws -> ActionMetadata {
+        let snapshot = try requireSnapshot(snapshotID: snapshotID)
+        try validateSnapshotForAction(snapshot: snapshot, strict: snapshot.strictMode)
+        guard let record = findRecord(ref: ref, index: index, in: snapshot) else {
+            throw ComputerUseError.invalidElement(ref ?? index.map(String.init) ?? "<missing>")
+        }
+        guard let fresh = freshRecord(matching: record, in: snapshot) else {
+            throw ComputerUseError.staleSnapshot("The target element changed. Take a new snapshot before setting its value.")
+        }
+        AgentCursorOverlay.shared.show(at: fresh.semantic.frame.center)
+        let axWriteAccepted = accessibility.setValue(record: fresh, value: value)
+        if axWriteAccepted, valueMatches(record: fresh, expected: value) {
+            return recordAction(ActionMetadata(ok: true, path: .accessibility, strictMode: snapshot.strictMode, backgroundSafe: true, fallbackUsed: false, message: "Set \(fresh.semantic.ref) via AXValue."))
+        }
+
+        // Web inputs (Chromium/Electron) accept AXValue writes without applying
+        // them. Fall back to focus + select-all + retype, then verify.
+        _ = accessibility.focus(record: fresh)
+        try BackgroundInputDispatcher.pressKey(pid: snapshot.pid, combo: "command+a")
+        try BackgroundInputDispatcher.typeText(pid: snapshot.pid, text: value)
+        Thread.sleep(forTimeInterval: 0.08)
+        let verified = valueMatches(record: fresh, expected: value)
+        return recordAction(ActionMetadata(
+            ok: verified,
+            path: .backgroundCGEvent,
+            strictMode: snapshot.strictMode,
+            backgroundSafe: true,
+            fallbackUsed: true,
+            message: verified
+                ? "AXValue write did not stick; set \(fresh.semantic.ref) via select-all + retype fallback."
+                : "Could not verify the value of \(fresh.semantic.ref) after AXValue and retype fallback."
+        ))
+    }
+
+    private func valueMatches(record: AXElementRecord, expected: String) -> Bool {
+        accessibility.value(record: record) == expected
+    }
+
+    func performAction(snapshotID: String?, ref: String?, index: Int?, action: String) throws -> ActionMetadata {
+        let snapshot = try requireSnapshot(snapshotID: snapshotID)
+        try validateSnapshotForAction(snapshot: snapshot, strict: snapshot.strictMode)
+        guard let record = findRecord(ref: ref, index: index, in: snapshot) else {
+            throw ComputerUseError.invalidElement(ref ?? index.map(String.init) ?? "<missing>")
+        }
+        guard let fresh = freshRecord(matching: record, in: snapshot) else {
+            throw ComputerUseError.staleSnapshot("The target element changed. Take a new snapshot before performing another action.")
+        }
+        AgentCursorOverlay.shared.show(at: fresh.semantic.frame.center)
+        let ok = accessibility.performAction(record: fresh, action: action)
+        return recordAction(ActionMetadata(ok: ok, path: .accessibility, strictMode: snapshot.strictMode, backgroundSafe: true, fallbackUsed: false, message: ok ? "Performed \(action) on \(fresh.semantic.ref)." : "AX action \(action) failed."))
+    }
+
+    func wait(milliseconds: Int) async -> ActionMetadata {
+        let clamped = max(0, min(milliseconds, 10_000))
+        try? await Task.sleep(nanoseconds: UInt64(clamped) * 1_000_000)
+        return ActionMetadata(ok: true, path: .none, strictMode: strictMode, backgroundSafe: true, fallbackUsed: false, message: "Waited \(clamped)ms.")
+    }
+
+    private func clickPoint(_ point: CGPoint, clickCount: Int, strict: Bool, fallbackUsed: Bool) async throws -> ActionMetadata {
+        let snapshot = try requireSnapshot()
+        try validateSnapshotForAction(snapshot: snapshot, strict: strict)
+        AgentCursorOverlay.shared.show(at: point)
+        if strict {
+            guard let windowNumber = snapshot.windowNumber else {
+                throw ComputerUseError.strictModeViolation("background click requires a CG window number")
+            }
+            try await BackgroundInputDispatcher.click(pid: snapshot.pid, windowNumber: windowNumber, point: point, doubleClick: clickCount >= 2)
+            return recordAction(ActionMetadata(ok: true, path: .backgroundCGEvent, strictMode: true, backgroundSafe: true, fallbackUsed: fallbackUsed, message: "Clicked with postToPid at \(Int(point.x)),\(Int(point.y))."))
+        }
+
+        try await foregroundInput.click(point: point, doubleClick: clickCount >= 2)
+        return recordAction(ActionMetadata(ok: true, path: .foregroundCGEvent, strictMode: false, backgroundSafe: false, fallbackUsed: true, message: "Clicked with foreground HID fallback at \(Int(point.x)),\(Int(point.y))."))
+    }
+
+    private func ensureBackgroundActivation(target: WindowTarget) async throws -> Bool {
+        ensureFrontmostMonitor()
+        guard let previousPID = NSWorkspace.shared.frontmostApplication?.processIdentifier else {
+            throw ComputerUseError.noFrontmostApplication
+        }
+        let nextActivationKey = "\(previousPID):\(target.pid)"
+        if activationKey != nextActivationKey {
+            resetBackgroundActivation()
+            let next = BackgroundActivationSession(previousPID: previousPID, targetPID: target.pid)
+            try next.start()
+            activationSession = next
+            activationKey = nextActivationKey
+            activationPreviousPID = previousPID
+            activationTargetPID = target.pid
+            activatedWindowKey = nil
+        }
+
+        let nextWindowKey = "\(target.pid):\(target.windowNumber ?? -1)"
+        if activatedWindowKey != nextWindowKey {
+            guard let activationSession else {
+                throw ComputerUseError.strictModeViolation("background activation session was not created")
+            }
+            try await activationSession.activate(target: target)
+            activatedWindowKey = nextWindowKey
+        }
+        return true
+    }
+
+    private func ensureFrontmostMonitor() {
+        if frontmostMonitor != nil { return }
+        frontmostMonitor = FrontmostApplicationMonitor { [weak self] pid in
+            guard let runtime = self else { return }
+            Task { await runtime.frontmostApplicationChanged(pid: pid) }
+        }
+    }
+
+    private func frontmostApplicationChanged(pid: pid_t?) {
+        guard let pid, activationSession != nil else { return }
+        if pid == activationPreviousPID {
+            staleSnapshotReason = "The user returned focus to the previous app. Take a new snapshot before sending more input."
+        } else if pid == activationTargetPID {
+            staleSnapshotReason = "The target app became frontmost. Take a new snapshot before continuing."
+        } else {
+            staleSnapshotReason = "The user changed app focus. Take a new snapshot before sending more input."
+        }
+        resetBackgroundActivation()
+    }
+
+    private func resetBackgroundActivation() {
+        activationSession?.stop()
+        activationSession = nil
+        activationKey = nil
+        activatedWindowKey = nil
+        activationPreviousPID = nil
+        activationTargetPID = nil
+    }
+
+    private func requireSnapshot(snapshotID: String? = nil) throws -> AppSnapshot {
+        guard let lastSnapshot else { throw ComputerUseError.noSnapshot }
+        if let snapshotID, snapshotID != lastSnapshot.id {
+            throw ComputerUseError.staleSnapshot("Snapshot \(snapshotID) is no longer current. Take a new snapshot before retrying.")
+        }
+        return lastSnapshot
+    }
+
+    private func validateSnapshotForAction(snapshot: AppSnapshot, strict: Bool) throws {
+        if let staleSnapshotReason {
+            throw ComputerUseError.staleSnapshot(staleSnapshotReason)
+        }
+        guard strict else { return }
+        if NSWorkspace.shared.frontmostApplication?.processIdentifier == snapshot.pid { return }
+        guard activationSession != nil, activationTargetPID == snapshot.pid else {
+            throw ComputerUseError.staleSnapshot("The target app is no longer safely activated. Take a new snapshot before retrying.")
+        }
+    }
+
+    private func annotate(snapshot: AppSnapshot) -> AppSnapshot {
+        snapshotSequence += 1
+        let key = windowKey(snapshot: snapshot)
+        let labels = Set(snapshot.records.map { stableLabel(for: $0.semantic) }.filter { !$0.isEmpty })
+        let previous = previousLabelsByWindow[key] ?? []
+        previousLabelsByWindow[key] = labels
+
+        let added = labels.subtracting(previous).sorted()
+        let removed = previous.subtracting(labels).sorted()
+        return snapshot.withSessionMetadata(
+            id: UUID().uuidString.lowercased(),
+            observation: snapshotSequence,
+            recentActions: Array(recentActions.suffix(8)),
+            addedLabels: Array(added.prefix(16)),
+            removedLabels: Array(removed.prefix(16))
+        )
+    }
+
+    private func recordAction(_ metadata: ActionMetadata) -> ActionMetadata {
+        recentActions.append(metadata.message)
+        if recentActions.count > 12 {
+            recentActions.removeFirst(recentActions.count - 12)
+        }
+        return metadata
+    }
+
+    private func freshRecord(matching record: AXElementRecord, in snapshot: AppSnapshot) -> AXElementRecord? {
+        guard let target = try? accessibility.resolveTarget(appName: snapshot.appName, pid: snapshot.pid, windowTitle: snapshot.windowTitle) else {
+            return nil
+        }
+        let records = accessibility.records(target: target)
+        if let index = records.firstIndex(where: { $0.semantic.id == record.semantic.id }), stableMatch(records[index].semantic, record.semantic) {
+            return records[index]
+        }
+        return records.first { stableMatch($0.semantic, record.semantic) }
+    }
+
+    private func stableMatch(_ candidate: SemanticAXElement, _ target: SemanticAXElement) -> Bool {
+        candidate.role == target.role && stableLabel(for: candidate) == stableLabel(for: target)
+    }
+
+    private func stableLabel(for element: SemanticAXElement) -> String {
+        let value = element.value?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
+        let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines)
+        let display = value.isEmpty ? label : "\(label) \(value)"
+        return "\(element.role):\(display)"
+            .components(separatedBy: .whitespacesAndNewlines)
+            .filter { !$0.isEmpty }
+            .joined(separator: " ")
+            .lowercased()
+    }
+
+    private func windowKey(snapshot: AppSnapshot) -> String {
+        "\(snapshot.pid):\(snapshot.windowNumber ?? -1)"
+    }
+
+    private func findRecord(ref: String?, index: Int?, in snapshot: AppSnapshot) -> AXElementRecord? {
+        if let ref {
+            let normalized = normalizeRef(ref)
+            return snapshot.records.first { $0.semantic.ref == normalized }
+        }
+        if let index {
+            if let byID = snapshot.records.first(where: { $0.semantic.id == index }) {
+                return byID
+            }
+            if index >= 0 && index < snapshot.records.count {
+                return snapshot.records[index]
+            }
+        }
+        return nil
+    }
+
+    private func normalizeRef(_ raw: String) -> String {
+        let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
+        if trimmed.hasPrefix("{e"), trimmed.hasSuffix("}") { return trimmed }
+        if trimmed.hasPrefix("e") { return "{\(trimmed)}" }
+        return "{e\(trimmed)}"
+    }
+
+    private func scrollDeltas(direction: String?, amount: Int32) -> (x: Int32, y: Int32) {
+        switch direction?.lowercased() ?? "down" {
+        case "up": return (0, amount)
+        case "left": return (amount, 0)
+        case "right": return (-amount, 0)
+        default: return (0, -amount)
+        }
+    }
+}

packages/handsfree/native/HandsFree/Sources/ComputerUse/FrontmostApplicationMonitor.swift

@@ -0,0 +1,28 @@
+import AppKit
+import Foundation
+
+final class FrontmostApplicationMonitor: @unchecked Sendable {
+    private let queue = DispatchQueue(label: "openwork.computer-use.frontmost-monitor")
+    private var timer: DispatchSourceTimer?
+    private var lastPID: pid_t?
+
+    init(onChange: @escaping @Sendable (pid_t?) -> Void) {
+        lastPID = NSWorkspace.shared.frontmostApplication?.processIdentifier
+        let source = DispatchSource.makeTimerSource(queue: queue)
+        source.schedule(deadline: .now() + .milliseconds(250), repeating: .milliseconds(250))
+        source.setEventHandler { [weak self] in
+            guard let self else { return }
+            let pid = NSWorkspace.shared.frontmostApplication?.processIdentifier
+            if pid != self.lastPID {
+                self.lastPID = pid
+                onChange(pid)
+            }
+        }
+        timer = source
+        source.resume()
+    }
+
+    deinit {
+        timer?.cancel()
+    }
+}

packages/handsfree/native/HandsFree/Sources/ComputerUse/InputService.swift

@@ -0,0 +1,102 @@
+import Foundation
+import CoreGraphics
+
+final class InputService: @unchecked Sendable {
+    func moveMouse(point: CGPoint) throws {
+        guard let source = CGEventSource(stateID: .combinedSessionState) else {
+            throw ComputerUseError.eventSourceFailed
+        }
+        guard let event = CGEvent(mouseEventSource: source, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left) else {
+            throw ComputerUseError.eventCreationFailed
+        }
+        event.post(tap: .cghidEventTap)
+    }
+
+    func click(point: CGPoint, doubleClick: Bool = false) async throws {
+        guard let source = CGEventSource(stateID: .combinedSessionState) else {
+            throw ComputerUseError.eventSourceFailed
+        }
+
+        if let move = CGEvent(mouseEventSource: source, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left) {
+            move.post(tap: .cghidEventTap)
+        }
+        try await Task.sleep(nanoseconds: 50_000_000)
+
+        let count = doubleClick ? 2 : 1
+        for clickState in 1...count {
+            guard let down = CGEvent(mouseEventSource: source, mouseType: .leftMouseDown, mouseCursorPosition: point, mouseButton: .left),
+                  let up = CGEvent(mouseEventSource: source, mouseType: .leftMouseUp, mouseCursorPosition: point, mouseButton: .left) else {
+                throw ComputerUseError.eventCreationFailed
+            }
+            down.setIntegerValueField(.mouseEventClickState, value: Int64(clickState))
+            up.setIntegerValueField(.mouseEventClickState, value: Int64(clickState))
+            down.post(tap: .cghidEventTap)
+            up.post(tap: .cghidEventTap)
+        }
+    }
+
+    func typeText(_ text: String) throws {
+        guard let source = CGEventSource(stateID: .combinedSessionState) else {
+            throw ComputerUseError.eventSourceFailed
+        }
+        let units = Array(text.utf16)
+        for start in stride(from: 0, to: units.count, by: 20) {
+            let end = min(start + 20, units.count)
+            let chunk = Array(units[start..<end])
+            guard let event = CGEvent(keyboardEventSource: source, virtualKey: 0, keyDown: true) else {
+                throw ComputerUseError.eventCreationFailed
+            }
+            event.keyboardSetUnicodeString(stringLength: chunk.count, unicodeString: chunk)
+            event.post(tap: .cghidEventTap)
+            Thread.sleep(forTimeInterval: 0.01)
+        }
+    }
+
+    func pressKey(_ combo: String) throws {
+        let parsed = try BackgroundInputDispatcher.parseCombo(combo)
+        guard let source = CGEventSource(stateID: .combinedSessionState) else {
+            throw ComputerUseError.eventSourceFailed
+        }
+        guard let down = CGEvent(keyboardEventSource: source, virtualKey: parsed.keyCode, keyDown: true),
+              let up = CGEvent(keyboardEventSource: source, virtualKey: parsed.keyCode, keyDown: false) else {
+            throw ComputerUseError.eventCreationFailed
+        }
+        down.flags = parsed.flags
+        up.flags = parsed.flags
+        down.post(tap: .cghidEventTap)
+        up.post(tap: .cghidEventTap)
+    }
+
+    func scroll(point: CGPoint?, deltaX: Int32, deltaY: Int32) throws {
+        guard let source = CGEventSource(stateID: .combinedSessionState) else {
+            throw ComputerUseError.eventSourceFailed
+        }
+        guard let event = CGEvent(scrollWheelEvent2Source: source, units: .line, wheelCount: 2, wheel1: deltaY, wheel2: deltaX, wheel3: 0) else {
+            throw ComputerUseError.eventCreationFailed
+        }
+        if let point { event.location = point }
+        event.post(tap: .cgSessionEventTap)
+    }
+
+    func drag(path: [CGPoint]) async throws {
+        guard path.count >= 2 else { return }
+        guard let source = CGEventSource(stateID: .combinedSessionState) else {
+            throw ComputerUseError.eventSourceFailed
+        }
+        guard let down = CGEvent(mouseEventSource: source, mouseType: .leftMouseDown, mouseCursorPosition: path[0], mouseButton: .left) else {
+            throw ComputerUseError.eventCreationFailed
+        }
+        down.post(tap: .cghidEventTap)
+        for point in path.dropFirst().dropLast() {
+            if let drag = CGEvent(mouseEventSource: source, mouseType: .leftMouseDragged, mouseCursorPosition: point, mouseButton: .left) {
+                drag.post(tap: .cghidEventTap)
+            }
+            try await Task.sleep(nanoseconds: 12_000_000)
+        }
+        guard let last = path.last,
+              let up = CGEvent(mouseEventSource: source, mouseType: .leftMouseUp, mouseCursorPosition: last, mouseButton: .left) else {
+            throw ComputerUseError.eventCreationFailed
+        }
+        up.post(tap: .cghidEventTap)
+    }
+}

packages/handsfree/native/HandsFree/Sources/ComputerUse/MCPServer.swift

@@ -0,0 +1,623 @@
+import AppKit
+import ApplicationServices
+import Foundation
+import ScreenCaptureKit
+
+actor MCPServer {
+    private let runtime = ComputerUseRuntime()
+    private let input = InputService()
+    private var cuaSnapshotFrontmostPID: pid_t?
+
+    func run() async {
+        log("Computer Use server starting")
+        while let line = readLine(strippingNewline: true) {
+            guard !line.isEmpty else { continue }
+            guard let data = line.data(using: .utf8),
+                  let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {
+                log("Invalid JSON-RPC line")
+                continue
+            }
+
+            let id = json["id"]
+            let method = json["method"] as? String ?? ""
+            let params = json["params"] as? [String: Any] ?? [:]
+
+            switch method {
+            case "initialize":
+                respond(id: id, result: [
+                    "protocolVersion": "2025-03-26",
+                    "capabilities": ["tools": [:]],
+                    "serverInfo": ["name": "openwork-computer-use", "version": "0.1.0"],
+                ])
+            case "notifications/initialized":
+                break
+            case "tools/list":
+                respond(id: id, result: ["tools": toolSchemas()])
+            case "tools/call":
+                let name = params["name"] as? String ?? ""
+                let args = params["arguments"] as? [String: Any] ?? [:]
+                let content = await executeTool(name: name, args: args)
+                respond(id: id, result: ["content": content])
+            default:
+                if id != nil {
+                    respondError(id: id, code: -32601, message: "Method not found: \(method)")
+                }
+            }
+        }
+    }
+
+    private func toolSchemas() -> [[String: Any]] {
+        [
+            toolSchema(
+                name: "snapshot",
+                description: "Return target-window screenshot plus compact semantic AX state. Uses strict background activation by default.",
+                properties: [
+                    "app": ["type": "string", "description": "Optional running app name. Omit for frontmost app."],
+                    "pid": ["type": "number", "description": "Optional process id. Disambiguates multiple instances of the same app."],
+                    "window_title": ["type": "string", "description": "Optional window title to target a specific window."],
+                    "strict": ["type": "boolean", "description": "Keep actions on background-safe AX/postToPid paths. Default true."],
+                ]
+            ),
+            toolSchema(
+                name: "click",
+                description: "Click a semantic ref like {e1}, an index, or screenshot x/y. AX is tried first; strict mode only falls back to background postToPid.",
+                properties: [
+                    "ref": ["type": "string", "description": "Semantic ref from snapshot, e.g. {e1}."],
+                    "snapshot_id": ["type": "string", "description": "Optional snapshot id from the latest snapshot response."],
+                    "index": ["type": "number", "description": "Element id or zero-based compatibility index."],
+                    "x": ["type": "number", "description": "Screenshot-image x coordinate (pixels in the snapshot image, e.g. element center.imageX). Not screen pixels."],
+                    "y": ["type": "number", "description": "Screenshot-image y coordinate (pixels in the snapshot image, e.g. element center.imageY). Not screen pixels."],
+                    "screen_x": ["type": "number", "description": "Absolute screen x coordinate (e.g. element center.screenX). Takes precedence over x/y."],
+                    "screen_y": ["type": "number", "description": "Absolute screen y coordinate (e.g. element center.screenY). Takes precedence over x/y."],
+                    "click_count": ["type": "number", "description": "1 or 2. Default 1."],
+                    "strict": ["type": "boolean", "description": "Override strict mode for this action."],
+                ]
+            ),
+            toolSchema(
+                name: "type_text",
+                description: "Type text into the target process. In strict mode this uses CGEvent.postToPid and does not move the real cursor.",
+                properties: [
+                    "text": ["type": "string", "description": "Text to type."],
+                    "snapshot_id": ["type": "string", "description": "Optional snapshot id from the latest snapshot response."],
+                    "strict": ["type": "boolean", "description": "Override strict mode for this action."],
+                ]
+            ),
+            toolSchema(
+                name: "press_key",
+                description: "Press a key combo such as command+k, return, tab, or escape.",
+                properties: [
+                    "combo": ["type": "string", "description": "Key combo."],
+                    "snapshot_id": ["type": "string", "description": "Optional snapshot id from the latest snapshot response."],
+                    "strict": ["type": "boolean", "description": "Override strict mode for this action."],
+                ]
+            ),
+            toolSchema(
+                name: "scroll",
+                description: "Scroll the target window without foregrounding it in strict mode.",
+                properties: [
+                    "direction": ["type": "string", "description": "up, down, left, or right."],
+                    "snapshot_id": ["type": "string", "description": "Optional snapshot id from the latest snapshot response."],
+                    "pages": ["type": "number", "description": "Approximate page count. Default 1."],
+                    "x": ["type": "number", "description": "Optional screenshot x coordinate."],
+                    "y": ["type": "number", "description": "Optional screenshot y coordinate."],
+                    "strict": ["type": "boolean", "description": "Override strict mode for this action."],
+                ]
+            ),
+            toolSchema(
+                name: "set_value",
+                description: "Set a semantic AX element value directly. This stays background-safe.",
+                properties: [
+                    "ref": ["type": "string", "description": "Semantic ref from snapshot."],
+                    "snapshot_id": ["type": "string", "description": "Optional snapshot id from the latest snapshot response."],
+                    "index": ["type": "number", "description": "Element id or zero-based compatibility index."],
+                    "value": ["type": "string", "description": "Value to set."],
+                ]
+            ),
+            toolSchema(
+                name: "perform_action",
+                description: "Perform a named AX action such as AXPress, AXShowMenu, AXIncrement, or AXDecrement.",
+                properties: [
+                    "ref": ["type": "string", "description": "Semantic ref from snapshot."],
+                    "snapshot_id": ["type": "string", "description": "Optional snapshot id from the latest snapshot response."],
+                    "index": ["type": "number", "description": "Element id or zero-based compatibility index."],
+                    "action": ["type": "string", "description": "AX action name. Default AXPress."],
+                ]
+            ),
+            toolSchema(
+                name: "wait",
+                description: "Wait for UI to settle.",
+                properties: ["milliseconds": ["type": "number", "description": "Wait time. Default 1000."]]
+            ),
+            toolSchema(
+                name: "set_strict_mode",
+                description: "Enable or disable strict background mode. Strict mode rejects foreground fallbacks.",
+                properties: ["enabled": ["type": "boolean", "description": "Whether strict background mode is enabled."]]
+            ),
+            toolSchema(name: "check_permissions", description: "Check Accessibility and Screen Recording permission status.", properties: [:]),
+            toolSchema(name: "get_app_state", description: "Compatibility alias for snapshot.", properties: ["app": ["type": "string"], "strict": ["type": "boolean"]]),
+            toolSchema(name: "launch_app", description: "Launch a macOS app by name.", properties: ["name": ["type": "string"]]),
+            toolSchema(name: "activate_app", description: "Bring a running macOS app to the foreground.", properties: ["name": ["type": "string"]]),
+            toolSchema(name: "list_apps", description: "List running regular macOS apps.", properties: [:]),
+            toolSchema(name: "open_url", description: "Open a URL in the default browser or a specific browser app.", properties: ["url": ["type": "string"], "app": ["type": "string"]]),
+            toolSchema(name: "clipboard_read", description: "Read text from the macOS clipboard.", properties: [:]),
+            toolSchema(name: "clipboard_write", description: "Write text to the macOS clipboard.", properties: ["text": ["type": "string"]]),
+            toolSchema(name: "display_info", description: "Return main display logical dimensions and scale factor.", properties: [:]),
+            toolSchema(name: "cua_screenshot", description: "Compatibility full-screen screenshot for CUA loops. Returns logical-size PNG.", properties: [:]),
+            toolSchema(name: "cua_click", description: "Compatibility click at absolute screen coordinates.", properties: ["x": ["type": "number"], "y": ["type": "number"]]),
+            toolSchema(name: "cua_double_click", description: "Compatibility double-click at absolute screen coordinates.", properties: ["x": ["type": "number"], "y": ["type": "number"]]),
+            toolSchema(name: "cua_move", description: "Compatibility mouse move to absolute screen coordinates.", properties: ["x": ["type": "number"], "y": ["type": "number"]]),
+            toolSchema(name: "cua_type", description: "Compatibility text typing into focused input.", properties: ["text": ["type": "string"]]),
+            toolSchema(name: "cua_keypress", description: "Compatibility keypress using CUA key names.", properties: ["keys": ["type": "array", "items": ["type": "string"]]]),
+            toolSchema(name: "cua_scroll", description: "Compatibility scroll at absolute screen coordinates.", properties: ["x": ["type": "number"], "y": ["type": "number"], "scroll_x": ["type": "number"], "scroll_y": ["type": "number"]]),
+            toolSchema(name: "cua_drag", description: "Compatibility drag over an array of [x,y] points.", properties: ["path": ["type": "array"]]),
+            toolSchema(name: "cua_wait", description: "Compatibility wait for UI to settle.", properties: [:]),
+        ]
+    }
+
+    private func toolSchema(name: String, description: String, properties: [String: Any]) -> [String: Any] {
+        ["name": name, "description": description, "inputSchema": ["type": "object", "properties": properties]]
+    }
+
+    private func executeTool(name: String, args: [String: Any]) async -> [[String: Any]] {
+        do {
+            switch name {
+            case "snapshot", "get_app_state":
+                return try await snapshotResult(args: args)
+            case "click":
+                let metadata = try await runtime.click(
+                    snapshotID: snapshotIDArg(args),
+                    ref: args["ref"] as? String,
+                    index: intArg(args, "index"),
+                    imageX: doubleArg(args, "x"),
+                    imageY: doubleArg(args, "y"),
+                    screenX: doubleArg(args, "screen_x"),
+                    screenY: doubleArg(args, "screen_y"),
+                    clickCount: intArg(args, "click_count") ?? 1,
+                    strict: boolArg(args, "strict")
+                )
+                return jsonResult(metadata.dictionary)
+            case "type_text":
+                let metadata = try await runtime.typeText(snapshotID: snapshotIDArg(args), text: args["text"] as? String ?? "", strict: boolArg(args, "strict"))
+                return jsonResult(metadata.dictionary)
+            case "press_key":
+                let metadata = try await runtime.pressKey(snapshotID: snapshotIDArg(args), combo: args["combo"] as? String ?? "", strict: boolArg(args, "strict"))
+                return jsonResult(metadata.dictionary)
+            case "scroll":
+                let metadata = try await runtime.scroll(
+                    snapshotID: snapshotIDArg(args),
+                    direction: args["direction"] as? String,
+                    pages: doubleArg(args, "pages") ?? 1,
+                    imageX: doubleArg(args, "x"),
+                    imageY: doubleArg(args, "y"),
+                    strict: boolArg(args, "strict")
+                )
+                return jsonResult(metadata.dictionary)
+            case "set_value":
+                let metadata = try await runtime.setValue(snapshotID: snapshotIDArg(args), ref: args["ref"] as? String, index: intArg(args, "index"), value: args["value"] as? String ?? "")
+                return jsonResult(metadata.dictionary)
+            case "perform_action":
+                let metadata = try await runtime.performAction(snapshotID: snapshotIDArg(args), ref: args["ref"] as? String, index: intArg(args, "index"), action: args["action"] as? String ?? kAXPressAction)
+                return jsonResult(metadata.dictionary)
+            case "wait":
+                let metadata = await runtime.wait(milliseconds: intArg(args, "milliseconds") ?? 1000)
+                return jsonResult(metadata.dictionary)
+            case "set_strict_mode":
+                let metadata = await runtime.setStrictMode(boolArg(args, "enabled") ?? true)
+                return jsonResult(metadata.dictionary)
+            case "check_permissions":
+                return jsonResult(checkPermissions())
+            case "launch_app":
+                return try await jsonResult(handleLaunchApp(args: args))
+            case "activate_app":
+                return jsonResult(handleActivateApp(args: args))
+            case "list_apps":
+                let details = runningApps()
+                    .compactMap { app -> [String: Any]? in
+                        guard let name = app.localizedName else { return nil }
+                        return ["name": name, "pid": Int(app.processIdentifier)]
+                    }
+                return jsonResult([
+                    "ok": true,
+                    "apps": runningApps().compactMap(\.localizedName).sorted(),
+                    "appDetails": details,
+                ])
+            case "open_url":
+                return try await jsonResult(handleOpenURL(args: args))
+            case "clipboard_read":
+                return jsonResult(["ok": true, "text": NSPasteboard.general.string(forType: .string) ?? ""])
+            case "clipboard_write":
+                let pasteboard = NSPasteboard.general
+                pasteboard.clearContents()
+                pasteboard.setString(args["text"] as? String ?? "", forType: .string)
+                return jsonResult(["ok": true])
+            case "display_info":
+                return jsonResult(displayInfo())
+            case "cua_screenshot":
+                return try await cuaScreenshotResult()
+            case "cua_click":
+                try validateCuaSnapshotFresh()
+                AgentCursorOverlay.shared.show(at: CGPoint(x: intArg(args, "x") ?? 0, y: intArg(args, "y") ?? 0))
+                try await input.click(point: CGPoint(x: intArg(args, "x") ?? 0, y: intArg(args, "y") ?? 0))
+                return jsonResult(["ok": true])
+            case "cua_double_click":
+                try validateCuaSnapshotFresh()
+                AgentCursorOverlay.shared.show(at: CGPoint(x: intArg(args, "x") ?? 0, y: intArg(args, "y") ?? 0))
+                try await input.click(point: CGPoint(x: intArg(args, "x") ?? 0, y: intArg(args, "y") ?? 0), doubleClick: true)
+                return jsonResult(["ok": true])
+            case "cua_move":
+                try validateCuaSnapshotFresh()
+                AgentCursorOverlay.shared.show(at: CGPoint(x: intArg(args, "x") ?? 0, y: intArg(args, "y") ?? 0))
+                try input.moveMouse(point: CGPoint(x: intArg(args, "x") ?? 0, y: intArg(args, "y") ?? 0))
+                return jsonResult(["ok": true])
+            case "cua_type":
+                try validateCuaSnapshotFresh()
+                try input.typeText(args["text"] as? String ?? "")
+                return jsonResult(["ok": true])
+            case "cua_keypress":
+                try validateCuaSnapshotFresh()
+                try input.pressKey(cuaKeysToCombo(args["keys"] as? [String] ?? []))
+                return jsonResult(["ok": true])
+            case "cua_scroll":
+                try validateCuaSnapshotFresh()
+                AgentCursorOverlay.shared.show(at: CGPoint(x: intArg(args, "x") ?? 0, y: intArg(args, "y") ?? 0))
+                try input.scroll(
+                    point: CGPoint(x: intArg(args, "x") ?? 0, y: intArg(args, "y") ?? 0),
+                    deltaX: Int32(intArg(args, "scroll_x") ?? 0),
+                    deltaY: Int32(-(intArg(args, "scroll_y") ?? 0))
+                )
+                return jsonResult(["ok": true])
+            case "cua_drag":
+                try validateCuaSnapshotFresh()
+                let path = parsePointPath(args["path"])
+                if let first = path.first {
+                    AgentCursorOverlay.shared.show(at: first)
+                }
+                try await input.drag(path: path)
+                return jsonResult(["ok": true])
+            case "cua_wait":
+                try await Task.sleep(nanoseconds: 1_000_000_000)
+                return jsonResult(["ok": true])
+            default:
+                return jsonResult(["ok": false, "error": "Unknown tool: \(name)"])
+            }
+        } catch {
+            return jsonResult(errorPayload(error))
+        }
+    }
+
+    private func snapshotResult(args: [String: Any]) async throws -> [[String: Any]] {
+        let snapshot = try await runtime.snapshot(
+            appName: args["app"] as? String,
+            pid: intArg(args, "pid"),
+            windowTitle: args["window_title"] as? String,
+            strict: boolArg(args, "strict")
+        )
+        let payload = snapshotPayload(snapshot)
+        guard let text = jsonString(payload) else {
+            return textResult("Failed to serialize semantic AX snapshot.")
+        }
+        return [
+            ["type": "image", "data": snapshot.screenshotData.base64EncodedString(), "mimeType": snapshot.screenshotMimeType],
+            ["type": "text", "text": text],
+        ]
+    }
+
+    private func snapshotPayload(_ snapshot: AppSnapshot) -> [String: Any] {
+        let elements = snapshot.elements.map { element -> [String: Any] in
+            var dict = element.dictionary
+            let imagePoint = snapshot.screenshotMeta.toImage(point: element.frame.center)
+            dict["center"] = [
+                "screenX": Int(element.frame.center.x),
+                "screenY": Int(element.frame.center.y),
+                "imageX": Int(imagePoint.x),
+                "imageY": Int(imagePoint.y),
+            ]
+            return dict
+        }
+
+        var result: [String: Any] = [
+            "ok": true,
+            "semanticAXVersion": 1,
+            "snapshotId": snapshot.id,
+            "snapshot_id": snapshot.id,
+            "observation": snapshot.observation,
+            "app": snapshot.appName,
+            "pid": Int(snapshot.pid),
+            "windowTitle": snapshot.windowTitle ?? "",
+            "screenshot": snapshot.screenshotMeta.dictionary,
+            "execution": [
+                "strictMode": snapshot.strictMode,
+                "backgroundActivated": snapshot.backgroundActivated,
+                "defaultPath": snapshot.strictMode ? "accessibility_then_background_cgevent" : "accessibility_then_foreground_fallback",
+            ],
+            "elements": elements,
+            "hint": "Use refs like {e1}. Prefer AX-capable refs; strict mode rejects foreground fallback and reports path metadata after every action.",
+        ]
+        if !snapshot.recentActions.isEmpty {
+            result["recentActions"] = snapshot.recentActions
+        }
+        if !snapshot.addedLabels.isEmpty || !snapshot.removedLabels.isEmpty {
+            result["stateDelta"] = ["added": snapshot.addedLabels, "removed": snapshot.removedLabels]
+        }
+        if let windowNumber = snapshot.windowNumber {
+            result["windowNumber"] = windowNumber
+        }
+        return result
+    }
+
+    private func checkPermissions() -> [String: Any] {
+        ComputerUsePermissions.status().dictionary
+    }
+
+    private func handleActivateApp(args: [String: Any]) -> [String: Any] {
+        let name = args["name"] as? String ?? ""
+        guard let app = runningApp(named: name) else {
+            return ["ok": false, "error": "App '\(name)' is not running."]
+        }
+        app.activate()
+        return ["ok": true, "app": app.localizedName ?? name]
+    }
+
+    private func handleLaunchApp(args: [String: Any]) async throws -> [String: Any] {
+        let name = (args["name"] as? String ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
+        guard !name.isEmpty else { return ["ok": false, "error": "App name is required."] }
+        if let app = runningApp(named: name) {
+            app.activate()
+            return ["ok": true, "app": app.localizedName ?? name, "alreadyRunning": true]
+        }
+        guard let appURL = applicationURL(named: name) else {
+            return ["ok": false, "error": "App '\(name)' was not found."]
+        }
+        let config = NSWorkspace.OpenConfiguration()
+        config.activates = true
+        let app = try await NSWorkspace.shared.openApplication(at: appURL, configuration: config)
+        return ["ok": true, "app": app.localizedName ?? name]
+    }
+
+    private func handleOpenURL(args: [String: Any]) async throws -> [String: Any] {
+        guard let rawURL = args["url"] as? String, let url = URL(string: rawURL) else {
+            return ["ok": false, "error": "Invalid URL."]
+        }
+        if let appName = args["app"] as? String, !appName.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
+            guard let appURL = applicationURL(named: appName) else {
+                return ["ok": false, "error": "App '\(appName)' was not found."]
+            }
+            _ = try await NSWorkspace.shared.open([url], withApplicationAt: appURL, configuration: NSWorkspace.OpenConfiguration())
+            return ["ok": true]
+        }
+        return ["ok": NSWorkspace.shared.open(url)]
+    }
+
+    private func displayInfo() -> [String: Any] {
+        guard let screen = NSScreen.main else { return ["ok": false, "error": "No main screen."] }
+        return [
+            "ok": true,
+            "width": Int(screen.frame.width),
+            "height": Int(screen.frame.height),
+            "scale_factor": screen.backingScaleFactor,
+        ]
+    }
+
+    private func cuaScreenshotResult() async throws -> [[String: Any]] {
+        guard let screen = NSScreen.main else { throw ComputerUseError.screenshotFailed }
+        cuaSnapshotFrontmostPID = NSWorkspace.shared.frontmostApplication?.processIdentifier
+        let cgImage = await screenCaptureKitDisplayImage() ?? CGWindowListCreateImage(CGRect.null, .optionOnScreenOnly, kCGNullWindowID, [.bestResolution])
+        guard let cgImage else {
+            throw ComputerUseError.screenshotFailed
+        }
+        let logicalWidth = Int(screen.frame.width)
+        let logicalHeight = Int(screen.frame.height)
+        guard let rep = NSBitmapImageRep(
+            bitmapDataPlanes: nil,
+            pixelsWide: logicalWidth,
+            pixelsHigh: logicalHeight,
+            bitsPerSample: 8,
+            samplesPerPixel: 4,
+            hasAlpha: true,
+            isPlanar: false,
+            colorSpaceName: .deviceRGB,
+            bytesPerRow: 0,
+            bitsPerPixel: 0
+        ) else {
+            throw ComputerUseError.screenshotFailed
+        }
+        rep.size = NSSize(width: logicalWidth, height: logicalHeight)
+        guard let context = NSGraphicsContext(bitmapImageRep: rep) else {
+            throw ComputerUseError.screenshotFailed
+        }
+        NSGraphicsContext.saveGraphicsState()
+        NSGraphicsContext.current = context
+        NSImage(cgImage: cgImage, size: NSSize(width: cgImage.width, height: cgImage.height))
+            .draw(in: NSRect(x: 0, y: 0, width: logicalWidth, height: logicalHeight))
+        NSGraphicsContext.restoreGraphicsState()
+        guard let png = rep.representation(using: .png, properties: [:]) else {
+            throw ComputerUseError.screenshotFailed
+        }
+        return [
+            ["type": "text", "text": jsonString(["ok": true, "width": logicalWidth, "height": logicalHeight]) ?? "{\"ok\":true}"],
+            ["type": "image", "data": png.base64EncodedString(), "mimeType": "image/png"],
+        ]
+    }
+
+    private func screenCaptureKitDisplayImage() async -> CGImage? {
+        do {
+            let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)
+            guard let display = content.displays.first else { return nil }
+            let configuration = SCStreamConfiguration()
+            configuration.width = display.width
+            configuration.height = display.height
+            configuration.showsCursor = true
+            let filter = SCContentFilter(display: display, excludingWindows: [])
+            return try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: configuration)
+        } catch {
+            return nil
+        }
+    }
+
+    private func runningApps() -> [NSRunningApplication] {
+        NSWorkspace.shared.runningApplications.filter { $0.activationPolicy == .regular }
+    }
+
+    private func runningApp(named name: String) -> NSRunningApplication? {
+        let needle = name.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
+        return runningApps().first { $0.localizedName?.lowercased() == needle }
+            ?? runningApps().first { $0.localizedName?.lowercased().contains(needle) == true }
+    }
+
+    private func applicationURL(named name: String) -> URL? {
+        let trimmed = name.trimmingCharacters(in: .whitespacesAndNewlines)
+        if let url = NSWorkspace.shared.urlForApplication(withBundleIdentifier: bundleId(for: trimmed)) {
+            return url
+        }
+        if let path = NSWorkspace.shared.fullPath(forApplication: trimmed) {
+            return URL(fileURLWithPath: path)
+        }
+        let candidates = [
+            "/Applications/\(trimmed).app",
+            "/System/Applications/\(trimmed).app",
+            "/Applications/Utilities/\(trimmed).app",
+            NSString(string: "~/Applications/\(trimmed).app").expandingTildeInPath,
+        ]
+        return candidates.map(URL.init(fileURLWithPath:)).first { FileManager.default.fileExists(atPath: $0.path) }
+    }
+
+    private func bundleId(for appName: String) -> String {
+        switch appName.lowercased() {
+        case "safari": return "com.apple.Safari"
+        case "google chrome", "chrome": return "com.google.Chrome"
+        case "arc": return "company.thebrowser.Browser"
+        case "microsoft edge", "edge": return "com.microsoft.edgemac"
+        case "brave", "brave browser": return "com.brave.Browser"
+        case "slack": return "com.tinyspeck.slackmacgap"
+        default: return ""
+        }
+    }
+
+    private func cuaKeysToCombo(_ keys: [String]) -> String {
+        keys.map { key in
+            switch key.lowercased() {
+            case "ctrl", "control": return "command"
+            case "meta", "super", "win", "cmd": return "command"
+            case "alt": return "option"
+            case "arrowup": return "up"
+            case "arrowdown": return "down"
+            case "arrowleft": return "left"
+            case "arrowright": return "right"
+            case "backspace": return "delete"
+            case " ": return "space"
+            default: return key.lowercased()
+            }
+        }.joined(separator: "+")
+    }
+
+    private func parsePointPath(_ raw: Any?) -> [CGPoint] {
+        guard let pairs = raw as? [[Any]] else { return [] }
+        return pairs.compactMap { pair in
+            guard pair.count >= 2 else { return nil }
+            let x = valueAsDouble(pair[0])
+            let y = valueAsDouble(pair[1])
+            guard let x, let y else { return nil }
+            return CGPoint(x: x, y: y)
+        }
+    }
+
+    private func validateCuaSnapshotFresh() throws {
+        guard let snapshotPID = cuaSnapshotFrontmostPID else {
+            throw ComputerUseError.staleSnapshot("CUA action requires a screenshot first.")
+        }
+        let currentPID = NSWorkspace.shared.frontmostApplication?.processIdentifier
+        guard currentPID == snapshotPID else {
+            throw ComputerUseError.staleSnapshot("The user changed focus since the last CUA screenshot. Capture a new screenshot before acting.")
+        }
+    }
+
+    private func valueAsDouble(_ value: Any) -> Double? {
+        if let value = value as? Double { return value }
+        if let value = value as? Int { return Double(value) }
+        if let value = value as? String { return Double(value) }
+        return nil
+    }
+
+    private func respond(id: Any?, result: Any) {
+        var response: [String: Any] = ["jsonrpc": "2.0", "result": result]
+        if let id { response["id"] = id }
+        writeLine(response)
+    }
+
+    private func respondError(id: Any?, code: Int, message: String) {
+        var response: [String: Any] = ["jsonrpc": "2.0", "error": ["code": code, "message": message]]
+        if let id { response["id"] = id }
+        writeLine(response)
+    }
+
+    private func writeLine(_ object: [String: Any]) {
+        guard let text = jsonString(object) else { return }
+        print(text)
+    }
+
+    private func textResult(_ text: String) -> [[String: Any]] {
+        [["type": "text", "text": text]]
+    }
+
+    private func jsonResult(_ payload: [String: Any]) -> [[String: Any]] {
+        textResult(jsonString(payload) ?? "{\"ok\":false,\"error\":\"Failed to serialize result.\"}")
+    }
+
+    private func jsonString(_ value: Any) -> String? {
+        guard JSONSerialization.isValidJSONObject(value),
+              let data = try? JSONSerialization.data(withJSONObject: value),
+              let text = String(data: data, encoding: .utf8) else {
+            return nil
+        }
+        return text
+    }
+
+    private func errorPayload(_ error: Error) -> [String: Any] {
+        let message = error.localizedDescription
+        var payload: [String: Any] = ["ok": false, "error": message]
+        if case ComputerUseError.staleSnapshot = error {
+            payload["staleSnapshot"] = true
+            payload["retryable"] = false
+            payload["requiredNextAction"] = "snapshot"
+            payload["hint"] = "Take a fresh snapshot before retrying. Do not repeat the same action against stale UI state."
+        }
+        if message.localizedCaseInsensitiveContains("accessibility") {
+            payload["permissionNeeded"] = "accessibility"
+        }
+        if message.localizedCaseInsensitiveContains("screenshot") || message.localizedCaseInsensitiveContains("screen recording") {
+            payload["permissionNeeded"] = "screen-recording"
+        }
+        return payload
+    }
+
+    private func log(_ message: String) {
+        fputs("[ComputerUse] \(message)\n", stderr)
+    }
+
+    private func intArg(_ args: [String: Any], _ key: String) -> Int? {
+        if let value = args[key] as? Int { return value }
+        if let value = args[key] as? Double { return Int(value) }
+        if let value = args[key] as? String { return Int(value) }
+        return nil
+    }
+
+    private func doubleArg(_ args: [String: Any], _ key: String) -> Double? {
+        if let value = args[key] as? Double { return value }
+        if let value = args[key] as? Int { return Double(value) }
+        if let value = args[key] as? String { return Double(value) }
+        return nil
+    }
+
+    private func boolArg(_ args: [String: Any], _ key: String) -> Bool? {
+        if let value = args[key] as? Bool { return value }
+        if let value = args[key] as? String {
+            if value == "true" { return true }
+            if value == "false" { return false }
+        }
+        return nil
+    }
+
+    private func snapshotIDArg(_ args: [String: Any]) -> String? {
+        args["snapshot_id"] as? String ?? args["snapshotId"] as? String
+    }
+}

packages/handsfree/native/HandsFree/Sources/ComputerUse/PermissionSetupApp.swift

@@ -0,0 +1,593 @@
+import AppKit
+import ApplicationServices
+import Foundation
+
+enum ComputerUsePermissionTarget: String {
+    case accessibility
+    case screenRecording = "screen-recording"
+}
+
+struct ComputerUsePermissionStatus {
+    let accessibility: Bool
+    let screenRecording: Bool
+    var ok: Bool { accessibility && screenRecording }
+    var dictionary: [String: Any] {
+        ["ok": ok, "accessibility": accessibility, "screenRecording": screenRecording]
+    }
+}
+
+enum ComputerUsePermissions {
+    /// Non-prompting check — safe to call on a timer / from HTTP polling.
+    static func status() -> ComputerUsePermissionStatus {
+        ComputerUsePermissionStatus(
+            accessibility: AXIsProcessTrusted(),
+            screenRecording: CGPreflightScreenCaptureAccess()
+        )
+    }
+
+    /// Prompting request — only call when the user explicitly clicks "Grant".
+    static func request(_ target: ComputerUsePermissionTarget) {
+        switch target {
+        case .accessibility:
+            let opts = [kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary
+            _ = AXIsProcessTrustedWithOptions(opts)
+        case .screenRecording:
+            _ = CGRequestScreenCaptureAccess()
+        }
+    }
+}
+
+// MARK: - App delegate
+
+@MainActor
+final class PermissionSetupAppDelegate: NSObject, NSApplicationDelegate {
+    private var windowController: NSWindowController?
+
+    func applicationDidFinishLaunching(_ notification: Notification) {
+        let window = PermissionSetupWindow()
+        let vc = PermissionSetupViewController()
+        window.contentViewController = vc
+        let wc = NSWindowController(window: window)
+        windowController = wc
+        wc.showWindow(nil)
+        NSApplication.shared.activate(ignoringOtherApps: true)
+    }
+
+    func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { true }
+}
+
+// MARK: - Window
+
+private final class PermissionSetupWindow: NSWindow {
+    init() {
+        super.init(
+            contentRect: NSRect(x: 0, y: 0, width: 460, height: 600),
+            styleMask: [.titled, .closable, .fullSizeContentView],
+            backing: .buffered,
+            defer: false
+        )
+        title = "OpenWork Computer Use"
+        titleVisibility = .hidden
+        titlebarAppearsTransparent = true
+        isMovableByWindowBackground = true
+        backgroundColor = .clear
+        isOpaque = false
+        hasShadow = true
+        center()
+    }
+}
+
+// MARK: - View controller
+
+@MainActor
+final class PermissionSetupViewController: NSViewController {
+    private var axBadge = StatusBadge()
+    private var srBadge = StatusBadge()
+    private var axGrantButton: NSButton?
+    private var srGrantButton: NSButton?
+    private var timer: Timer?
+
+    init() { super.init(nibName: nil, bundle: nil) }
+    required init?(coder: NSCoder) { nil }
+    deinit { timer?.invalidate() }
+
+    override func loadView() {
+        // Liquid glass: NSVisualEffectView blending with content behind the window
+        let blur = NSVisualEffectView()
+        blur.material = .underWindowBackground
+        blur.blendingMode = .behindWindow
+        blur.state = .active
+        blur.wantsLayer = true
+        blur.layer?.cornerRadius = 12
+        blur.layer?.masksToBounds = true
+        view = blur
+    }
+
+    override func viewDidLoad() {
+        super.viewDidLoad()
+        buildLayout()
+        refresh()
+        timer = Timer.scheduledTimer(withTimeInterval: 1.5, repeats: true) { [weak self] _ in
+            guard let controller = self else { return }
+            Task { @MainActor in controller.refresh() }
+        }
+    }
+
+    private func buildLayout() {
+        let iconView = NSImageView()
+        iconView.image = NSApplication.shared.applicationIconImage
+        iconView.imageScaling = .scaleProportionallyUpOrDown
+        iconView.translatesAutoresizingMaskIntoConstraints = false
+        iconView.widthAnchor.constraint(equalToConstant: 52).isActive = true
+        iconView.heightAnchor.constraint(equalToConstant: 52).isActive = true
+
+        let titleField = textField("Computer Use Setup", size: 20, weight: .semibold)
+        titleField.alignment = .center
+
+        let subtitleField = wrappingField(
+            "Grant two permissions so agents can see and control apps in the background.",
+            size: 13
+        )
+        subtitleField.textColor = .secondaryLabelColor
+        subtitleField.alignment = .center
+
+        let header = vstack([iconView, titleField, subtitleField], spacing: 8, alignment: NSLayoutConstraint.Attribute.centerX)
+
+        let axCard = makeAccessibilityCard()
+        let srCard = makeScreenRecordingCard()
+
+        let doneBtn = NSButton(title: "Done — Return to OpenWork", target: self, action: #selector(done))
+        doneBtn.bezelStyle = .rounded
+        doneBtn.controlSize = .large
+        doneBtn.keyEquivalent = "\r"
+
+        let root = vstack([header, axCard, srCard, doneBtn], spacing: 14, alignment: NSLayoutConstraint.Attribute.leading)
+        root.translatesAutoresizingMaskIntoConstraints = false
+        view.addSubview(root)
+
+        NSLayoutConstraint.activate([
+            root.leadingAnchor.constraint(equalTo: view.leadingAnchor, constant: 20),
+            root.trailingAnchor.constraint(equalTo: view.trailingAnchor, constant: -20),
+            root.topAnchor.constraint(greaterThanOrEqualTo: view.topAnchor, constant: 52),
+            root.bottomAnchor.constraint(lessThanOrEqualTo: view.bottomAnchor, constant: -20),
+            root.centerYAnchor.constraint(equalTo: view.centerYAnchor, constant: 10),
+            axCard.widthAnchor.constraint(equalTo: root.widthAnchor),
+            srCard.widthAnchor.constraint(equalTo: root.widthAnchor),
+            doneBtn.widthAnchor.constraint(equalTo: root.widthAnchor),
+        ])
+    }
+
+    // MARK: Accessibility card
+
+    private func makeAccessibilityCard() -> NSView {
+        let step = StepCircle(number: "1")
+        let title = textField("Accessibility", size: 15, weight: .semibold)
+        let body = wrappingField(
+            "Allows agents to interact with UI controls, click buttons, and type text entirely in the background.",
+            size: 13
+        )
+        body.textColor = .secondaryLabelColor
+
+        let btn = NSButton(title: "Grant Accessibility", target: self, action: #selector(grantAccessibility))
+        btn.bezelStyle = .rounded
+        btn.controlSize = .regular
+        axGrantButton = btn
+
+        let top = hstack([step, title, springy(), axBadge], spacing: 8)
+        let inner = vstack([top, body, btn], spacing: 10, alignment: NSLayoutConstraint.Attribute.leading)
+        btn.widthAnchor.constraint(equalTo: inner.widthAnchor).isActive = true
+
+        return glassCard(inner)
+    }
+
+    // MARK: Screen Recording card
+
+    private func makeScreenRecordingCard() -> NSView {
+        let step = StepCircle(number: "2")
+        let title = textField("Screen Recording", size: 15, weight: .semibold)
+        let body = wrappingField(
+            "Lets agents see what is on screen. If macOS does not prompt automatically, drag the app icon below into the Screen Recording list.",
+            size: 13
+        )
+        body.textColor = .secondaryLabelColor
+
+        let dragFlow = makeDragFlowView()
+
+        let reqBtn = NSButton(title: "Request Screen Recording", target: self, action: #selector(requestScreenRecording))
+        reqBtn.bezelStyle = .rounded
+        reqBtn.controlSize = .regular
+        srGrantButton = reqBtn
+
+        let openBtn = NSButton(title: "Open Privacy & Security", target: self, action: #selector(openPrivacySecurity))
+        openBtn.bezelStyle = .rounded
+        openBtn.controlSize = .small
+
+        let top = hstack([step, title, springy(), srBadge], spacing: 8)
+        let inner = vstack([top, body, dragFlow, reqBtn, openBtn], spacing: 10, alignment: NSLayoutConstraint.Attribute.leading)
+        reqBtn.widthAnchor.constraint(equalTo: inner.widthAnchor).isActive = true
+        openBtn.widthAnchor.constraint(equalTo: inner.widthAnchor).isActive = true
+        dragFlow.widthAnchor.constraint(equalTo: inner.widthAnchor).isActive = true
+
+        return glassCard(inner)
+    }
+
+    // MARK: Drag flow visualization
+
+    private func makeDragFlowView() -> NSView {
+        // Large draggable app icon
+        let iconView = DraggableAppIconView(frame: .zero)
+        iconView.image = NSApplication.shared.applicationIconImage
+        iconView.imageScaling = .scaleProportionallyUpOrDown
+        iconView.translatesAutoresizingMaskIntoConstraints = false
+        iconView.widthAnchor.constraint(equalToConstant: 56).isActive = true
+        iconView.heightAnchor.constraint(equalToConstant: 56).isActive = true
+
+        let dragHint = textField("Drag me", size: 10)
+        dragHint.textColor = .tertiaryLabelColor
+        dragHint.alignment = .center
+
+        let iconCol = vstack([iconView, dragHint], spacing: 5, alignment: NSLayoutConstraint.Attribute.centerX)
+
+        let arrow = textField("→", size: 20)
+        arrow.textColor = .tertiaryLabelColor
+
+        let dropZone = PrivacyDropZoneView()
+        dropZone.translatesAutoresizingMaskIntoConstraints = false
+        dropZone.widthAnchor.constraint(equalToConstant: 116).isActive = true
+        dropZone.heightAnchor.constraint(equalToConstant: 60).isActive = true
+
+        let row = hstack([iconCol, arrow, dropZone], spacing: 14)
+        row.alignment = NSLayoutConstraint.Attribute.centerY
+        row.distribution = NSStackView.Distribution.fill
+
+        let hint = wrappingField(
+            "Drag this icon into the Screen Recording list in Privacy & Security, then enable it.",
+            size: 11
+        )
+        hint.textColor = .secondaryLabelColor
+        hint.alignment = .center
+
+        let content = vstack([row, hint], spacing: 8, alignment: NSLayoutConstraint.Attribute.centerX)
+        content.translatesAutoresizingMaskIntoConstraints = false
+
+        let wrapper = InnerTintView()
+        wrapper.translatesAutoresizingMaskIntoConstraints = false
+        wrapper.addSubview(content)
+        NSLayoutConstraint.activate([
+            content.topAnchor.constraint(equalTo: wrapper.topAnchor, constant: 12),
+            content.bottomAnchor.constraint(equalTo: wrapper.bottomAnchor, constant: -12),
+            content.leadingAnchor.constraint(equalTo: wrapper.leadingAnchor, constant: 12),
+            content.trailingAnchor.constraint(equalTo: wrapper.trailingAnchor, constant: -12),
+        ])
+        return wrapper
+    }
+
+    // MARK: Refresh
+
+    private func refresh() {
+        let status = ComputerUsePermissions.status()
+        axBadge.update(granted: status.accessibility)
+        srBadge.update(granted: status.screenRecording)
+        axGrantButton?.isEnabled = !status.accessibility
+        srGrantButton?.isEnabled = !status.screenRecording
+    }
+
+    // MARK: Actions
+
+    @objc private func grantAccessibility() {
+        ComputerUsePermissions.request(.accessibility)
+        DispatchQueue.main.asyncAfter(deadline: .now() + 0.4) { [weak self] in self?.refresh() }
+    }
+
+    @objc private func requestScreenRecording() {
+        ComputerUsePermissions.request(.screenRecording)
+        DispatchQueue.main.asyncAfter(deadline: .now() + 0.4) { [weak self] in self?.refresh() }
+    }
+
+    @objc private func openPrivacySecurity() {
+        guard let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture") else { return }
+        NSWorkspace.shared.open(url)
+    }
+
+    @objc private func done() {
+        NSApplication.shared.terminate(nil)
+    }
+
+    // MARK: Builder helpers
+
+    private func glassCard(_ content: NSView) -> GlassCardView {
+        let card = GlassCardView()
+        card.addContent(content)
+        return card
+    }
+
+    private func textField(_ text: String, size: CGFloat, weight: NSFont.Weight = .regular) -> NSTextField {
+        let f = NSTextField(labelWithString: text)
+        f.font = .systemFont(ofSize: size, weight: weight)
+        return f
+    }
+
+    private func wrappingField(_ text: String, size: CGFloat) -> NSTextField {
+        let f = NSTextField(wrappingLabelWithString: text)
+        f.font = .systemFont(ofSize: size)
+        return f
+    }
+
+    private func vstack(_ views: [NSView], spacing: CGFloat, alignment: NSLayoutConstraint.Attribute = NSLayoutConstraint.Attribute.leading) -> NSStackView {
+        let sv = NSStackView(views: views)
+        sv.orientation = .vertical
+        sv.spacing = spacing
+        sv.alignment = alignment
+        return sv
+    }
+
+    private func hstack(_ views: [NSView], spacing: CGFloat) -> NSStackView {
+        let sv = NSStackView(views: views)
+        sv.orientation = .horizontal
+        sv.spacing = spacing
+        sv.alignment = NSLayoutConstraint.Attribute.centerY
+        return sv
+    }
+
+    private func springy() -> NSView {
+        let v = NSView()
+        v.setContentHuggingPriority(.defaultLow, for: .horizontal)
+        v.setContentCompressionResistancePriority(.defaultLow, for: .horizontal)
+        return v
+    }
+}
+
+// MARK: - Glass card
+
+/// NSView container with an NSVisualEffectView inside for the glass effect.
+/// Uses updateLayer() so the border color responds to appearance changes.
+final class GlassCardView: NSView {
+    private let blur: NSVisualEffectView
+
+    override init(frame: NSRect) {
+        blur = NSVisualEffectView()
+        blur.material = .contentBackground
+        blur.blendingMode = .withinWindow
+        blur.state = .active
+        blur.wantsLayer = true
+        blur.layer?.cornerRadius = 13
+        blur.layer?.masksToBounds = true
+        blur.translatesAutoresizingMaskIntoConstraints = false
+
+        super.init(frame: frame)
+        wantsLayer = true
+        layer?.cornerRadius = 14
+
+        addSubview(blur)
+        NSLayoutConstraint.activate([
+            blur.topAnchor.constraint(equalTo: topAnchor),
+            blur.bottomAnchor.constraint(equalTo: bottomAnchor),
+            blur.leadingAnchor.constraint(equalTo: leadingAnchor),
+            blur.trailingAnchor.constraint(equalTo: trailingAnchor),
+        ])
+    }
+
+    required init?(coder: NSCoder) { nil }
+
+    override var wantsUpdateLayer: Bool { true }
+    override func updateLayer() {
+        layer?.borderWidth = 0.5
+        layer?.borderColor = NSColor.separatorColor.cgColor
+    }
+
+    func addContent(_ content: NSView) {
+        content.translatesAutoresizingMaskIntoConstraints = false
+        blur.addSubview(content)
+        NSLayoutConstraint.activate([
+            content.topAnchor.constraint(equalTo: blur.topAnchor, constant: 16),
+            content.bottomAnchor.constraint(equalTo: blur.bottomAnchor, constant: -16),
+            content.leadingAnchor.constraint(equalTo: blur.leadingAnchor, constant: 16),
+            content.trailingAnchor.constraint(equalTo: blur.trailingAnchor, constant: -16),
+        ])
+    }
+}
+
+// MARK: - Status badge
+
+/// Pill badge showing Granted/Needed with adaptive color.
+@MainActor
+final class StatusBadge: NSView {
+    private let dot = NSView()
+    private let label = NSTextField(labelWithString: "")
+    private var isGranted = false
+
+    override init(frame: NSRect) {
+        super.init(frame: frame)
+        wantsLayer = true
+        layer?.cornerRadius = 10
+
+        dot.wantsLayer = true
+        dot.layer?.cornerRadius = 4
+        dot.translatesAutoresizingMaskIntoConstraints = false
+        dot.widthAnchor.constraint(equalToConstant: 8).isActive = true
+        dot.heightAnchor.constraint(equalToConstant: 8).isActive = true
+
+        label.font = .systemFont(ofSize: 12, weight: .medium)
+        label.translatesAutoresizingMaskIntoConstraints = false
+
+        let row = NSStackView(views: [dot, label])
+        row.orientation = .horizontal
+        row.spacing = 5
+        row.alignment = .centerY
+        row.translatesAutoresizingMaskIntoConstraints = false
+        addSubview(row)
+
+        NSLayoutConstraint.activate([
+            row.topAnchor.constraint(equalTo: topAnchor, constant: 4),
+            row.bottomAnchor.constraint(equalTo: bottomAnchor, constant: -4),
+            row.leadingAnchor.constraint(equalTo: leadingAnchor, constant: 8),
+            row.trailingAnchor.constraint(equalTo: trailingAnchor, constant: -8),
+        ])
+    }
+
+    required init?(coder: NSCoder) { nil }
+
+    func update(granted: Bool) {
+        isGranted = granted
+        applyColors()
+        label.stringValue = granted ? "Granted" : "Needed"
+        label.textColor = granted ? .systemGreen : .systemOrange
+    }
+
+    private func applyColors() {
+        layer?.backgroundColor = isGranted
+            ? NSColor.systemGreen.withAlphaComponent(0.15).cgColor
+            : NSColor.systemOrange.withAlphaComponent(0.12).cgColor
+        dot.layer?.backgroundColor = isGranted
+            ? NSColor.systemGreen.cgColor
+            : NSColor.systemOrange.cgColor
+    }
+
+    override var wantsUpdateLayer: Bool { true }
+    override func updateLayer() { applyColors() }
+}
+
+// MARK: - Step circle
+
+private final class StepCircle: NSView {
+    init(number: String) {
+        super.init(frame: .zero)
+        wantsLayer = true
+        layer?.backgroundColor = NSColor.controlAccentColor.cgColor
+        layer?.cornerRadius = 11
+        translatesAutoresizingMaskIntoConstraints = false
+        widthAnchor.constraint(equalToConstant: 22).isActive = true
+        heightAnchor.constraint(equalToConstant: 22).isActive = true
+
+        let label = NSTextField(labelWithString: number)
+        label.font = .systemFont(ofSize: 12, weight: .semibold)
+        label.textColor = .white
+        label.alignment = .center
+        label.translatesAutoresizingMaskIntoConstraints = false
+        addSubview(label)
+        NSLayoutConstraint.activate([
+            label.centerXAnchor.constraint(equalTo: centerXAnchor),
+            label.centerYAnchor.constraint(equalTo: centerYAnchor),
+        ])
+    }
+
+    required init?(coder: NSCoder) { nil }
+
+    override var wantsUpdateLayer: Bool { true }
+    override func updateLayer() {
+        layer?.backgroundColor = NSColor.controlAccentColor.cgColor
+    }
+}
+
+// MARK: - Inner tint (drag flow background)
+
+/// Subtle tinted background for the drag-flow visualization.
+/// Adapts to light/dark appearance via updateLayer().
+private final class InnerTintView: NSView {
+    override init(frame: NSRect) {
+        super.init(frame: frame)
+        wantsLayer = true
+        layer?.cornerRadius = 10
+    }
+
+    required init?(coder: NSCoder) { nil }
+
+    override var wantsUpdateLayer: Bool { true }
+    override func updateLayer() {
+        let isDark = effectiveAppearance.bestMatch(from: [.darkAqua, .aqua]) == .darkAqua
+        layer?.backgroundColor = isDark
+            ? NSColor.white.withAlphaComponent(0.05).cgColor
+            : NSColor.black.withAlphaComponent(0.04).cgColor
+    }
+}
+
+// MARK: - Privacy drop zone
+
+/// Dashed rounded rect showing where the user should drop the app icon.
+final class PrivacyDropZoneView: NSView {
+    private let label: NSTextField
+
+    override init(frame: NSRect) {
+        label = NSTextField(labelWithString: "Screen Recording\nlist")
+        super.init(frame: frame)
+        label.font = .systemFont(ofSize: 11)
+        label.textColor = .tertiaryLabelColor
+        label.alignment = .center
+        label.maximumNumberOfLines = 2
+        label.translatesAutoresizingMaskIntoConstraints = false
+        addSubview(label)
+        NSLayoutConstraint.activate([
+            label.centerXAnchor.constraint(equalTo: centerXAnchor),
+            label.centerYAnchor.constraint(equalTo: centerYAnchor),
+            label.leadingAnchor.constraint(greaterThanOrEqualTo: leadingAnchor, constant: 4),
+            label.trailingAnchor.constraint(lessThanOrEqualTo: trailingAnchor, constant: -4),
+        ])
+    }
+
+    required init?(coder: NSCoder) { nil }
+
+    override func draw(_ dirtyRect: NSRect) {
+        super.draw(dirtyRect)
+        NSGraphicsContext.saveGraphicsState()
+        let path = NSBezierPath(roundedRect: bounds.insetBy(dx: 1, dy: 1), xRadius: 8, yRadius: 8)
+        path.lineWidth = 1.5
+        path.setLineDash([6.0, 4.0], count: 2, phase: 0)
+        NSColor.tertiaryLabelColor.setStroke()
+        path.stroke()
+        NSGraphicsContext.restoreGraphicsState()
+    }
+}
+
+// MARK: - Draggable app icon
+
+/// NSImageView that initiates a drag session carrying the app bundle URL,
+/// so it can be dropped into System Settings > Privacy & Security > Screen Recording.
+final class DraggableAppIconView: NSImageView, NSDraggingSource {
+    override init(frame: NSRect) {
+        super.init(frame: frame)
+        wantsLayer = true
+        layer?.cornerRadius = 12
+        layer?.masksToBounds = true
+        addTrackingArea(NSTrackingArea(
+            rect: .zero,
+            options: [.mouseEnteredAndExited, .activeAlways, .inVisibleRect],
+            owner: self,
+            userInfo: nil
+        ))
+    }
+
+    required init?(coder: NSCoder) { nil }
+
+    override func mouseEntered(with event: NSEvent) {
+        NSAnimationContext.runAnimationGroup { ctx in
+            ctx.duration = 0.12
+            animator().alphaValue = 0.75
+        }
+    }
+
+    override func mouseExited(with event: NSEvent) {
+        NSAnimationContext.runAnimationGroup { ctx in
+            ctx.duration = 0.12
+            animator().alphaValue = 1.0
+        }
+    }
+
+    override func mouseDown(with event: NSEvent) {
+        guard let image else { return }
+        let appURL = Bundle.main.bundleURL
+        let item = NSPasteboardItem()
+        item.setString(appURL.absoluteString, forType: .fileURL)
+        let dragItem = NSDraggingItem(pasteboardWriter: item)
+        dragItem.setDraggingFrame(bounds, contents: image)
+        beginDraggingSession(with: [dragItem], event: event, source: self)
+    }
+
+    func draggingSession(
+        _ session: NSDraggingSession,
+        sourceOperationMaskFor context: NSDraggingContext
+    ) -> NSDragOperation {
+        .copy
+    }
+}
+
+

packages/handsfree/native/HandsFree/Sources/ComputerUse/Types.swift

@@ -0,0 +1,248 @@
+import AppKit
+import ApplicationServices
+
+struct ElementFrame: Sendable {
+    let x: Int
+    let y: Int
+    let width: Int
+    let height: Int
+
+    var center: CGPoint {
+        CGPoint(x: x + width / 2, y: y + height / 2)
+    }
+
+    var dictionary: [String: Any] {
+        ["x": x, "y": y, "width": width, "height": height]
+    }
+}
+
+struct AXElementState: Sendable {
+    let enabled: Bool?
+    let focused: Bool?
+    let selected: Bool?
+    let expanded: Bool?
+    let checked: Bool?
+
+    var dictionary: [String: Any] {
+        var result: [String: Any] = [:]
+        if let enabled { result["enabled"] = enabled }
+        if let focused { result["focused"] = focused }
+        if let selected { result["selected"] = selected }
+        if let expanded { result["expanded"] = expanded }
+        if let checked { result["checked"] = checked }
+        return result
+    }
+}
+
+struct AXElementCapabilities: Sendable {
+    let canPress: Bool
+    let canFocus: Bool
+    let canScroll: Bool
+    let canAdjust: Bool
+    let canSetValue: Bool
+    let actions: [String]
+
+    var dictionary: [String: Any] {
+        [
+            "press": canPress,
+            "focus": canFocus,
+            "scroll": canScroll,
+            "adjust": canAdjust,
+            "setValue": canSetValue,
+            "actions": actions,
+        ]
+    }
+}
+
+struct SemanticAXElement: Identifiable, Sendable {
+    let id: Int
+    let ref: String
+    let role: String
+    let label: String
+    let value: String?
+    let frame: ElementFrame
+    let state: AXElementState
+    let capabilities: AXElementCapabilities
+
+    var dictionary: [String: Any] {
+        var result: [String: Any] = [
+            "id": id,
+            "ref": ref,
+            "role": role,
+            "label": label,
+            "frame": frame.dictionary,
+            "state": state.dictionary,
+            "capabilities": capabilities.dictionary,
+        ]
+        if let value { result["value"] = value }
+        return result
+    }
+}
+
+struct AXElementRecord: @unchecked Sendable {
+    let element: AXUIElement
+    let semantic: SemanticAXElement
+}
+
+struct ScreenshotMetadata: Sendable {
+    let imageWidth: Int
+    let imageHeight: Int
+    let capturedBounds: CGRect
+
+    var scaleX: CGFloat { capturedBounds.width / CGFloat(imageWidth) }
+    var scaleY: CGFloat { capturedBounds.height / CGFloat(imageHeight) }
+
+    func toScreen(imageX: Double, imageY: Double) -> CGPoint {
+        CGPoint(
+            x: capturedBounds.origin.x + imageX * scaleX,
+            y: capturedBounds.origin.y + imageY * scaleY
+        )
+    }
+
+    func toImage(point: CGPoint) -> CGPoint {
+        CGPoint(
+            x: (point.x - capturedBounds.origin.x) / scaleX,
+            y: (point.y - capturedBounds.origin.y) / scaleY
+        )
+    }
+
+    var dictionary: [String: Any] {
+        [
+            "imageWidth": imageWidth,
+            "imageHeight": imageHeight,
+            "capturedBounds": [
+                "x": Int(capturedBounds.origin.x),
+                "y": Int(capturedBounds.origin.y),
+                "width": Int(capturedBounds.width),
+                "height": Int(capturedBounds.height),
+            ],
+        ]
+    }
+}
+
+struct WindowTarget: @unchecked Sendable {
+    let appName: String
+    let pid: pid_t
+    let windowNumber: Int?
+    let windowTitle: String?
+    let bounds: CGRect
+    let isFrontmost: Bool
+    let axWindow: AXUIElement?
+
+    var center: CGPoint {
+        CGPoint(x: bounds.midX, y: bounds.midY)
+    }
+}
+
+struct AppSnapshot: @unchecked Sendable {
+    let id: String
+    let observation: Int
+    let appName: String
+    let pid: pid_t
+    let windowNumber: Int?
+    let windowTitle: String?
+    let screenshotData: Data
+    let screenshotMimeType: String
+    let screenshotMeta: ScreenshotMetadata
+    let records: [AXElementRecord]
+    let strictMode: Bool
+    let backgroundActivated: Bool
+    let recentActions: [String]
+    let addedLabels: [String]
+    let removedLabels: [String]
+
+    var elements: [SemanticAXElement] {
+        records.map(\.semantic)
+    }
+
+    func withSessionMetadata(id: String, observation: Int, recentActions: [String], addedLabels: [String], removedLabels: [String]) -> AppSnapshot {
+        AppSnapshot(
+            id: id,
+            observation: observation,
+            appName: appName,
+            pid: pid,
+            windowNumber: windowNumber,
+            windowTitle: windowTitle,
+            screenshotData: screenshotData,
+            screenshotMimeType: screenshotMimeType,
+            screenshotMeta: screenshotMeta,
+            records: records,
+            strictMode: strictMode,
+            backgroundActivated: backgroundActivated,
+            recentActions: recentActions,
+            addedLabels: addedLabels,
+            removedLabels: removedLabels
+        )
+    }
+}
+
+enum ExecutionPath: String {
+    case accessibility = "accessibility"
+    case backgroundCGEvent = "background_cgevent"
+    case foregroundCGEvent = "foreground_cgevent"
+    case none = "none"
+}
+
+struct ActionMetadata: Sendable {
+    let ok: Bool
+    let path: ExecutionPath
+    let strictMode: Bool
+    let backgroundSafe: Bool
+    let fallbackUsed: Bool
+    let message: String
+
+    var dictionary: [String: Any] {
+        [
+            "ok": ok,
+            "path": path.rawValue,
+            "strictMode": strictMode,
+            "backgroundSafe": backgroundSafe,
+            "fallbackUsed": fallbackUsed,
+            "message": message,
+        ]
+    }
+}
+
+enum ComputerUseError: LocalizedError {
+    case accessibilityDenied
+    case screenshotFailed
+    case appNotFound(String)
+    case noFrontmostApplication
+    case noWindow(String)
+    case noSnapshot
+    case staleSnapshot(String)
+    case invalidElement(String)
+    case strictModeViolation(String)
+    case eventSourceFailed
+    case eventCreationFailed
+    case unknownKey(String)
+
+    var errorDescription: String? {
+        switch self {
+        case .accessibilityDenied:
+            return "Accessibility permission is not granted."
+        case .screenshotFailed:
+            return "Screenshot capture failed."
+        case .appNotFound(let name):
+            return "App '\(name)' is not running."
+        case .noFrontmostApplication:
+            return "No frontmost application found."
+        case .noWindow(let app):
+            return "No usable window found for \(app)."
+        case .noSnapshot:
+            return "No snapshot is available. Call snapshot first."
+        case .staleSnapshot(let reason):
+            return "The last snapshot is stale. \(reason)"
+        case .invalidElement(let ref):
+            return "Element \(ref) was not found in the last semantic snapshot."
+        case .strictModeViolation(let reason):
+            return "Strict background mode rejected the action: \(reason)"
+        case .eventSourceFailed:
+            return "Failed to create CGEventSource."
+        case .eventCreationFailed:
+            return "Failed to create CGEvent."
+        case .unknownKey(let key):
+            return "Unknown key: \(key)"
+        }
+    }
+}

packages/handsfree/native/HandsFree/Sources/ComputerUse/main.swift

@@ -0,0 +1,65 @@
+/// Computer Use: semantic AX and background-safe macOS computer use.
+///
+/// Four modes:
+///   mcp          — run the MCP server over stdio
+///   --check      — print permission status as JSON to stdout and exit
+///   --list-apps  — print running regular apps as JSON to stdout and exit
+///   (default)    — open the permission setup GUI
+
+import AppKit
+import Foundation
+
+setbuf(stdout, nil)
+
+let args = CommandLine.arguments
+let subcommand = args.count >= 2 ? args[1] : ""
+
+switch subcommand {
+case "mcp":
+    if ProcessInfo.processInfo.environment["OPENWORK_COMPUTER_USE_CURSOR_OVERLAY"] == "0" {
+        let server = MCPServer()
+        await server.run()
+    } else {
+        await runMCPServerWithOverlay()
+    }
+case "--check":
+    // Fresh process → fresh TCC read → always accurate.
+    let status = ComputerUsePermissions.status()
+    let json = "{\"ok\":\(status.ok),\"accessibility\":\(status.accessibility),\"screenRecording\":\(status.screenRecording)}"
+    print(json)
+    exit(0)
+case "--list-apps":
+    // Running regular apps (Dock-visible). Needs no TCC permissions, so this
+    // is safe to call from the composer at any time.
+    let apps = NSWorkspace.shared.runningApplications
+        .filter { $0.activationPolicy == .regular }
+        .compactMap(\.localizedName)
+        .sorted { $0.localizedCaseInsensitiveCompare($1) == .orderedAscending }
+    let payload: [String: Any] = ["ok": true, "apps": apps]
+    let data = (try? JSONSerialization.data(withJSONObject: payload)) ?? Data("{\"ok\":false,\"apps\":[]}".utf8)
+    print(String(decoding: data, as: UTF8.self))
+    exit(0)
+default:
+    await runPermissionSetupApp()
+}
+
+@MainActor
+func runMCPServerWithOverlay() async {
+    NSApplication.shared.setActivationPolicy(.accessory)
+    let server = MCPServer()
+    Task.detached {
+        await server.run()
+        await MainActor.run {
+            NSApplication.shared.terminate(nil)
+        }
+    }
+    NSApplication.shared.run()
+}
+
+@MainActor
+func runPermissionSetupApp() async {
+    NSApplication.shared.setActivationPolicy(.regular)
+    let appDelegate = PermissionSetupAppDelegate()
+    NSApplication.shared.delegate = appDelegate
+    NSApplication.shared.run()
+}