源码快照 @ 2026-09-02 (source-only, 剔除非源码素材)

packages/handsfree/README.md

@@ -0,0 +1,38 @@
+# OpenWork Computer Use
+
+Native macOS computer-use runtime for OpenWork.
+Packaged OpenWork builds wrap this runtime in a bundled `Computer Use.app`
+helper so macOS Accessibility and Screen Recording permissions belong to the
+helper app instead of a transient Node or Swift process.
+
+This package focuses on the reusable control layer:
+
+- Semantic AX snapshots with compact refs like `{e1}`.
+- Strict background mode that avoids foreground cursor/HID fallbacks.
+- Target-window screenshots via `CGWindowListCreateImage(.optionIncludingWindow)`.
+- Background input through `CGEvent.postToPid` with window-addressing fields.
+- Background activation using per-process event taps plus AppKit and center-click primers.
+- Non-UI orchestration modules from the original Electron prototype: realtime tool schemas/instructions and the GPT computer-use loop.
+
+Computer Use renders a lightweight second cursor overlay from the helper app so
+users can see where the agent is acting. The overlay is visual only: strict mode
+does not move the real system cursor. Actual control uses the same
+non-interrupting mechanism described in Bridge-style background computer use: AX
+first, then `CGEvent.postToPid` addressed to the target process/window, with
+per-process focus-message taps and activation primers so the user's frontmost app
+can remain frontmost while the target app accepts events. Foreground HID fallback
+is only used when strict mode is disabled.
+
+Build the native stdio server:
+
+```bash
+pnpm --filter @openwork/handsfree check:native
+```
+
+Run it as an MCP-compatible adapter:
+
+```bash
+pnpm --filter @openwork/handsfree exec openwork-handsfree-computer-use mcp
+```
+
+The core runtime is intentionally MCP-independent. `ComputerUseRuntime` exposes a small direct surface (`snapshot`, `click`, `typeText`, `pressKey`, `scroll`, `wait`, `setValue`, `performAction`); `MCPServer` is only a thin stdio wrapper.

packages/handsfree/bin/openwork-handsfree-computer-use.mjs

@@ -0,0 +1,41 @@
+#!/usr/bin/env node
+
+import { spawn } from "node:child_process";
+import { existsSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const packageRoot = path.resolve(__dirname, "..");
+const swiftPackagePath = path.join(packageRoot, "native", "HandsFree");
+
+const explicitBinary = process.env.HANDSFREE_COMPUTER_USE_BINARY?.trim();
+const candidates = [
+  explicitBinary,
+  path.join(swiftPackagePath, ".build", "release", "HandsFreeComputerUse"),
+  path.join(swiftPackagePath, ".build", "arm64-apple-macosx", "release", "HandsFreeComputerUse"),
+  path.join(swiftPackagePath, ".build", "debug", "HandsFreeComputerUse"),
+  path.join(swiftPackagePath, ".build", "arm64-apple-macosx", "debug", "HandsFreeComputerUse"),
+].filter(Boolean);
+
+const args = process.argv.slice(2);
+const binary = candidates.find((candidate) => existsSync(candidate));
+const command = binary ?? "swift";
+const commandArgs = binary
+  ? args
+  : ["run", "--package-path", swiftPackagePath, "HandsFreeComputerUse", ...args];
+
+const child = spawn(command, commandArgs, {
+  stdio: "inherit",
+  env: process.env,
+});
+
+child.on("exit", (code, signal) => {
+  if (signal) process.kill(process.pid, signal);
+  process.exit(code ?? 0);
+});
+
+child.on("error", (error) => {
+  console.error(`Failed to start HandsFreeComputerUse: ${error.message}`);
+  process.exit(1);
+});

packages/handsfree/native/HandsFree/.gitignore

@@ -0,0 +1,2 @@
+.build/
+.swiftpm/

packages/handsfree/native/HandsFree/Package.swift

@@ -0,0 +1,13 @@
+// swift-tools-version: 5.9
+import PackageDescription
+
+let package = Package(
+    name: "HandsFree",
+    platforms: [.macOS(.v14)],
+    targets: [
+        .executableTarget(
+            name: "HandsFreeComputerUse",
+            path: "Sources/ComputerUse"
+        ),
+    ]
+)

packages/handsfree/native/HandsFree/Sources/ComputerUse/AccessibilityService.swift

@@ -0,0 +1,451 @@
+import AppKit
+import ApplicationServices
+import ScreenCaptureKit
+
+final class AccessibilityService: @unchecked Sendable {
+    private let screenshotImageWidth: CGFloat = 768
+    private let maxElements = 250
+    private let maxDepth = 22
+
+    private let importantRoles: Set<String> = [
+        "AXButton", "AXCheckBox", "AXRadioButton", "AXPopUpButton", "AXMenuButton",
+        "AXComboBox", "AXTextField", "AXTextArea", "AXSearchField", "AXLink",
+        "AXSlider", "AXIncrementor", "AXScrollArea", "AXScrollBar", "AXTabGroup",
+        "AXTab", "AXMenuItem", "AXCell", "AXRow", "AXStaticText", "AXImage",
+        "AXOutline", "AXTable", "AXList", "AXGroup",
+    ]
+
+    private let enhancedLock = NSLock()
+    private var enhancedPIDs = Set<pid_t>()
+
+    func resolveTarget(appName: String?, pid requestedPID: pid_t? = nil, windowTitle: String? = nil) throws -> WindowTarget {
+        guard AXIsProcessTrusted() else { throw ComputerUseError.accessibilityDenied }
+
+        let app = try resolveApp(appName: appName, pid: requestedPID)
+        let pid = app.processIdentifier
+        let axApp = AXUIElementCreateApplication(pid)
+        if enableEnhancedAccessibility(axApp: axApp, pid: pid) {
+            // Chromium and Electron build the renderer AX tree lazily; give it a moment.
+            Thread.sleep(forTimeInterval: 0.35)
+        }
+        let axWindow = firstAXWindow(axApp: axApp, title: windowTitle)
+        let title = axWindow.flatMap { axString($0, kAXTitleAttribute) }
+        let info = firstCGWindowInfo(pid: pid, title: title)
+        let bounds = axWindow.flatMap(axFrame) ?? info?.bounds
+
+        guard let bounds, bounds.width > 20, bounds.height > 20 else {
+            throw ComputerUseError.noWindow(app.localizedName ?? appName ?? "frontmost app")
+        }
+
+        return WindowTarget(
+            appName: app.localizedName ?? "Unknown",
+            pid: pid,
+            windowNumber: info?.number,
+            windowTitle: title ?? info?.title,
+            bounds: bounds,
+            isFrontmost: NSWorkspace.shared.frontmostApplication?.processIdentifier == pid,
+            axWindow: axWindow
+        )
+    }
+
+    func snapshot(target: WindowTarget, strictMode: Bool, backgroundActivated: Bool) async throws -> AppSnapshot {
+        let records = records(target: target)
+        let (data, meta) = try await captureScreenshot(target: target)
+
+        return AppSnapshot(
+            id: "",
+            observation: 0,
+            appName: target.appName,
+            pid: target.pid,
+            windowNumber: target.windowNumber,
+            windowTitle: target.windowTitle,
+            screenshotData: data,
+            screenshotMimeType: "image/jpeg",
+            screenshotMeta: meta,
+            records: records,
+            strictMode: strictMode,
+            backgroundActivated: backgroundActivated,
+            recentActions: [],
+            addedLabels: [],
+            removedLabels: []
+        )
+    }
+
+    func records(target: WindowTarget) -> [AXElementRecord] {
+        target.axWindow.map(semanticRecords(window:)) ?? []
+    }
+
+    func press(record: AXElementRecord) -> Bool {
+        AXUIElementPerformAction(record.element, kAXPressAction as CFString) == .success
+    }
+
+    func focus(record: AXElementRecord) -> Bool {
+        AXUIElementSetAttributeValue(record.element, kAXFocusedAttribute as CFString, true as CFBoolean) == .success
+    }
+
+    func setValue(record: AXElementRecord, value: String) -> Bool {
+        var settable = DarwinBoolean(false)
+        guard AXUIElementIsAttributeSettable(record.element, kAXValueAttribute as CFString, &settable) == .success,
+              settable.boolValue else {
+            return false
+        }
+        return AXUIElementSetAttributeValue(record.element, kAXValueAttribute as CFString, value as CFString) == .success
+    }
+
+    func performAction(record: AXElementRecord, action: String) -> Bool {
+        AXUIElementPerformAction(record.element, action as CFString) == .success
+    }
+
+    func value(record: AXElementRecord) -> String? {
+        axString(record.element, kAXValueAttribute)
+    }
+
+    /// Chromium browsers and Electron apps only expose web content to AX after an
+    /// assistive client announces itself. VoiceOver does this by setting
+    /// AXEnhancedUserInterface; Electron additionally supports AXManualAccessibility.
+    /// Returns true when the attribute was newly applied to this pid.
+    private func enableEnhancedAccessibility(axApp: AXUIElement, pid: pid_t) -> Bool {
+        enhancedLock.lock()
+        let isNew = enhancedPIDs.insert(pid).inserted
+        enhancedLock.unlock()
+        guard isNew else { return false }
+        // Chromium reacts to the set attempt even though it reports an AXError
+        // (.notImplemented / .attributeUnsupported), so the return codes are
+        // logged for diagnostics but intentionally not treated as failure.
+        let enhanced = AXUIElementSetAttributeValue(axApp, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue)
+        let manual = AXUIElementSetAttributeValue(axApp, "AXManualAccessibility" as CFString, kCFBooleanTrue)
+        fputs("[ComputerUse] enhanced-ax pid=\(pid) AXEnhancedUserInterface=\(enhanced.rawValue) AXManualAccessibility=\(manual.rawValue)\n", stderr)
+        return true
+    }
+
+    private func resolveApp(appName: String?, pid: pid_t?) throws -> NSRunningApplication {
+        if let pid {
+            guard let app = NSRunningApplication(processIdentifier: pid) else {
+                throw ComputerUseError.appNotFound("pid \(pid)")
+            }
+            return app
+        }
+        guard let rawName = appName?.trimmingCharacters(in: .whitespacesAndNewlines), !rawName.isEmpty else {
+            guard let frontmost = NSWorkspace.shared.frontmostApplication else {
+                throw ComputerUseError.noFrontmostApplication
+            }
+            return frontmost
+        }
+
+        let needle = rawName.lowercased()
+        let regularApps = NSWorkspace.shared.runningApplications.filter { $0.activationPolicy == .regular }
+        if let exact = regularApps.first(where: { $0.localizedName?.lowercased() == needle }) {
+            return exact
+        }
+        if let contains = regularApps.first(where: { $0.localizedName?.lowercased().contains(needle) == true }) {
+            return contains
+        }
+        throw ComputerUseError.appNotFound(rawName)
+    }
+
+    private func firstAXWindow(axApp: AXUIElement, title preferredTitle: String?) -> AXUIElement? {
+        var windowValue: CFTypeRef?
+        guard AXUIElementCopyAttributeValue(axApp, kAXWindowsAttribute as CFString, &windowValue) == .success,
+              let windows = windowValue as? [AXUIElement] else {
+            return nil
+        }
+        let usable = windows.filter { window in
+            guard let frame = axFrame(window) else { return false }
+            return frame.width > 20 && frame.height > 20
+        }
+        if let preferredTitle, let match = usable.first(where: { axString($0, kAXTitleAttribute) == preferredTitle }) {
+            return match
+        }
+        return usable.first
+    }
+
+    private func semanticRecords(window: AXUIElement) -> [AXElementRecord] {
+        var records: [AXElementRecord] = []
+        collect(element: window, depth: 0, records: &records)
+        return records
+    }
+
+    private func collect(element: AXUIElement, depth: Int, records: inout [AXElementRecord]) {
+        guard depth <= maxDepth, records.count < maxElements else { return }
+
+        let rawRole = axString(element, kAXRoleAttribute) ?? "AXUnknown"
+        let role = normalizedRole(rawRole)
+        let value = axString(element, kAXValueAttribute).map { String($0.prefix(120)) }
+        let label = semanticLabel(element: element, role: role, value: value)
+        let actions = axActions(element)
+        let frame = axFrame(element)
+        let capabilities = capabilitiesFor(element: element, rawRole: rawRole, actions: actions)
+        let shouldSurface = shouldSurfaceElement(rawRole: rawRole, label: label, value: value, frame: frame, capabilities: capabilities)
+
+        if shouldSurface, let frame {
+            let id = records.count + 1
+            let semantic = SemanticAXElement(
+                id: id,
+                ref: "{e\(id)}",
+                role: role,
+                label: label,
+                value: value,
+                frame: ElementFrame(
+                    x: Int(frame.origin.x),
+                    y: Int(frame.origin.y),
+                    width: Int(frame.width),
+                    height: Int(frame.height)
+                ),
+                state: stateFor(element: element, rawRole: rawRole),
+                capabilities: capabilities
+            )
+            records.append(AXElementRecord(element: element, semantic: semantic))
+        }
+
+        for child in axChildren(element) {
+            collect(element: child, depth: depth + 1, records: &records)
+            if records.count >= maxElements { break }
+        }
+    }
+
+    private func shouldSurfaceElement(rawRole: String, label: String, value: String?, frame: CGRect?, capabilities: AXElementCapabilities) -> Bool {
+        guard let frame, frame.width > 1, frame.height > 1 else { return false }
+        let hasSemanticText = !label.isEmpty || value?.isEmpty == false
+        let interactive = capabilities.canPress || capabilities.canFocus || capabilities.canScroll || capabilities.canAdjust || capabilities.canSetValue
+        if interactive { return true }
+        if !importantRoles.contains(rawRole) { return false }
+        if rawRole == "AXGroup" { return hasSemanticText && frame.width < 900 && frame.height < 700 }
+        return hasSemanticText
+    }
+
+    private func semanticLabel(element: AXUIElement, role: String, value: String?) -> String {
+        let candidates = [
+            axString(element, kAXTitleAttribute),
+            axString(element, kAXDescriptionAttribute),
+            axString(element, kAXHelpAttribute),
+            axString(element, kAXIdentifierAttribute),
+            value,
+        ]
+        for candidate in candidates {
+            if let trimmed = candidate?.trimmingCharacters(in: .whitespacesAndNewlines), !trimmed.isEmpty {
+                return String(trimmed.prefix(120))
+            }
+        }
+        return role
+    }
+
+    private func capabilitiesFor(element: AXUIElement, rawRole: String, actions: [String]) -> AXElementCapabilities {
+        var valueSettable = DarwinBoolean(false)
+        let canSetValue = AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &valueSettable) == .success && valueSettable.boolValue
+
+        var focusSettable = DarwinBoolean(false)
+        let canFocus = AXUIElementIsAttributeSettable(element, kAXFocusedAttribute as CFString, &focusSettable) == .success && focusSettable.boolValue
+
+        let canAdjust = actions.contains(kAXIncrementAction) || actions.contains(kAXDecrementAction) || rawRole == "AXSlider" || rawRole == "AXIncrementor"
+        let canScroll = actions.contains("AXScrollToVisible") || rawRole == "AXScrollArea" || rawRole == "AXScrollBar"
+        let canPress = actions.contains(kAXPressAction) || ["AXButton", "AXCheckBox", "AXRadioButton", "AXLink", "AXMenuItem", "AXPopUpButton", "AXMenuButton", "AXCell"].contains(rawRole)
+
+        return AXElementCapabilities(
+            canPress: canPress,
+            canFocus: canFocus,
+            canScroll: canScroll,
+            canAdjust: canAdjust,
+            canSetValue: canSetValue,
+            actions: actions
+        )
+    }
+
+    private func stateFor(element: AXUIElement, rawRole: String) -> AXElementState {
+        AXElementState(
+            enabled: axBool(element, kAXEnabledAttribute),
+            focused: axBool(element, kAXFocusedAttribute),
+            selected: axBool(element, kAXSelectedAttribute),
+            expanded: axBool(element, kAXExpandedAttribute),
+            checked: rawRole == "AXCheckBox" || rawRole == "AXRadioButton" ? axBool(element, kAXValueAttribute) : nil
+        )
+    }
+
+    private func axChildren(_ element: AXUIElement) -> [AXUIElement] {
+        var value: CFTypeRef?
+        guard AXUIElementCopyAttributeValue(element, kAXChildrenAttribute as CFString, &value) == .success,
+              let children = value as? [AXUIElement] else {
+            return []
+        }
+        return children
+    }
+
+    private func axActions(_ element: AXUIElement) -> [String] {
+        var actionNames: CFArray?
+        guard AXUIElementCopyActionNames(element, &actionNames) == .success,
+              let names = actionNames as? [String] else {
+            return []
+        }
+        return names
+    }
+
+    private func axString(_ element: AXUIElement, _ attribute: String) -> String? {
+        var value: CFTypeRef?
+        guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success,
+              let value else {
+            return nil
+        }
+        if let string = value as? String, !string.isEmpty { return string }
+        if let attributed = value as? NSAttributedString, !attributed.string.isEmpty { return attributed.string }
+        if let number = value as? NSNumber { return number.stringValue }
+        return nil
+    }
+
+    private func axBool(_ element: AXUIElement, _ attribute: String) -> Bool? {
+        var value: CFTypeRef?
+        guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success,
+              let value else {
+            return nil
+        }
+        if let bool = value as? Bool { return bool }
+        if let number = value as? NSNumber { return number.boolValue }
+        return nil
+    }
+
+    private func axFrame(_ element: AXUIElement) -> CGRect? {
+        guard let position = axPoint(element, kAXPositionAttribute), let size = axSize(element, kAXSizeAttribute) else {
+            return nil
+        }
+        return CGRect(origin: position, size: size)
+    }
+
+    private func axPoint(_ element: AXUIElement, _ attribute: String) -> CGPoint? {
+        var value: CFTypeRef?
+        guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success,
+              let value,
+              CFGetTypeID(value) == AXValueGetTypeID() else {
+            return nil
+        }
+        let axValue = value as! AXValue
+        guard AXValueGetType(axValue) == .cgPoint else { return nil }
+        var point = CGPoint.zero
+        guard AXValueGetValue(axValue, .cgPoint, &point) else { return nil }
+        return point
+    }
+
+    private func axSize(_ element: AXUIElement, _ attribute: String) -> CGSize? {
+        var value: CFTypeRef?
+        guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success,
+              let value,
+              CFGetTypeID(value) == AXValueGetTypeID() else {
+            return nil
+        }
+        let axValue = value as! AXValue
+        guard AXValueGetType(axValue) == .cgSize else { return nil }
+        var size = CGSize.zero
+        guard AXValueGetValue(axValue, .cgSize, &size) else { return nil }
+        return size
+    }
+
+    private func normalizedRole(_ rawRole: String) -> String {
+        rawRole.hasPrefix("AX") ? String(rawRole.dropFirst(2)) : rawRole
+    }
+
+    private func firstCGWindowInfo(pid: pid_t, title: String?) -> (number: Int, title: String?, bounds: CGRect)? {
+        guard let list = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else {
+            return nil
+        }
+
+        let candidates = list.compactMap { info -> (number: Int, title: String?, bounds: CGRect)? in
+            guard let ownerPID = info[kCGWindowOwnerPID as String] as? Int32,
+                  ownerPID == pid,
+                  let layer = info[kCGWindowLayer as String] as? Int,
+                  layer == 0,
+                  let number = info[kCGWindowNumber as String] as? Int,
+                  let boundsDict = info[kCGWindowBounds as String] as? [String: Any] else {
+                return nil
+            }
+            let bounds = CGRect(
+                x: cgFloat(boundsDict["X"]),
+                y: cgFloat(boundsDict["Y"]),
+                width: cgFloat(boundsDict["Width"]),
+                height: cgFloat(boundsDict["Height"])
+            )
+            guard bounds.width > 20, bounds.height > 20 else { return nil }
+            return (number, info[kCGWindowName as String] as? String, bounds)
+        }
+
+        if let title, let exact = candidates.first(where: { $0.title == title }) {
+            return exact
+        }
+        return candidates.first
+    }
+
+    private func captureScreenshot(target: WindowTarget) async throws -> (Data, ScreenshotMetadata) {
+        let cgImage = await screenCaptureKitImage(target: target) ?? legacyCaptureImage(target: target)
+        guard let cgImage else { throw ComputerUseError.screenshotFailed }
+
+        return try encodeScreenshot(cgImage, capturedBounds: target.bounds)
+    }
+
+    private func legacyCaptureImage(target: WindowTarget) -> CGImage? {
+        if let windowNumber = target.windowNumber {
+            return CGWindowListCreateImage(
+                CGRect.null,
+                .optionIncludingWindow,
+                CGWindowID(windowNumber),
+                [.bestResolution, .boundsIgnoreFraming]
+            )
+        }
+
+        return CGWindowListCreateImage(target.bounds, .optionOnScreenOnly, kCGNullWindowID, [.bestResolution])
+    }
+
+    private func screenCaptureKitImage(target: WindowTarget) async -> CGImage? {
+        guard let windowNumber = target.windowNumber else { return nil }
+        do {
+            let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)
+            guard let window = content.windows.first(where: { Int($0.windowID) == windowNumber }) else { return nil }
+            let configuration = SCStreamConfiguration()
+            let scale = screenScale(for: target.bounds)
+            configuration.width = max(1, Int(target.bounds.width * scale))
+            configuration.height = max(1, Int(target.bounds.height * scale))
+            configuration.showsCursor = false
+            let filter = SCContentFilter(desktopIndependentWindow: window)
+            return try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: configuration)
+        } catch {
+            return nil
+        }
+    }
+
+    private func screenScale(for bounds: CGRect) -> CGFloat {
+        NSScreen.screens.first(where: { $0.frame.intersects(bounds) })?.backingScaleFactor
+            ?? NSScreen.main?.backingScaleFactor
+            ?? 2
+    }
+
+    private func encodeScreenshot(_ cgImage: CGImage, capturedBounds: CGRect) throws -> (Data, ScreenshotMetadata) {
+        let rawWidth = CGFloat(cgImage.width)
+        let rawHeight = CGFloat(cgImage.height)
+        let targetWidth = min(screenshotImageWidth, rawWidth)
+        let targetHeight = rawHeight * (targetWidth / rawWidth)
+
+        let source = NSImage(cgImage: cgImage, size: NSSize(width: rawWidth, height: rawHeight))
+        let resized = NSImage(size: NSSize(width: targetWidth, height: targetHeight))
+        resized.lockFocus()
+        source.draw(in: NSRect(x: 0, y: 0, width: targetWidth, height: targetHeight))
+        resized.unlockFocus()
+
+        guard let tiff = resized.tiffRepresentation,
+              let rep = NSBitmapImageRep(data: tiff),
+              let jpeg = rep.representation(using: .jpeg, properties: [.compressionFactor: 0.45]) else {
+            throw ComputerUseError.screenshotFailed
+        }
+
+        return (
+            jpeg,
+            ScreenshotMetadata(
+                imageWidth: Int(targetWidth),
+                imageHeight: Int(targetHeight),
+                capturedBounds: capturedBounds
+            )
+        )
+    }
+
+    private func cgFloat(_ value: Any?) -> CGFloat {
+        if let value = value as? CGFloat { return value }
+        if let value = value as? Double { return CGFloat(value) }
+        if let value = value as? Int { return CGFloat(value) }
+        if let value = value as? NSNumber { return CGFloat(truncating: value) }
+        return 0
+    }
+}

packages/handsfree/native/HandsFree/Sources/ComputerUse/AgentCursorOverlay.swift

@@ -0,0 +1,104 @@
+import AppKit
+
+final class AgentCursorOverlay: @unchecked Sendable {
+    static let shared = AgentCursorOverlay()
+
+    private var panel: AgentCursorPanel?
+    private var hideWorkItem: DispatchWorkItem?
+    private let size = CGSize(width: 30, height: 34)
+
+    func show(at point: CGPoint) {
+        DispatchQueue.main.async { [weak self] in
+            self?.showOnMain(at: point)
+        }
+    }
+
+    private func showOnMain(at point: CGPoint) {
+        let panel = ensurePanel()
+        panel.setFrame(frame(for: point), display: true)
+        panel.orderFrontRegardless()
+
+        hideWorkItem?.cancel()
+        let nextHide = DispatchWorkItem { [weak panel] in
+            panel?.orderOut(nil)
+        }
+        hideWorkItem = nextHide
+        DispatchQueue.main.asyncAfter(deadline: .now() + 1.5, execute: nextHide)
+    }
+
+    private func ensurePanel() -> AgentCursorPanel {
+        if let panel { return panel }
+
+        let panel = AgentCursorPanel(
+            contentRect: NSRect(origin: .zero, size: size),
+            styleMask: [.borderless, .nonactivatingPanel],
+            backing: .buffered,
+            defer: false
+        )
+        panel.backgroundColor = .clear
+        panel.isOpaque = false
+        panel.hasShadow = false
+        panel.ignoresMouseEvents = true
+        panel.level = .floating
+        panel.collectionBehavior = [.canJoinAllSpaces, .stationary, .ignoresCycle, .fullScreenAuxiliary]
+        panel.contentView = AgentCursorView(frame: NSRect(origin: .zero, size: size))
+        self.panel = panel
+        return panel
+    }
+
+    private func frame(for topLeftPoint: CGPoint) -> NSRect {
+        let screen = screenContaining(topLeftPoint) ?? NSScreen.main
+        let screenFrame = screen?.frame ?? NSRect(x: 0, y: 0, width: 1440, height: 900)
+        let x = screenFrame.minX + topLeftPoint.x - 2
+        let y = screenFrame.maxY - topLeftPoint.y - size.height + 2
+        return NSRect(x: x, y: y, width: size.width, height: size.height)
+    }
+
+    private func screenContaining(_ topLeftPoint: CGPoint) -> NSScreen? {
+        NSScreen.screens.first { screen in
+            let frame = screen.frame
+            return topLeftPoint.x >= frame.minX
+                && topLeftPoint.x <= frame.maxX
+                && topLeftPoint.y >= 0
+                && topLeftPoint.y <= frame.height
+        }
+    }
+}
+
+final class AgentCursorPanel: NSPanel {
+    override var canBecomeKey: Bool { false }
+    override var canBecomeMain: Bool { false }
+}
+
+final class AgentCursorView: NSView {
+    override var isOpaque: Bool { false }
+
+    override func draw(_ dirtyRect: NSRect) {
+        super.draw(dirtyRect)
+
+        let shadow = NSShadow()
+        shadow.shadowBlurRadius = 3
+        shadow.shadowOffset = NSSize(width: 0, height: -1)
+        shadow.shadowColor = NSColor.black.withAlphaComponent(0.35)
+        shadow.set()
+
+        let path = NSBezierPath()
+        path.move(to: NSPoint(x: 4, y: 30))
+        path.line(to: NSPoint(x: 4, y: 4))
+        path.line(to: NSPoint(x: 12, y: 11))
+        path.line(to: NSPoint(x: 17, y: 1))
+        path.line(to: NSPoint(x: 22, y: 4))
+        path.line(to: NSPoint(x: 17, y: 14))
+        path.line(to: NSPoint(x: 27, y: 14))
+        path.close()
+
+        NSColor.white.setFill()
+        path.fill()
+
+        shadow.shadowColor = .clear
+        shadow.set()
+        NSColor.systemBlue.setStroke()
+        path.lineWidth = 2
+        path.stroke()
+    }
+}

packages/handsfree/native/HandsFree/Sources/ComputerUse/BackgroundActivationSession.swift

@@ -0,0 +1,164 @@
+import AppKit
+import CoreGraphics
+
+final class BackgroundActivationSession: @unchecked Sendable {
+    private final class TapContext {
+        let suppressFocusMessages: Bool
+
+        init(suppressFocusMessages: Bool) {
+            self.suppressFocusMessages = suppressFocusMessages
+        }
+    }
+
+    private struct TapRef {
+        let tap: CFMachPort
+        let source: CFRunLoopSource
+        let context: TapContext
+    }
+
+    private let previousPID: pid_t
+    private let targetPID: pid_t
+    private var taps: [TapRef] = []
+    private var runLoop: CFRunLoop?
+    private var thread: Thread?
+    private var startupError: Error?
+    private var started = false
+    private var lastTarget: WindowTarget?
+
+    init(previousPID: pid_t, targetPID: pid_t) {
+        self.previousPID = previousPID
+        self.targetPID = targetPID
+    }
+
+    deinit {
+        stop()
+    }
+
+    func start() throws {
+        guard !started else { return }
+
+        let ready = DispatchSemaphore(value: 0)
+        let thread = Thread { [weak self] in
+            guard let self else {
+                ready.signal()
+                return
+            }
+            self.runLoop = CFRunLoopGetCurrent()
+            do {
+                try self.installTapsOnCurrentRunLoop()
+            } catch {
+                self.startupError = error
+            }
+            ready.signal()
+            if self.startupError == nil {
+                CFRunLoopRun()
+            }
+        }
+        thread.name = "OpenWorkBackgroundActivationSession"
+        self.thread = thread
+        thread.start()
+        ready.wait()
+
+        if let startupError {
+            throw startupError
+        }
+        started = true
+    }
+
+    func activate(target: WindowTarget) async throws {
+        guard let windowNumber = target.windowNumber else {
+            throw ComputerUseError.strictModeViolation("target window has no CG window number")
+        }
+
+        postAppKitDefined(subtype: 1, target: target, windowNumber: windowNumber)
+        try await Task.sleep(nanoseconds: 25_000_000)
+        try await BackgroundInputDispatcher.click(pid: target.pid, windowNumber: windowNumber, point: target.center)
+        try await Task.sleep(nanoseconds: 80_000_000)
+        lastTarget = target
+    }
+
+    func stop() {
+        if let target = lastTarget, let windowNumber = target.windowNumber {
+            postAppKitDefined(subtype: 2, target: target, windowNumber: windowNumber)
+        }
+        for tapRef in taps {
+            CFMachPortInvalidate(tapRef.tap)
+        }
+        taps.removeAll()
+        if let runLoop {
+            CFRunLoopStop(runLoop)
+        }
+        runLoop = nil
+        thread = nil
+        started = false
+    }
+
+    private func installTapsOnCurrentRunLoop() throws {
+        if previousPID != targetPID {
+            taps.append(try installTap(pid: previousPID, suppressFocusMessages: true))
+        }
+        taps.append(try installTap(pid: targetPID, suppressFocusMessages: false))
+    }
+
+    private func installTap(pid: pid_t, suppressFocusMessages: Bool) throws -> TapRef {
+        let context = TapContext(suppressFocusMessages: suppressFocusMessages)
+        let userInfo = Unmanaged.passUnretained(context).toOpaque()
+        guard let tap = CGEvent.tapCreateForPid(
+            pid: pid,
+            place: .headInsertEventTap,
+            options: .defaultTap,
+            eventsOfInterest: CGEventMask.max,
+            callback: BackgroundActivationSession.eventTapCallback,
+            userInfo: userInfo
+        ) else {
+            throw ComputerUseError.strictModeViolation("could not install per-process event tap for pid \(pid)")
+        }
+
+        guard let source = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, tap, 0) else {
+            CFMachPortInvalidate(tap)
+            throw ComputerUseError.strictModeViolation("could not create run loop source for pid \(pid) event tap")
+        }
+
+        CFRunLoopAddSource(CFRunLoopGetCurrent(), source, .commonModes)
+        CGEvent.tapEnable(tap: tap, enable: true)
+        return TapRef(tap: tap, source: source, context: context)
+    }
+
+    private func postAppKitDefined(subtype: Int16, target: WindowTarget, windowNumber: Int) {
+        guard let event = NSEvent.otherEvent(
+            with: .appKitDefined,
+            location: target.center,
+            modifierFlags: [],
+            timestamp: ProcessInfo.processInfo.systemUptime,
+            windowNumber: windowNumber,
+            context: nil,
+            subtype: subtype,
+            data1: 0,
+            data2: 0
+        )?.cgEvent else {
+            return
+        }
+
+        BackgroundInputDispatcher.address(event, pid: target.pid, windowNumber: windowNumber)
+        event.postToPid(target.pid)
+    }
+
+    private static let eventTapCallback: CGEventTapCallBack = { _, type, event, userInfo in
+        if type == .tapDisabledByTimeout || type == .tapDisabledByUserInput {
+            return Unmanaged.passUnretained(event)
+        }
+        guard let userInfo else {
+            return Unmanaged.passUnretained(event)
+        }
+        let context = Unmanaged<TapContext>.fromOpaque(userInfo).takeUnretainedValue()
+        if context.suppressFocusMessages && BackgroundActivationSession.isFocusMessage(type) {
+            return nil
+        }
+        return Unmanaged.passUnretained(event)
+    }
+
+    private static func isFocusMessage(_ type: CGEventType) -> Bool {
+        let raw = Int(type.rawValue)
+        return raw == 13 || raw == 19 || raw == 20
+    }
+}

packages/handsfree/native/HandsFree/Sources/ComputerUse/BackgroundInputDispatcher.swift

@@ -0,0 +1,163 @@
+import Foundation
+import CoreGraphics
+
+enum BackgroundInputDispatcher {
+    private static let privateWindowField = CGEventField(rawValue: 51)
+    private static let privateRouteField = CGEventField(rawValue: 58)
+
+    static func click(pid: pid_t, windowNumber: Int, point: CGPoint, doubleClick: Bool = false) async throws {
+        guard let source = CGEventSource(stateID: .combinedSessionState) else {
+            throw ComputerUseError.eventSourceFailed
+        }
+
+        let clickCount = doubleClick ? 2 : 1
+        for clickState in 1...clickCount {
+            guard let down = CGEvent(mouseEventSource: source, mouseType: .leftMouseDown, mouseCursorPosition: point, mouseButton: .left),
+                  let up = CGEvent(mouseEventSource: source, mouseType: .leftMouseUp, mouseCursorPosition: point, mouseButton: .left) else {
+                throw ComputerUseError.eventCreationFailed
+            }
+
+            address(down, pid: pid, windowNumber: windowNumber)
+            down.setIntegerValueField(.mouseEventClickState, value: Int64(clickState))
+            down.setDoubleValueField(.mouseEventPressure, value: 1)
+            down.postToPid(pid)
+
+            try await Task.sleep(nanoseconds: 30_000_000)
+
+            address(up, pid: pid, windowNumber: windowNumber)
+            up.setIntegerValueField(.mouseEventClickState, value: Int64(clickState))
+            up.setDoubleValueField(.mouseEventPressure, value: 0)
+            up.postToPid(pid)
+
+            if clickState < clickCount {
+                try await Task.sleep(nanoseconds: 50_000_000)
+            }
+        }
+    }
+
+    static func scroll(pid: pid_t, windowNumber: Int, point: CGPoint, deltaX: Int32, deltaY: Int32) throws {
+        guard let source = CGEventSource(stateID: .combinedSessionState) else {
+            throw ComputerUseError.eventSourceFailed
+        }
+        guard let event = CGEvent(scrollWheelEvent2Source: source, units: .line, wheelCount: 2, wheel1: deltaY, wheel2: deltaX, wheel3: 0) else {
+            throw ComputerUseError.eventCreationFailed
+        }
+        event.location = point
+        address(event, pid: pid, windowNumber: windowNumber)
+        event.postToPid(pid)
+    }
+
+    static func typeText(pid: pid_t, text: String) throws {
+        guard let source = CGEventSource(stateID: .combinedSessionState) else {
+            throw ComputerUseError.eventSourceFailed
+        }
+
+        let units = Array(text.utf16)
+        let chunkSize = 20
+        for start in stride(from: 0, to: units.count, by: chunkSize) {
+            let end = min(start + chunkSize, units.count)
+            let chunk = Array(units[start..<end])
+            guard let event = CGEvent(keyboardEventSource: source, virtualKey: 0, keyDown: true) else {
+                throw ComputerUseError.eventCreationFailed
+            }
+            event.setIntegerValueField(.eventTargetUnixProcessID, value: Int64(pid))
+            event.keyboardSetUnicodeString(stringLength: chunk.count, unicodeString: chunk)
+            event.postToPid(pid)
+            Thread.sleep(forTimeInterval: 0.01)
+        }
+    }
+
+    static func pressKey(pid: pid_t, combo: String) throws {
+        let parsed = try parseCombo(combo)
+        guard let source = CGEventSource(stateID: .combinedSessionState) else {
+            throw ComputerUseError.eventSourceFailed
+        }
+
+        // Post real modifier key events around the main key. Chromium and other
+        // apps ignore bare flag bits on background-posted events, so flags alone
+        // are not enough for combos like command+a.
+        let modifierKeys: [(flag: CGEventFlags, keyCode: CGKeyCode)] = [
+            (.maskCommand, 0x37), (.maskShift, 0x38), (.maskAlternate, 0x3A), (.maskControl, 0x3B),
+        ]
+        let activeModifiers = modifierKeys.filter { parsed.flags.contains($0.flag) }
+
+        var accumulated: CGEventFlags = []
+        for modifier in activeModifiers {
+            accumulated.insert(modifier.flag)
+            try postKey(source: source, pid: pid, keyCode: modifier.keyCode, keyDown: true, flags: accumulated)
+            Thread.sleep(forTimeInterval: 0.005)
+        }
+
+        try postKey(source: source, pid: pid, keyCode: parsed.keyCode, keyDown: true, flags: parsed.flags)
+        Thread.sleep(forTimeInterval: 0.01)
+        try postKey(source: source, pid: pid, keyCode: parsed.keyCode, keyDown: false, flags: parsed.flags)
+
+        for modifier in activeModifiers.reversed() {
+            accumulated.remove(modifier.flag)
+            Thread.sleep(forTimeInterval: 0.005)
+            try postKey(source: source, pid: pid, keyCode: modifier.keyCode, keyDown: false, flags: accumulated)
+        }
+    }
+
+    private static func postKey(source: CGEventSource, pid: pid_t, keyCode: CGKeyCode, keyDown: Bool, flags: CGEventFlags) throws {
+        guard let event = CGEvent(keyboardEventSource: source, virtualKey: keyCode, keyDown: keyDown) else {
+            throw ComputerUseError.eventCreationFailed
+        }
+        event.flags = flags
+        event.setIntegerValueField(.eventTargetUnixProcessID, value: Int64(pid))
+        event.postToPid(pid)
+    }
+
+    static func address(_ event: CGEvent, pid: pid_t, windowNumber: Int) {
+        event.setIntegerValueField(.eventTargetUnixProcessID, value: Int64(pid))
+        event.setIntegerValueField(.mouseEventWindowUnderMousePointer, value: Int64(windowNumber))
+        event.setIntegerValueField(.mouseEventWindowUnderMousePointerThatCanHandleThisEvent, value: Int64(windowNumber))
+        if let privateWindowField {
+            event.setIntegerValueField(privateWindowField, value: Int64(windowNumber))
+        }
+        if let privateRouteField {
+            event.setIntegerValueField(privateRouteField, value: 1)
+        }
+    }
+
+    static func parseCombo(_ combo: String) throws -> (flags: CGEventFlags, keyCode: CGKeyCode) {
+        let parts = combo.lowercased().split(separator: "+").map(String.init)
+        var flags: CGEventFlags = []
+        var keyName = ""
+
+        for part in parts {
+            switch part {
+            case "command", "cmd", "meta": flags.insert(.maskCommand)
+            case "shift": flags.insert(.maskShift)
+            case "control", "ctrl": flags.insert(.maskControl)
+            case "option", "alt": flags.insert(.maskAlternate)
+            default: keyName = part
+            }
+        }
+
+        guard let keyCode = keyCodes[keyName] else {
+            throw ComputerUseError.unknownKey(keyName)
+        }
+        return (flags, keyCode)
+    }
+
+    private static let keyCodes: [String: CGKeyCode] = [
+        "return": 0x24, "enter": 0x24, "tab": 0x30, "space": 0x31,
+        "delete": 0x33, "backspace": 0x33, "escape": 0x35, "esc": 0x35,
+        "up": 0x7E, "down": 0x7D, "left": 0x7B, "right": 0x7C,
+        "home": 0x73, "end": 0x77, "pageup": 0x74, "pagedown": 0x79,
+        "a": 0x00, "b": 0x0B, "c": 0x08, "d": 0x02, "e": 0x0E,
+        "f": 0x03, "g": 0x05, "h": 0x04, "i": 0x22, "j": 0x26,
+        "k": 0x28, "l": 0x25, "m": 0x2E, "n": 0x2D, "o": 0x1F,
+        "p": 0x23, "q": 0x0C, "r": 0x0F, "s": 0x01, "t": 0x11,
+        "u": 0x20, "v": 0x09, "w": 0x0D, "x": 0x07, "y": 0x10, "z": 0x06,
+        "0": 0x1D, "1": 0x12, "2": 0x13, "3": 0x14, "4": 0x15,
+        "5": 0x17, "6": 0x16, "7": 0x1A, "8": 0x1C, "9": 0x19,
+        "f1": 0x7A, "f2": 0x78, "f3": 0x63, "f4": 0x76,
+        "f5": 0x60, "f6": 0x61, "f7": 0x62, "f8": 0x64,
+        "f9": 0x65, "f10": 0x6D, "f11": 0x67, "f12": 0x6F,
+        "-": 0x1B, "=": 0x18, "[": 0x21, "]": 0x1E,
+        "\\": 0x2A, ";": 0x29, "'": 0x27, ",": 0x2B,
+        ".": 0x2F, "/": 0x2C, "`": 0x32,
+    ]
+}

packages/handsfree/native/HandsFree/Sources/ComputerUse/ComputerUseRuntime.swift

@@ -0,0 +1,383 @@
+import AppKit
+import Foundation
+
+actor ComputerUseRuntime {
+    private let accessibility = AccessibilityService()
+    private let foregroundInput = InputService()
+    private var lastSnapshot: AppSnapshot?
+    private var strictMode = true
+    private var activationSession: BackgroundActivationSession?
+    private var activationKey: String?
+    private var activatedWindowKey: String?
+    private var activationPreviousPID: pid_t?
+    private var activationTargetPID: pid_t?
+    private var frontmostMonitor: FrontmostApplicationMonitor?
+    private var snapshotSequence = 0
+    private var previousLabelsByWindow: [String: Set<String>] = [:]
+    private var recentActions: [String] = []
+    private var staleSnapshotReason: String?
+
+    func setStrictMode(_ enabled: Bool) -> ActionMetadata {
+        strictMode = enabled
+        if !enabled {
+            resetBackgroundActivation()
+        }
+        return ActionMetadata(
+            ok: true,
+            path: .none,
+            strictMode: enabled,
+            backgroundSafe: enabled,
+            fallbackUsed: false,
+            message: enabled ? "Strict background mode enabled." : "Strict background mode disabled. Foreground fallback is allowed."
+        )
+    }
+
+    func snapshot(appName: String?, pid: Int? = nil, windowTitle: String? = nil, strict requestedStrict: Bool?) async throws -> AppSnapshot {
+        let effectiveStrict = requestedStrict ?? strictMode
+        if !effectiveStrict {
+            resetBackgroundActivation()
+        }
+
+        let requestedPID = pid.map(pid_t.init)
+        var target = try accessibility.resolveTarget(appName: appName, pid: requestedPID, windowTitle: windowTitle)
+        let backgroundActivated: Bool
+        if effectiveStrict, !target.isFrontmost {
+            backgroundActivated = try await ensureBackgroundActivation(target: target)
+            target = try accessibility.resolveTarget(appName: appName, pid: requestedPID, windowTitle: windowTitle)
+        } else {
+            backgroundActivated = false
+        }
+
+        let snapshot = try await accessibility.snapshot(
+            target: target,
+            strictMode: effectiveStrict,
+            backgroundActivated: backgroundActivated
+        )
+        let annotatedSnapshot = annotate(snapshot: snapshot)
+        staleSnapshotReason = nil
+        lastSnapshot = annotatedSnapshot
+        return annotatedSnapshot
+    }
+
+    func click(snapshotID: String?, ref: String?, index: Int?, imageX: Double?, imageY: Double?, screenX: Double? = nil, screenY: Double? = nil, clickCount: Int, strict requestedStrict: Bool?) async throws -> ActionMetadata {
+        let snapshot = try requireSnapshot(snapshotID: snapshotID)
+        let effectiveStrict = requestedStrict ?? snapshot.strictMode
+        try validateSnapshotForAction(snapshot: snapshot, strict: effectiveStrict)
+
+        if let record = findRecord(ref: ref, index: index, in: snapshot) {
+            let fresh = freshRecord(matching: record, in: snapshot)
+            if let fresh {
+                AgentCursorOverlay.shared.show(at: fresh.semantic.frame.center)
+                if fresh.semantic.capabilities.canPress, accessibility.press(record: fresh) {
+                    return recordAction(ActionMetadata(ok: true, path: .accessibility, strictMode: effectiveStrict, backgroundSafe: true, fallbackUsed: false, message: "Pressed \(fresh.semantic.ref) via AXPress."))
+                }
+                if fresh.semantic.capabilities.canFocus, accessibility.focus(record: fresh) {
+                    return recordAction(ActionMetadata(ok: true, path: .accessibility, strictMode: effectiveStrict, backgroundSafe: true, fallbackUsed: false, message: "Focused \(fresh.semantic.ref) via AX."))
+                }
+                return try await clickPoint(fresh.semantic.frame.center, clickCount: clickCount, strict: effectiveStrict, fallbackUsed: true)
+            }
+            return try await clickPoint(record.semantic.frame.center, clickCount: clickCount, strict: effectiveStrict, fallbackUsed: true)
+        }
+
+        if let screenX, let screenY {
+            return try await clickPoint(CGPoint(x: screenX, y: screenY), clickCount: clickCount, strict: effectiveStrict, fallbackUsed: false)
+        }
+
+        if let imageX, let imageY {
+            let point = snapshot.screenshotMeta.toScreen(imageX: imageX, imageY: imageY)
+            return try await clickPoint(point, clickCount: clickCount, strict: effectiveStrict, fallbackUsed: false)
+        }
+
+        throw ComputerUseError.invalidElement(ref ?? index.map(String.init) ?? "<missing>")
+    }
+
+    func typeText(snapshotID: String?, text: String, strict requestedStrict: Bool?) throws -> ActionMetadata {
+        let snapshot = try requireSnapshot(snapshotID: snapshotID)
+        let effectiveStrict = requestedStrict ?? snapshot.strictMode
+        try validateSnapshotForAction(snapshot: snapshot, strict: effectiveStrict)
+        if effectiveStrict {
+            try BackgroundInputDispatcher.typeText(pid: snapshot.pid, text: text)
+            return recordAction(ActionMetadata(ok: true, path: .backgroundCGEvent, strictMode: true, backgroundSafe: true, fallbackUsed: false, message: "Typed text with postToPid."))
+        }
+
+        try foregroundInput.typeText(text)
+        return recordAction(ActionMetadata(ok: true, path: .foregroundCGEvent, strictMode: false, backgroundSafe: false, fallbackUsed: true, message: "Typed text with foreground HID fallback."))
+    }
+
+    func pressKey(snapshotID: String?, combo: String, strict requestedStrict: Bool?) throws -> ActionMetadata {
+        let snapshot = try requireSnapshot(snapshotID: snapshotID)
+        let effectiveStrict = requestedStrict ?? snapshot.strictMode
+        try validateSnapshotForAction(snapshot: snapshot, strict: effectiveStrict)
+        if effectiveStrict {
+            try BackgroundInputDispatcher.pressKey(pid: snapshot.pid, combo: combo)
+            return recordAction(ActionMetadata(ok: true, path: .backgroundCGEvent, strictMode: true, backgroundSafe: true, fallbackUsed: false, message: "Pressed key with postToPid."))
+        }
+
+        try foregroundInput.pressKey(combo)
+        return recordAction(ActionMetadata(ok: true, path: .foregroundCGEvent, strictMode: false, backgroundSafe: false, fallbackUsed: true, message: "Pressed key with foreground HID fallback."))
+    }
+
+    func scroll(snapshotID: String?, direction: String?, pages: Double, imageX: Double?, imageY: Double?, strict requestedStrict: Bool?) throws -> ActionMetadata {
+        let snapshot = try requireSnapshot(snapshotID: snapshotID)
+        let effectiveStrict = requestedStrict ?? snapshot.strictMode
+        try validateSnapshotForAction(snapshot: snapshot, strict: effectiveStrict)
+        let amount = max(1, Int32(pages * 5))
+        let deltas = scrollDeltas(direction: direction, amount: amount)
+        let point: CGPoint = {
+            if let imageX, let imageY {
+                return snapshot.screenshotMeta.toScreen(imageX: imageX, imageY: imageY)
+            }
+            return CGPoint(x: snapshot.screenshotMeta.capturedBounds.midX, y: snapshot.screenshotMeta.capturedBounds.midY)
+        }()
+        AgentCursorOverlay.shared.show(at: point)
+
+        if effectiveStrict {
+            guard let windowNumber = snapshot.windowNumber else {
+                throw ComputerUseError.strictModeViolation("background scroll requires a CG window number")
+            }
+            try BackgroundInputDispatcher.scroll(pid: snapshot.pid, windowNumber: windowNumber, point: point, deltaX: deltas.x, deltaY: deltas.y)
+            return recordAction(ActionMetadata(ok: true, path: .backgroundCGEvent, strictMode: true, backgroundSafe: true, fallbackUsed: false, message: "Scrolled with postToPid."))
+        }
+
+        try foregroundInput.scroll(point: point, deltaX: deltas.x, deltaY: deltas.y)
+        return recordAction(ActionMetadata(ok: true, path: .foregroundCGEvent, strictMode: false, backgroundSafe: false, fallbackUsed: true, message: "Scrolled with foreground HID fallback."))
+    }
+
+    func setValue(snapshotID: String?, ref: String?, index: Int?, value: String) throws -> ActionMetadata {
+        let snapshot = try requireSnapshot(snapshotID: snapshotID)
+        try validateSnapshotForAction(snapshot: snapshot, strict: snapshot.strictMode)
+        guard let record = findRecord(ref: ref, index: index, in: snapshot) else {
+            throw ComputerUseError.invalidElement(ref ?? index.map(String.init) ?? "<missing>")
+        }
+        guard let fresh = freshRecord(matching: record, in: snapshot) else {
+            throw ComputerUseError.staleSnapshot("The target element changed. Take a new snapshot before setting its value.")
+        }
+        AgentCursorOverlay.shared.show(at: fresh.semantic.frame.center)
+        let axWriteAccepted = accessibility.setValue(record: fresh, value: value)
+        if axWriteAccepted, valueMatches(record: fresh, expected: value) {
+            return recordAction(ActionMetadata(ok: true, path: .accessibility, strictMode: snapshot.strictMode, backgroundSafe: true, fallbackUsed: false, message: "Set \(fresh.semantic.ref) via AXValue."))
+        }
+
+        // Web inputs (Chromium/Electron) accept AXValue writes without applying
+        // them. Fall back to focus + select-all + retype, then verify.
+        _ = accessibility.focus(record: fresh)
+        try BackgroundInputDispatcher.pressKey(pid: snapshot.pid, combo: "command+a")
+        try BackgroundInputDispatcher.typeText(pid: snapshot.pid, text: value)
+        Thread.sleep(forTimeInterval: 0.08)
+        let verified = valueMatches(record: fresh, expected: value)
+        return recordAction(ActionMetadata(
+            ok: verified,
+            path: .backgroundCGEvent,
+            strictMode: snapshot.strictMode,
+            backgroundSafe: true,
+            fallbackUsed: true,
+            message: verified
+                ? "AXValue write did not stick; set \(fresh.semantic.ref) via select-all + retype fallback."
+                : "Could not verify the value of \(fresh.semantic.ref) after AXValue and retype fallback."
+        ))
+    }
+
+    private func valueMatches(record: AXElementRecord, expected: String) -> Bool {
+        accessibility.value(record: record) == expected
+    }
+
+    func performAction(snapshotID: String?, ref: String?, index: Int?, action: String) throws -> ActionMetadata {
+        let snapshot = try requireSnapshot(snapshotID: snapshotID)
+        try validateSnapshotForAction(snapshot: snapshot, strict: snapshot.strictMode)
+        guard let record = findRecord(ref: ref, index: index, in: snapshot) else {
+            throw ComputerUseError.invalidElement(ref ?? index.map(String.init) ?? "<missing>")
+        }
+        guard let fresh = freshRecord(matching: record, in: snapshot) else {
+            throw ComputerUseError.staleSnapshot("The target element changed. Take a new snapshot before performing another action.")
+        }
+        AgentCursorOverlay.shared.show(at: fresh.semantic.frame.center)
+        let ok = accessibility.performAction(record: fresh, action: action)
+        return recordAction(ActionMetadata(ok: ok, path: .accessibility, strictMode: snapshot.strictMode, backgroundSafe: true, fallbackUsed: false, message: ok ? "Performed \(action) on \(fresh.semantic.ref)." : "AX action \(action) failed."))
+    }
+
+    func wait(milliseconds: Int) async -> ActionMetadata {
+        let clamped = max(0, min(milliseconds, 10_000))
+        try? await Task.sleep(nanoseconds: UInt64(clamped) * 1_000_000)
+        return ActionMetadata(ok: true, path: .none, strictMode: strictMode, backgroundSafe: true, fallbackUsed: false, message: "Waited \(clamped)ms.")
+    }
+
+    private func clickPoint(_ point: CGPoint, clickCount: Int, strict: Bool, fallbackUsed: Bool) async throws -> ActionMetadata {
+        let snapshot = try requireSnapshot()
+        try validateSnapshotForAction(snapshot: snapshot, strict: strict)
+        AgentCursorOverlay.shared.show(at: point)
+        if strict {
+            guard let windowNumber = snapshot.windowNumber else {
+                throw ComputerUseError.strictModeViolation("background click requires a CG window number")
+            }
+            try await BackgroundInputDispatcher.click(pid: snapshot.pid, windowNumber: windowNumber, point: point, doubleClick: clickCount >= 2)
+            return recordAction(ActionMetadata(ok: true, path: .backgroundCGEvent, strictMode: true, backgroundSafe: true, fallbackUsed: fallbackUsed, message: "Clicked with postToPid at \(Int(point.x)),\(Int(point.y))."))
+        }
+
+        try await foregroundInput.click(point: point, doubleClick: clickCount >= 2)
+        return recordAction(ActionMetadata(ok: true, path: .foregroundCGEvent, strictMode: false, backgroundSafe: false, fallbackUsed: true, message: "Clicked with foreground HID fallback at \(Int(point.x)),\(Int(point.y))."))
+    }
+
+    private func ensureBackgroundActivation(target: WindowTarget) async throws -> Bool {
+        ensureFrontmostMonitor()
+        guard let previousPID = NSWorkspace.shared.frontmostApplication?.processIdentifier else {
+            throw ComputerUseError.noFrontmostApplication
+        }
+        let nextActivationKey = "\(previousPID):\(target.pid)"
+        if activationKey != nextActivationKey {
+            resetBackgroundActivation()
+            let next = BackgroundActivationSession(previousPID: previousPID, targetPID: target.pid)
+            try next.start()
+            activationSession = next
+            activationKey = nextActivationKey
+            activationPreviousPID = previousPID
+            activationTargetPID = target.pid
+            activatedWindowKey = nil
+        }
+
+        let nextWindowKey = "\(target.pid):\(target.windowNumber ?? -1)"
+        if activatedWindowKey != nextWindowKey {
+            guard let activationSession else {
+                throw ComputerUseError.strictModeViolation("background activation session was not created")
+            }
+            try await activationSession.activate(target: target)
+            activatedWindowKey = nextWindowKey
+        }
+        return true
+    }
+
+    private func ensureFrontmostMonitor() {
+        if frontmostMonitor != nil { return }
+        frontmostMonitor = FrontmostApplicationMonitor { [weak self] pid in
+            guard let runtime = self else { return }
+            Task { await runtime.frontmostApplicationChanged(pid: pid) }
+        }
+    }
+
+    private func frontmostApplicationChanged(pid: pid_t?) {
+        guard let pid, activationSession != nil else { return }
+        if pid == activationPreviousPID {
+            staleSnapshotReason = "The user returned focus to the previous app. Take a new snapshot before sending more input."
+        } else if pid == activationTargetPID {
+            staleSnapshotReason = "The target app became frontmost. Take a new snapshot before continuing."
+        } else {
+            staleSnapshotReason = "The user changed app focus. Take a new snapshot before sending more input."
+        }
+        resetBackgroundActivation()
+    }
+
+    private func resetBackgroundActivation() {
+        activationSession?.stop()
+        activationSession = nil
+        activationKey = nil
+        activatedWindowKey = nil
+        activationPreviousPID = nil
+        activationTargetPID = nil
+    }
+
+    private func requireSnapshot(snapshotID: String? = nil) throws -> AppSnapshot {
+        guard let lastSnapshot else { throw ComputerUseError.noSnapshot }
+        if let snapshotID, snapshotID != lastSnapshot.id {
+            throw ComputerUseError.staleSnapshot("Snapshot \(snapshotID) is no longer current. Take a new snapshot before retrying.")
+        }
+        return lastSnapshot
+    }
+
+    private func validateSnapshotForAction(snapshot: AppSnapshot, strict: Bool) throws {
+        if let staleSnapshotReason {
+            throw ComputerUseError.staleSnapshot(staleSnapshotReason)
+        }
+        guard strict else { return }
+        if NSWorkspace.shared.frontmostApplication?.processIdentifier == snapshot.pid { return }
+        guard activationSession != nil, activationTargetPID == snapshot.pid else {
+            throw ComputerUseError.staleSnapshot("The target app is no longer safely activated. Take a new snapshot before retrying.")
+        }
+    }
+
+    private func annotate(snapshot: AppSnapshot) -> AppSnapshot {
+        snapshotSequence += 1
+        let key = windowKey(snapshot: snapshot)
+        let labels = Set(snapshot.records.map { stableLabel(for: $0.semantic) }.filter { !$0.isEmpty })
+        let previous = previousLabelsByWindow[key] ?? []
+        previousLabelsByWindow[key] = labels
+
+        let added = labels.subtracting(previous).sorted()
+        let removed = previous.subtracting(labels).sorted()
+        return snapshot.withSessionMetadata(
+            id: UUID().uuidString.lowercased(),
+            observation: snapshotSequence,
+            recentActions: Array(recentActions.suffix(8)),
+            addedLabels: Array(added.prefix(16)),
+            removedLabels: Array(removed.prefix(16))
+        )
+    }
+
+    private func recordAction(_ metadata: ActionMetadata) -> ActionMetadata {
+        recentActions.append(metadata.message)
+        if recentActions.count > 12 {
+            recentActions.removeFirst(recentActions.count - 12)
+        }
+        return metadata
+    }
+
+    private func freshRecord(matching record: AXElementRecord, in snapshot: AppSnapshot) -> AXElementRecord? {
+        guard let target = try? accessibility.resolveTarget(appName: snapshot.appName, pid: snapshot.pid, windowTitle: snapshot.windowTitle) else {
+            return nil
+        }
+        let records = accessibility.records(target: target)
+        if let index = records.firstIndex(where: { $0.semantic.id == record.semantic.id }), stableMatch(records[index].semantic, record.semantic) {
+            return records[index]
+        }
+        return records.first { stableMatch($0.semantic, record.semantic) }
+    }
+
+    private func stableMatch(_ candidate: SemanticAXElement, _ target: SemanticAXElement) -> Bool {
+        candidate.role == target.role && stableLabel(for: candidate) == stableLabel(for: target)
+    }
+
+    private func stableLabel(for element: SemanticAXElement) -> String {
+        let value = element.value?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
+        let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines)
+        let display = value.isEmpty ? label : "\(label) \(value)"
+        return "\(element.role):\(display)"
+            .components(separatedBy: .whitespacesAndNewlines)
+            .filter { !$0.isEmpty }
+            .joined(separator: " ")
+            .lowercased()
+    }
+
+    private func windowKey(snapshot: AppSnapshot) -> String {
+        "\(snapshot.pid):\(snapshot.windowNumber ?? -1)"
+    }
+
+    private func findRecord(ref: String?, index: Int?, in snapshot: AppSnapshot) -> AXElementRecord? {
+        if let ref {
+            let normalized = normalizeRef(ref)
+            return snapshot.records.first { $0.semantic.ref == normalized }
+        }
+        if let index {
+            if let byID = snapshot.records.first(where: { $0.semantic.id == index }) {
+                return byID
+            }
+            if index >= 0 && index < snapshot.records.count {
+                return snapshot.records[index]
+            }
+        }
+        return nil
+    }
+
+    private func normalizeRef(_ raw: String) -> String {
+        let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
+        if trimmed.hasPrefix("{e"), trimmed.hasSuffix("}") { return trimmed }
+        if trimmed.hasPrefix("e") { return "{\(trimmed)}" }
+        return "{e\(trimmed)}"
+    }
+
+    private func scrollDeltas(direction: String?, amount: Int32) -> (x: Int32, y: Int32) {
+        switch direction?.lowercased() ?? "down" {
+        case "up": return (0, amount)
+        case "left": return (amount, 0)
+        case "right": return (-amount, 0)
+        default: return (0, -amount)
+        }
+    }
+}

packages/handsfree/native/HandsFree/Sources/ComputerUse/FrontmostApplicationMonitor.swift

@@ -0,0 +1,28 @@
+import AppKit
+import Foundation
+
+final class FrontmostApplicationMonitor: @unchecked Sendable {
+    private let queue = DispatchQueue(label: "openwork.computer-use.frontmost-monitor")
+    private var timer: DispatchSourceTimer?
+    private var lastPID: pid_t?
+
+    init(onChange: @escaping @Sendable (pid_t?) -> Void) {
+        lastPID = NSWorkspace.shared.frontmostApplication?.processIdentifier
+        let source = DispatchSource.makeTimerSource(queue: queue)
+        source.schedule(deadline: .now() + .milliseconds(250), repeating: .milliseconds(250))
+        source.setEventHandler { [weak self] in
+            guard let self else { return }
+            let pid = NSWorkspace.shared.frontmostApplication?.processIdentifier
+            if pid != self.lastPID {
+                self.lastPID = pid
+                onChange(pid)
+            }
+        }
+        timer = source
+        source.resume()
+    }
+
+    deinit {
+        timer?.cancel()
+    }
+}

packages/handsfree/native/HandsFree/Sources/ComputerUse/InputService.swift

@@ -0,0 +1,102 @@
+import Foundation
+import CoreGraphics
+
+final class InputService: @unchecked Sendable {
+    func moveMouse(point: CGPoint) throws {
+        guard let source = CGEventSource(stateID: .combinedSessionState) else {
+            throw ComputerUseError.eventSourceFailed
+        }
+        guard let event = CGEvent(mouseEventSource: source, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left) else {
+            throw ComputerUseError.eventCreationFailed
+        }
+        event.post(tap: .cghidEventTap)
+    }
+
+    func click(point: CGPoint, doubleClick: Bool = false) async throws {
+        guard let source = CGEventSource(stateID: .combinedSessionState) else {
+            throw ComputerUseError.eventSourceFailed
+        }
+
+        if let move = CGEvent(mouseEventSource: source, mouseType: .mouseMoved, mouseCursorPosition: point, mouseButton: .left) {
+            move.post(tap: .cghidEventTap)
+        }
+        try await Task.sleep(nanoseconds: 50_000_000)
+
+        let count = doubleClick ? 2 : 1
+        for clickState in 1...count {
+            guard let down = CGEvent(mouseEventSource: source, mouseType: .leftMouseDown, mouseCursorPosition: point, mouseButton: .left),
+                  let up = CGEvent(mouseEventSource: source, mouseType: .leftMouseUp, mouseCursorPosition: point, mouseButton: .left) else {
+                throw ComputerUseError.eventCreationFailed
+            }
+            down.setIntegerValueField(.mouseEventClickState, value: Int64(clickState))
+            up.setIntegerValueField(.mouseEventClickState, value: Int64(clickState))
+            down.post(tap: .cghidEventTap)
+            up.post(tap: .cghidEventTap)
+        }
+    }
+
+    func typeText(_ text: String) throws {
+        guard let source = CGEventSource(stateID: .combinedSessionState) else {
+            throw ComputerUseError.eventSourceFailed
+        }
+        let units = Array(text.utf16)
+        for start in stride(from: 0, to: units.count, by: 20) {
+            let end = min(start + 20, units.count)
+            let chunk = Array(units[start..<end])
+            guard let event = CGEvent(keyboardEventSource: source, virtualKey: 0, keyDown: true) else {
+                throw ComputerUseError.eventCreationFailed
+            }
+            event.keyboardSetUnicodeString(stringLength: chunk.count, unicodeString: chunk)
+            event.post(tap: .cghidEventTap)
+            Thread.sleep(forTimeInterval: 0.01)
+        }
+    }
+
+    func pressKey(_ combo: String) throws {
+        let parsed = try BackgroundInputDispatcher.parseCombo(combo)
+        guard let source = CGEventSource(stateID: .combinedSessionState) else {
+            throw ComputerUseError.eventSourceFailed
+        }
+        guard let down = CGEvent(keyboardEventSource: source, virtualKey: parsed.keyCode, keyDown: true),
+              let up = CGEvent(keyboardEventSource: source, virtualKey: parsed.keyCode, keyDown: false) else {
+            throw ComputerUseError.eventCreationFailed
+        }
+        down.flags = parsed.flags
+        up.flags = parsed.flags
+        down.post(tap: .cghidEventTap)
+        up.post(tap: .cghidEventTap)
+    }
+
+    func scroll(point: CGPoint?, deltaX: Int32, deltaY: Int32) throws {
+        guard let source = CGEventSource(stateID: .combinedSessionState) else {
+            throw ComputerUseError.eventSourceFailed
+        }
+        guard let event = CGEvent(scrollWheelEvent2Source: source, units: .line, wheelCount: 2, wheel1: deltaY, wheel2: deltaX, wheel3: 0) else {
+            throw ComputerUseError.eventCreationFailed
+        }
+        if let point { event.location = point }
+        event.post(tap: .cgSessionEventTap)
+    }
+
+    func drag(path: [CGPoint]) async throws {
+        guard path.count >= 2 else { return }
+        guard let source = CGEventSource(stateID: .combinedSessionState) else {
+            throw ComputerUseError.eventSourceFailed
+        }
+        guard let down = CGEvent(mouseEventSource: source, mouseType: .leftMouseDown, mouseCursorPosition: path[0], mouseButton: .left) else {
+            throw ComputerUseError.eventCreationFailed
+        }
+        down.post(tap: .cghidEventTap)
+        for point in path.dropFirst().dropLast() {
+            if let drag = CGEvent(mouseEventSource: source, mouseType: .leftMouseDragged, mouseCursorPosition: point, mouseButton: .left) {
+                drag.post(tap: .cghidEventTap)
+            }
+            try await Task.sleep(nanoseconds: 12_000_000)
+        }
+        guard let last = path.last,
+              let up = CGEvent(mouseEventSource: source, mouseType: .leftMouseUp, mouseCursorPosition: last, mouseButton: .left) else {
+            throw ComputerUseError.eventCreationFailed
+        }
+        up.post(tap: .cghidEventTap)
+    }
+}

packages/handsfree/native/HandsFree/Sources/ComputerUse/MCPServer.swift

@@ -0,0 +1,623 @@
+import AppKit
+import ApplicationServices
+import Foundation
+import ScreenCaptureKit
+
+actor MCPServer {
+    private let runtime = ComputerUseRuntime()
+    private let input = InputService()
+    private var cuaSnapshotFrontmostPID: pid_t?
+
+    func run() async {
+        log("Computer Use server starting")
+        while let line = readLine(strippingNewline: true) {
+            guard !line.isEmpty else { continue }
+            guard let data = line.data(using: .utf8),
+                  let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {
+                log("Invalid JSON-RPC line")
+                continue
+            }
+
+            let id = json["id"]
+            let method = json["method"] as? String ?? ""
+            let params = json["params"] as? [String: Any] ?? [:]
+
+            switch method {
+            case "initialize":
+                respond(id: id, result: [
+                    "protocolVersion": "2025-03-26",
+                    "capabilities": ["tools": [:]],
+                    "serverInfo": ["name": "openwork-computer-use", "version": "0.1.0"],
+                ])
+            case "notifications/initialized":
+                break
+            case "tools/list":
+                respond(id: id, result: ["tools": toolSchemas()])
+            case "tools/call":
+                let name = params["name"] as? String ?? ""
+                let args = params["arguments"] as? [String: Any] ?? [:]
+                let content = await executeTool(name: name, args: args)
+                respond(id: id, result: ["content": content])
+            default:
+                if id != nil {
+                    respondError(id: id, code: -32601, message: "Method not found: \(method)")
+                }
+            }
+        }
+    }
+
+    private func toolSchemas() -> [[String: Any]] {
+        [
+            toolSchema(
+                name: "snapshot",
+                description: "Return target-window screenshot plus compact semantic AX state. Uses strict background activation by default.",
+                properties: [
+                    "app": ["type": "string", "description": "Optional running app name. Omit for frontmost app."],
+                    "pid": ["type": "number", "description": "Optional process id. Disambiguates multiple instances of the same app."],
+                    "window_title": ["type": "string", "description": "Optional window title to target a specific window."],
+                    "strict": ["type": "boolean", "description": "Keep actions on background-safe AX/postToPid paths. Default true."],
+                ]
+            ),
+            toolSchema(
+                name: "click",
+                description: "Click a semantic ref like {e1}, an index, or screenshot x/y. AX is tried first; strict mode only falls back to background postToPid.",
+                properties: [
+                    "ref": ["type": "string", "description": "Semantic ref from snapshot, e.g. {e1}."],
+                    "snapshot_id": ["type": "string", "description": "Optional snapshot id from the latest snapshot response."],
+                    "index": ["type": "number", "description": "Element id or zero-based compatibility index."],
+                    "x": ["type": "number", "description": "Screenshot-image x coordinate (pixels in the snapshot image, e.g. element center.imageX). Not screen pixels."],
+                    "y": ["type": "number", "description": "Screenshot-image y coordinate (pixels in the snapshot image, e.g. element center.imageY). Not screen pixels."],
+                    "screen_x": ["type": "number", "description": "Absolute screen x coordinate (e.g. element center.screenX). Takes precedence over x/y."],
+                    "screen_y": ["type": "number", "description": "Absolute screen y coordinate (e.g. element center.screenY). Takes precedence over x/y."],
+                    "click_count": ["type": "number", "description": "1 or 2. Default 1."],
+                    "strict": ["type": "boolean", "description": "Override strict mode for this action."],
+                ]
+            ),
+            toolSchema(
+                name: "type_text",
+                description: "Type text into the target process. In strict mode this uses CGEvent.postToPid and does not move the real cursor.",
+                properties: [
+                    "text": ["type": "string", "description": "Text to type."],
+                    "snapshot_id": ["type": "string", "description": "Optional snapshot id from the latest snapshot response."],
+                    "strict": ["type": "boolean", "description": "Override strict mode for this action."],
+                ]
+            ),
+            toolSchema(
+                name: "press_key",
+                description: "Press a key combo such as command+k, return, tab, or escape.",
+                properties: [
+                    "combo": ["type": "string", "description": "Key combo."],
+                    "snapshot_id": ["type": "string", "description": "Optional snapshot id from the latest snapshot response."],
+                    "strict": ["type": "boolean", "description": "Override strict mode for this action."],
+                ]
+            ),
+            toolSchema(
+                name: "scroll",
+                description: "Scroll the target window without foregrounding it in strict mode.",
+                properties: [
+                    "direction": ["type": "string", "description": "up, down, left, or right."],
+                    "snapshot_id": ["type": "string", "description": "Optional snapshot id from the latest snapshot response."],
+                    "pages": ["type": "number", "description": "Approximate page count. Default 1."],
+                    "x": ["type": "number", "description": "Optional screenshot x coordinate."],
+                    "y": ["type": "number", "description": "Optional screenshot y coordinate."],
+                    "strict": ["type": "boolean", "description": "Override strict mode for this action."],
+                ]
+            ),
+            toolSchema(
+                name: "set_value",
+                description: "Set a semantic AX element value directly. This stays background-safe.",
+                properties: [
+                    "ref": ["type": "string", "description": "Semantic ref from snapshot."],
+                    "snapshot_id": ["type": "string", "description": "Optional snapshot id from the latest snapshot response."],
+                    "index": ["type": "number", "description": "Element id or zero-based compatibility index."],
+                    "value": ["type": "string", "description": "Value to set."],
+                ]
+            ),
+            toolSchema(
+                name: "perform_action",
+                description: "Perform a named AX action such as AXPress, AXShowMenu, AXIncrement, or AXDecrement.",
+                properties: [
+                    "ref": ["type": "string", "description": "Semantic ref from snapshot."],
+                    "snapshot_id": ["type": "string", "description": "Optional snapshot id from the latest snapshot response."],
+                    "index": ["type": "number", "description": "Element id or zero-based compatibility index."],
+                    "action": ["type": "string", "description": "AX action name. Default AXPress."],
+                ]
+            ),
+            toolSchema(
+                name: "wait",
+                description: "Wait for UI to settle.",
+                properties: ["milliseconds": ["type": "number", "description": "Wait time. Default 1000."]]
+            ),
+            toolSchema(
+                name: "set_strict_mode",
+                description: "Enable or disable strict background mode. Strict mode rejects foreground fallbacks.",
+                properties: ["enabled": ["type": "boolean", "description": "Whether strict background mode is enabled."]]
+            ),
+            toolSchema(name: "check_permissions", description: "Check Accessibility and Screen Recording permission status.", properties: [:]),
+            toolSchema(name: "get_app_state", description: "Compatibility alias for snapshot.", properties: ["app": ["type": "string"], "strict": ["type": "boolean"]]),
+            toolSchema(name: "launch_app", description: "Launch a macOS app by name.", properties: ["name": ["type": "string"]]),
+            toolSchema(name: "activate_app", description: "Bring a running macOS app to the foreground.", properties: ["name": ["type": "string"]]),
+            toolSchema(name: "list_apps", description: "List running regular macOS apps.", properties: [:]),
+            toolSchema(name: "open_url", description: "Open a URL in the default browser or a specific browser app.", properties: ["url": ["type": "string"], "app": ["type": "string"]]),
+            toolSchema(name: "clipboard_read", description: "Read text from the macOS clipboard.", properties: [:]),
+            toolSchema(name: "clipboard_write", description: "Write text to the macOS clipboard.", properties: ["text": ["type": "string"]]),
+            toolSchema(name: "display_info", description: "Return main display logical dimensions and scale factor.", properties: [:]),
+            toolSchema(name: "cua_screenshot", description: "Compatibility full-screen screenshot for CUA loops. Returns logical-size PNG.", properties: [:]),
+            toolSchema(name: "cua_click", description: "Compatibility click at absolute screen coordinates.", properties: ["x": ["type": "number"], "y": ["type": "number"]]),
+            toolSchema(name: "cua_double_click", description: "Compatibility double-click at absolute screen coordinates.", properties: ["x": ["type": "number"], "y": ["type": "number"]]),
+            toolSchema(name: "cua_move", description: "Compatibility mouse move to absolute screen coordinates.", properties: ["x": ["type": "number"], "y": ["type": "number"]]),
+            toolSchema(name: "cua_type", description: "Compatibility text typing into focused input.", properties: ["text": ["type": "string"]]),
+            toolSchema(name: "cua_keypress", description: "Compatibility keypress using CUA key names.", properties: ["keys": ["type": "array", "items": ["type": "string"]]]),
+            toolSchema(name: "cua_scroll", description: "Compatibility scroll at absolute screen coordinates.", properties: ["x": ["type": "number"], "y": ["type": "number"], "scroll_x": ["type": "number"], "scroll_y": ["type": "number"]]),
+            toolSchema(name: "cua_drag", description: "Compatibility drag over an array of [x,y] points.", properties: ["path": ["type": "array"]]),
+            toolSchema(name: "cua_wait", description: "Compatibility wait for UI to settle.", properties: [:]),
+        ]
+    }
+
+    private func toolSchema(name: String, description: String, properties: [String: Any]) -> [String: Any] {
+        ["name": name, "description": description, "inputSchema": ["type": "object", "properties": properties]]
+    }
+
+    private func executeTool(name: String, args: [String: Any]) async -> [[String: Any]] {
+        do {
+            switch name {
+            case "snapshot", "get_app_state":
+                return try await snapshotResult(args: args)
+            case "click":
+                let metadata = try await runtime.click(
+                    snapshotID: snapshotIDArg(args),
+                    ref: args["ref"] as? String,
+                    index: intArg(args, "index"),
+                    imageX: doubleArg(args, "x"),
+                    imageY: doubleArg(args, "y"),
+                    screenX: doubleArg(args, "screen_x"),
+                    screenY: doubleArg(args, "screen_y"),
+                    clickCount: intArg(args, "click_count") ?? 1,
+                    strict: boolArg(args, "strict")
+                )
+                return jsonResult(metadata.dictionary)
+            case "type_text":
+                let metadata = try await runtime.typeText(snapshotID: snapshotIDArg(args), text: args["text"] as? String ?? "", strict: boolArg(args, "strict"))
+                return jsonResult(metadata.dictionary)
+            case "press_key":
+                let metadata = try await runtime.pressKey(snapshotID: snapshotIDArg(args), combo: args["combo"] as? String ?? "", strict: boolArg(args, "strict"))
+                return jsonResult(metadata.dictionary)
+            case "scroll":
+                let metadata = try await runtime.scroll(
+                    snapshotID: snapshotIDArg(args),
+                    direction: args["direction"] as? String,
+                    pages: doubleArg(args, "pages") ?? 1,
+                    imageX: doubleArg(args, "x"),
+                    imageY: doubleArg(args, "y"),
+                    strict: boolArg(args, "strict")
+                )
+                return jsonResult(metadata.dictionary)
+            case "set_value":
+                let metadata = try await runtime.setValue(snapshotID: snapshotIDArg(args), ref: args["ref"] as? String, index: intArg(args, "index"), value: args["value"] as? String ?? "")
+                return jsonResult(metadata.dictionary)
+            case "perform_action":
+                let metadata = try await runtime.performAction(snapshotID: snapshotIDArg(args), ref: args["ref"] as? String, index: intArg(args, "index"), action: args["action"] as? String ?? kAXPressAction)
+                return jsonResult(metadata.dictionary)
+            case "wait":
+                let metadata = await runtime.wait(milliseconds: intArg(args, "milliseconds") ?? 1000)
+                return jsonResult(metadata.dictionary)
+            case "set_strict_mode":
+                let metadata = await runtime.setStrictMode(boolArg(args, "enabled") ?? true)
+                return jsonResult(metadata.dictionary)
+            case "check_permissions":
+                return jsonResult(checkPermissions())
+            case "launch_app":
+                return try await jsonResult(handleLaunchApp(args: args))
+            case "activate_app":
+                return jsonResult(handleActivateApp(args: args))
+            case "list_apps":
+                let details = runningApps()
+                    .compactMap { app -> [String: Any]? in
+                        guard let name = app.localizedName else { return nil }
+                        return ["name": name, "pid": Int(app.processIdentifier)]
+                    }
+                return jsonResult([
+                    "ok": true,
+                    "apps": runningApps().compactMap(\.localizedName).sorted(),
+                    "appDetails": details,
+                ])
+            case "open_url":
+                return try await jsonResult(handleOpenURL(args: args))
+            case "clipboard_read":
+                return jsonResult(["ok": true, "text": NSPasteboard.general.string(forType: .string) ?? ""])
+            case "clipboard_write":
+                let pasteboard = NSPasteboard.general
+                pasteboard.clearContents()
+                pasteboard.setString(args["text"] as? String ?? "", forType: .string)
+                return jsonResult(["ok": true])
+            case "display_info":
+                return jsonResult(displayInfo())
+            case "cua_screenshot":
+                return try await cuaScreenshotResult()
+            case "cua_click":
+                try validateCuaSnapshotFresh()
+                AgentCursorOverlay.shared.show(at: CGPoint(x: intArg(args, "x") ?? 0, y: intArg(args, "y") ?? 0))
+                try await input.click(point: CGPoint(x: intArg(args, "x") ?? 0, y: intArg(args, "y") ?? 0))
+                return jsonResult(["ok": true])
+            case "cua_double_click":
+                try validateCuaSnapshotFresh()
+                AgentCursorOverlay.shared.show(at: CGPoint(x: intArg(args, "x") ?? 0, y: intArg(args, "y") ?? 0))
+                try await input.click(point: CGPoint(x: intArg(args, "x") ?? 0, y: intArg(args, "y") ?? 0), doubleClick: true)
+                return jsonResult(["ok": true])
+            case "cua_move":
+                try validateCuaSnapshotFresh()
+                AgentCursorOverlay.shared.show(at: CGPoint(x: intArg(args, "x") ?? 0, y: intArg(args, "y") ?? 0))
+                try input.moveMouse(point: CGPoint(x: intArg(args, "x") ?? 0, y: intArg(args, "y") ?? 0))
+                return jsonResult(["ok": true])
+            case "cua_type":
+                try validateCuaSnapshotFresh()
+                try input.typeText(args["text"] as? String ?? "")
+                return jsonResult(["ok": true])
+            case "cua_keypress":
+                try validateCuaSnapshotFresh()
+                try input.pressKey(cuaKeysToCombo(args["keys"] as? [String] ?? []))
+                return jsonResult(["ok": true])
+            case "cua_scroll":
+                try validateCuaSnapshotFresh()
+                AgentCursorOverlay.shared.show(at: CGPoint(x: intArg(args, "x") ?? 0, y: intArg(args, "y") ?? 0))
+                try input.scroll(
+                    point: CGPoint(x: intArg(args, "x") ?? 0, y: intArg(args, "y") ?? 0),
+                    deltaX: Int32(intArg(args, "scroll_x") ?? 0),
+                    deltaY: Int32(-(intArg(args, "scroll_y") ?? 0))
+                )
+                return jsonResult(["ok": true])
+            case "cua_drag":
+                try validateCuaSnapshotFresh()
+                let path = parsePointPath(args["path"])
+                if let first = path.first {
+                    AgentCursorOverlay.shared.show(at: first)
+                }
+                try await input.drag(path: path)
+                return jsonResult(["ok": true])
+            case "cua_wait":
+                try await Task.sleep(nanoseconds: 1_000_000_000)
+                return jsonResult(["ok": true])
+            default:
+                return jsonResult(["ok": false, "error": "Unknown tool: \(name)"])
+            }
+        } catch {
+            return jsonResult(errorPayload(error))
+        }
+    }
+
+    private func snapshotResult(args: [String: Any]) async throws -> [[String: Any]] {
+        let snapshot = try await runtime.snapshot(
+            appName: args["app"] as? String,
+            pid: intArg(args, "pid"),
+            windowTitle: args["window_title"] as? String,
+            strict: boolArg(args, "strict")
+        )
+        let payload = snapshotPayload(snapshot)
+        guard let text = jsonString(payload) else {
+            return textResult("Failed to serialize semantic AX snapshot.")
+        }
+        return [
+            ["type": "image", "data": snapshot.screenshotData.base64EncodedString(), "mimeType": snapshot.screenshotMimeType],
+            ["type": "text", "text": text],
+        ]
+    }
+
+    private func snapshotPayload(_ snapshot: AppSnapshot) -> [String: Any] {
+        let elements = snapshot.elements.map { element -> [String: Any] in
+            var dict = element.dictionary
+            let imagePoint = snapshot.screenshotMeta.toImage(point: element.frame.center)
+            dict["center"] = [
+                "screenX": Int(element.frame.center.x),
+                "screenY": Int(element.frame.center.y),
+                "imageX": Int(imagePoint.x),
+                "imageY": Int(imagePoint.y),
+            ]
+            return dict
+        }
+
+        var result: [String: Any] = [
+            "ok": true,
+            "semanticAXVersion": 1,
+            "snapshotId": snapshot.id,
+            "snapshot_id": snapshot.id,
+            "observation": snapshot.observation,
+            "app": snapshot.appName,
+            "pid": Int(snapshot.pid),
+            "windowTitle": snapshot.windowTitle ?? "",
+            "screenshot": snapshot.screenshotMeta.dictionary,
+            "execution": [
+                "strictMode": snapshot.strictMode,
+                "backgroundActivated": snapshot.backgroundActivated,
+                "defaultPath": snapshot.strictMode ? "accessibility_then_background_cgevent" : "accessibility_then_foreground_fallback",
+            ],
+            "elements": elements,
+            "hint": "Use refs like {e1}. Prefer AX-capable refs; strict mode rejects foreground fallback and reports path metadata after every action.",
+        ]
+        if !snapshot.recentActions.isEmpty {
+            result["recentActions"] = snapshot.recentActions
+        }
+        if !snapshot.addedLabels.isEmpty || !snapshot.removedLabels.isEmpty {
+            result["stateDelta"] = ["added": snapshot.addedLabels, "removed": snapshot.removedLabels]
+        }
+        if let windowNumber = snapshot.windowNumber {
+            result["windowNumber"] = windowNumber
+        }
+        return result
+    }
+
+    private func checkPermissions() -> [String: Any] {
+        ComputerUsePermissions.status().dictionary
+    }
+
+    private func handleActivateApp(args: [String: Any]) -> [String: Any] {
+        let name = args["name"] as? String ?? ""
+        guard let app = runningApp(named: name) else {
+            return ["ok": false, "error": "App '\(name)' is not running."]
+        }
+        app.activate()
+        return ["ok": true, "app": app.localizedName ?? name]
+    }
+
+    private func handleLaunchApp(args: [String: Any]) async throws -> [String: Any] {
+        let name = (args["name"] as? String ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
+        guard !name.isEmpty else { return ["ok": false, "error": "App name is required."] }
+        if let app = runningApp(named: name) {
+            app.activate()
+            return ["ok": true, "app": app.localizedName ?? name, "alreadyRunning": true]
+        }
+        guard let appURL = applicationURL(named: name) else {
+            return ["ok": false, "error": "App '\(name)' was not found."]
+        }
+        let config = NSWorkspace.OpenConfiguration()
+        config.activates = true
+        let app = try await NSWorkspace.shared.openApplication(at: appURL, configuration: config)
+        return ["ok": true, "app": app.localizedName ?? name]
+    }
+
+    private func handleOpenURL(args: [String: Any]) async throws -> [String: Any] {
+        guard let rawURL = args["url"] as? String, let url = URL(string: rawURL) else {
+            return ["ok": false, "error": "Invalid URL."]
+        }
+        if let appName = args["app"] as? String, !appName.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
+            guard let appURL = applicationURL(named: appName) else {
+                return ["ok": false, "error": "App '\(appName)' was not found."]
+            }
+            _ = try await NSWorkspace.shared.open([url], withApplicationAt: appURL, configuration: NSWorkspace.OpenConfiguration())
+            return ["ok": true]
+        }
+        return ["ok": NSWorkspace.shared.open(url)]
+    }
+
+    private func displayInfo() -> [String: Any] {
+        guard let screen = NSScreen.main else { return ["ok": false, "error": "No main screen."] }
+        return [
+            "ok": true,
+            "width": Int(screen.frame.width),
+            "height": Int(screen.frame.height),
+            "scale_factor": screen.backingScaleFactor,
+        ]
+    }
+
+    private func cuaScreenshotResult() async throws -> [[String: Any]] {
+        guard let screen = NSScreen.main else { throw ComputerUseError.screenshotFailed }
+        cuaSnapshotFrontmostPID = NSWorkspace.shared.frontmostApplication?.processIdentifier
+        let cgImage = await screenCaptureKitDisplayImage() ?? CGWindowListCreateImage(CGRect.null, .optionOnScreenOnly, kCGNullWindowID, [.bestResolution])
+        guard let cgImage else {
+            throw ComputerUseError.screenshotFailed
+        }
+        let logicalWidth = Int(screen.frame.width)
+        let logicalHeight = Int(screen.frame.height)
+        guard let rep = NSBitmapImageRep(
+            bitmapDataPlanes: nil,
+            pixelsWide: logicalWidth,
+            pixelsHigh: logicalHeight,
+            bitsPerSample: 8,
+            samplesPerPixel: 4,
+            hasAlpha: true,
+            isPlanar: false,
+            colorSpaceName: .deviceRGB,
+            bytesPerRow: 0,
+            bitsPerPixel: 0
+        ) else {
+            throw ComputerUseError.screenshotFailed
+        }
+        rep.size = NSSize(width: logicalWidth, height: logicalHeight)
+        guard let context = NSGraphicsContext(bitmapImageRep: rep) else {
+            throw ComputerUseError.screenshotFailed
+        }
+        NSGraphicsContext.saveGraphicsState()
+        NSGraphicsContext.current = context
+        NSImage(cgImage: cgImage, size: NSSize(width: cgImage.width, height: cgImage.height))
+            .draw(in: NSRect(x: 0, y: 0, width: logicalWidth, height: logicalHeight))
+        NSGraphicsContext.restoreGraphicsState()
+        guard let png = rep.representation(using: .png, properties: [:]) else {
+            throw ComputerUseError.screenshotFailed
+        }
+        return [
+            ["type": "text", "text": jsonString(["ok": true, "width": logicalWidth, "height": logicalHeight]) ?? "{\"ok\":true}"],
+            ["type": "image", "data": png.base64EncodedString(), "mimeType": "image/png"],
+        ]
+    }
+
+    private func screenCaptureKitDisplayImage() async -> CGImage? {
+        do {
+            let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)
+            guard let display = content.displays.first else { return nil }
+            let configuration = SCStreamConfiguration()
+            configuration.width = display.width
+            configuration.height = display.height
+            configuration.showsCursor = true
+            let filter = SCContentFilter(display: display, excludingWindows: [])
+            return try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: configuration)
+        } catch {
+            return nil
+        }
+    }
+
+    private func runningApps() -> [NSRunningApplication] {
+        NSWorkspace.shared.runningApplications.filter { $0.activationPolicy == .regular }
+    }
+
+    private func runningApp(named name: String) -> NSRunningApplication? {
+        let needle = name.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
+        return runningApps().first { $0.localizedName?.lowercased() == needle }
+            ?? runningApps().first { $0.localizedName?.lowercased().contains(needle) == true }
+    }
+
+    private func applicationURL(named name: String) -> URL? {
+        let trimmed = name.trimmingCharacters(in: .whitespacesAndNewlines)
+        if let url = NSWorkspace.shared.urlForApplication(withBundleIdentifier: bundleId(for: trimmed)) {
+            return url
+        }
+        if let path = NSWorkspace.shared.fullPath(forApplication: trimmed) {
+            return URL(fileURLWithPath: path)
+        }
+        let candidates = [
+            "/Applications/\(trimmed).app",
+            "/System/Applications/\(trimmed).app",
+            "/Applications/Utilities/\(trimmed).app",
+            NSString(string: "~/Applications/\(trimmed).app").expandingTildeInPath,
+        ]
+        return candidates.map(URL.init(fileURLWithPath:)).first { FileManager.default.fileExists(atPath: $0.path) }
+    }
+
+    private func bundleId(for appName: String) -> String {
+        switch appName.lowercased() {
+        case "safari": return "com.apple.Safari"
+        case "google chrome", "chrome": return "com.google.Chrome"
+        case "arc": return "company.thebrowser.Browser"
+        case "microsoft edge", "edge": return "com.microsoft.edgemac"
+        case "brave", "brave browser": return "com.brave.Browser"
+        case "slack": return "com.tinyspeck.slackmacgap"
+        default: return ""
+        }
+    }
+
+    private func cuaKeysToCombo(_ keys: [String]) -> String {
+        keys.map { key in
+            switch key.lowercased() {
+            case "ctrl", "control": return "command"
+            case "meta", "super", "win", "cmd": return "command"
+            case "alt": return "option"
+            case "arrowup": return "up"
+            case "arrowdown": return "down"
+            case "arrowleft": return "left"
+            case "arrowright": return "right"
+            case "backspace": return "delete"
+            case " ": return "space"
+            default: return key.lowercased()
+            }
+        }.joined(separator: "+")
+    }
+
+    private func parsePointPath(_ raw: Any?) -> [CGPoint] {
+        guard let pairs = raw as? [[Any]] else { return [] }
+        return pairs.compactMap { pair in
+            guard pair.count >= 2 else { return nil }
+            let x = valueAsDouble(pair[0])
+            let y = valueAsDouble(pair[1])
+            guard let x, let y else { return nil }
+            return CGPoint(x: x, y: y)
+        }
+    }
+
+    private func validateCuaSnapshotFresh() throws {
+        guard let snapshotPID = cuaSnapshotFrontmostPID else {
+            throw ComputerUseError.staleSnapshot("CUA action requires a screenshot first.")
+        }
+        let currentPID = NSWorkspace.shared.frontmostApplication?.processIdentifier
+        guard currentPID == snapshotPID else {
+            throw ComputerUseError.staleSnapshot("The user changed focus since the last CUA screenshot. Capture a new screenshot before acting.")
+        }
+    }
+
+    private func valueAsDouble(_ value: Any) -> Double? {
+        if let value = value as? Double { return value }
+        if let value = value as? Int { return Double(value) }
+        if let value = value as? String { return Double(value) }
+        return nil
+    }
+
+    private func respond(id: Any?, result: Any) {
+        var response: [String: Any] = ["jsonrpc": "2.0", "result": result]
+        if let id { response["id"] = id }
+        writeLine(response)
+    }
+
+    private func respondError(id: Any?, code: Int, message: String) {
+        var response: [String: Any] = ["jsonrpc": "2.0", "error": ["code": code, "message": message]]
+        if let id { response["id"] = id }
+        writeLine(response)
+    }
+
+    private func writeLine(_ object: [String: Any]) {
+        guard let text = jsonString(object) else { return }
+        print(text)
+    }
+
+    private func textResult(_ text: String) -> [[String: Any]] {
+        [["type": "text", "text": text]]
+    }
+
+    private func jsonResult(_ payload: [String: Any]) -> [[String: Any]] {
+        textResult(jsonString(payload) ?? "{\"ok\":false,\"error\":\"Failed to serialize result.\"}")
+    }
+
+    private func jsonString(_ value: Any) -> String? {
+        guard JSONSerialization.isValidJSONObject(value),
+              let data = try? JSONSerialization.data(withJSONObject: value),
+              let text = String(data: data, encoding: .utf8) else {
+            return nil
+        }
+        return text
+    }
+
+    private func errorPayload(_ error: Error) -> [String: Any] {
+        let message = error.localizedDescription
+        var payload: [String: Any] = ["ok": false, "error": message]
+        if case ComputerUseError.staleSnapshot = error {
+            payload["staleSnapshot"] = true
+            payload["retryable"] = false
+            payload["requiredNextAction"] = "snapshot"
+            payload["hint"] = "Take a fresh snapshot before retrying. Do not repeat the same action against stale UI state."
+        }
+        if message.localizedCaseInsensitiveContains("accessibility") {
+            payload["permissionNeeded"] = "accessibility"
+        }
+        if message.localizedCaseInsensitiveContains("screenshot") || message.localizedCaseInsensitiveContains("screen recording") {
+            payload["permissionNeeded"] = "screen-recording"
+        }
+        return payload
+    }
+
+    private func log(_ message: String) {
+        fputs("[ComputerUse] \(message)\n", stderr)
+    }
+
+    private func intArg(_ args: [String: Any], _ key: String) -> Int? {
+        if let value = args[key] as? Int { return value }
+        if let value = args[key] as? Double { return Int(value) }
+        if let value = args[key] as? String { return Int(value) }
+        return nil
+    }
+
+    private func doubleArg(_ args: [String: Any], _ key: String) -> Double? {
+        if let value = args[key] as? Double { return value }
+        if let value = args[key] as? Int { return Double(value) }
+        if let value = args[key] as? String { return Double(value) }
+        return nil
+    }
+
+    private func boolArg(_ args: [String: Any], _ key: String) -> Bool? {
+        if let value = args[key] as? Bool { return value }
+        if let value = args[key] as? String {
+            if value == "true" { return true }
+            if value == "false" { return false }
+        }
+        return nil
+    }
+
+    private func snapshotIDArg(_ args: [String: Any]) -> String? {
+        args["snapshot_id"] as? String ?? args["snapshotId"] as? String
+    }
+}

packages/handsfree/native/HandsFree/Sources/ComputerUse/PermissionSetupApp.swift

@@ -0,0 +1,593 @@
+import AppKit
+import ApplicationServices
+import Foundation
+
+enum ComputerUsePermissionTarget: String {
+    case accessibility
+    case screenRecording = "screen-recording"
+}
+
+struct ComputerUsePermissionStatus {
+    let accessibility: Bool
+    let screenRecording: Bool
+    var ok: Bool { accessibility && screenRecording }
+    var dictionary: [String: Any] {
+        ["ok": ok, "accessibility": accessibility, "screenRecording": screenRecording]
+    }
+}
+
+enum ComputerUsePermissions {
+    /// Non-prompting check — safe to call on a timer / from HTTP polling.
+    static func status() -> ComputerUsePermissionStatus {
+        ComputerUsePermissionStatus(
+            accessibility: AXIsProcessTrusted(),
+            screenRecording: CGPreflightScreenCaptureAccess()
+        )
+    }
+
+    /// Prompting request — only call when the user explicitly clicks "Grant".
+    static func request(_ target: ComputerUsePermissionTarget) {
+        switch target {
+        case .accessibility:
+            let opts = [kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary
+            _ = AXIsProcessTrustedWithOptions(opts)
+        case .screenRecording:
+            _ = CGRequestScreenCaptureAccess()
+        }
+    }
+}
+
+// MARK: - App delegate
+
+@MainActor
+final class PermissionSetupAppDelegate: NSObject, NSApplicationDelegate {
+    private var windowController: NSWindowController?
+
+    func applicationDidFinishLaunching(_ notification: Notification) {
+        let window = PermissionSetupWindow()
+        let vc = PermissionSetupViewController()
+        window.contentViewController = vc
+        let wc = NSWindowController(window: window)
+        windowController = wc
+        wc.showWindow(nil)
+        NSApplication.shared.activate(ignoringOtherApps: true)
+    }
+
+    func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { true }
+}
+
+// MARK: - Window
+
+private final class PermissionSetupWindow: NSWindow {
+    init() {
+        super.init(
+            contentRect: NSRect(x: 0, y: 0, width: 460, height: 600),
+            styleMask: [.titled, .closable, .fullSizeContentView],
+            backing: .buffered,
+            defer: false
+        )
+        title = "OpenWork Computer Use"
+        titleVisibility = .hidden
+        titlebarAppearsTransparent = true
+        isMovableByWindowBackground = true
+        backgroundColor = .clear
+        isOpaque = false
+        hasShadow = true
+        center()
+    }
+}
+
+// MARK: - View controller
+
+@MainActor
+final class PermissionSetupViewController: NSViewController {
+    private var axBadge = StatusBadge()
+    private var srBadge = StatusBadge()
+    private var axGrantButton: NSButton?
+    private var srGrantButton: NSButton?
+    private var timer: Timer?
+
+    init() { super.init(nibName: nil, bundle: nil) }
+    required init?(coder: NSCoder) { nil }
+    deinit { timer?.invalidate() }
+
+    override func loadView() {
+        // Liquid glass: NSVisualEffectView blending with content behind the window
+        let blur = NSVisualEffectView()
+        blur.material = .underWindowBackground
+        blur.blendingMode = .behindWindow
+        blur.state = .active
+        blur.wantsLayer = true
+        blur.layer?.cornerRadius = 12
+        blur.layer?.masksToBounds = true
+        view = blur
+    }
+
+    override func viewDidLoad() {
+        super.viewDidLoad()
+        buildLayout()
+        refresh()
+        timer = Timer.scheduledTimer(withTimeInterval: 1.5, repeats: true) { [weak self] _ in
+            guard let controller = self else { return }
+            Task { @MainActor in controller.refresh() }
+        }
+    }
+
+    private func buildLayout() {
+        let iconView = NSImageView()
+        iconView.image = NSApplication.shared.applicationIconImage
+        iconView.imageScaling = .scaleProportionallyUpOrDown
+        iconView.translatesAutoresizingMaskIntoConstraints = false
+        iconView.widthAnchor.constraint(equalToConstant: 52).isActive = true
+        iconView.heightAnchor.constraint(equalToConstant: 52).isActive = true
+
+        let titleField = textField("Computer Use Setup", size: 20, weight: .semibold)
+        titleField.alignment = .center
+
+        let subtitleField = wrappingField(
+            "Grant two permissions so agents can see and control apps in the background.",
+            size: 13
+        )
+        subtitleField.textColor = .secondaryLabelColor
+        subtitleField.alignment = .center
+
+        let header = vstack([iconView, titleField, subtitleField], spacing: 8, alignment: NSLayoutConstraint.Attribute.centerX)
+
+        let axCard = makeAccessibilityCard()
+        let srCard = makeScreenRecordingCard()
+
+        let doneBtn = NSButton(title: "Done — Return to OpenWork", target: self, action: #selector(done))
+        doneBtn.bezelStyle = .rounded
+        doneBtn.controlSize = .large
+        doneBtn.keyEquivalent = "\r"
+
+        let root = vstack([header, axCard, srCard, doneBtn], spacing: 14, alignment: NSLayoutConstraint.Attribute.leading)
+        root.translatesAutoresizingMaskIntoConstraints = false
+        view.addSubview(root)
+
+        NSLayoutConstraint.activate([
+            root.leadingAnchor.constraint(equalTo: view.leadingAnchor, constant: 20),
+            root.trailingAnchor.constraint(equalTo: view.trailingAnchor, constant: -20),
+            root.topAnchor.constraint(greaterThanOrEqualTo: view.topAnchor, constant: 52),
+            root.bottomAnchor.constraint(lessThanOrEqualTo: view.bottomAnchor, constant: -20),
+            root.centerYAnchor.constraint(equalTo: view.centerYAnchor, constant: 10),
+            axCard.widthAnchor.constraint(equalTo: root.widthAnchor),
+            srCard.widthAnchor.constraint(equalTo: root.widthAnchor),
+            doneBtn.widthAnchor.constraint(equalTo: root.widthAnchor),
+        ])
+    }
+
+    // MARK: Accessibility card
+
+    private func makeAccessibilityCard() -> NSView {
+        let step = StepCircle(number: "1")
+        let title = textField("Accessibility", size: 15, weight: .semibold)
+        let body = wrappingField(
+            "Allows agents to interact with UI controls, click buttons, and type text entirely in the background.",
+            size: 13
+        )
+        body.textColor = .secondaryLabelColor
+
+        let btn = NSButton(title: "Grant Accessibility", target: self, action: #selector(grantAccessibility))
+        btn.bezelStyle = .rounded
+        btn.controlSize = .regular
+        axGrantButton = btn
+
+        let top = hstack([step, title, springy(), axBadge], spacing: 8)
+        let inner = vstack([top, body, btn], spacing: 10, alignment: NSLayoutConstraint.Attribute.leading)
+        btn.widthAnchor.constraint(equalTo: inner.widthAnchor).isActive = true
+
+        return glassCard(inner)
+    }
+
+    // MARK: Screen Recording card
+
+    private func makeScreenRecordingCard() -> NSView {
+        let step = StepCircle(number: "2")
+        let title = textField("Screen Recording", size: 15, weight: .semibold)
+        let body = wrappingField(
+            "Lets agents see what is on screen. If macOS does not prompt automatically, drag the app icon below into the Screen Recording list.",
+            size: 13
+        )
+        body.textColor = .secondaryLabelColor
+
+        let dragFlow = makeDragFlowView()
+
+        let reqBtn = NSButton(title: "Request Screen Recording", target: self, action: #selector(requestScreenRecording))
+        reqBtn.bezelStyle = .rounded
+        reqBtn.controlSize = .regular
+        srGrantButton = reqBtn
+
+        let openBtn = NSButton(title: "Open Privacy & Security", target: self, action: #selector(openPrivacySecurity))
+        openBtn.bezelStyle = .rounded
+        openBtn.controlSize = .small
+
+        let top = hstack([step, title, springy(), srBadge], spacing: 8)
+        let inner = vstack([top, body, dragFlow, reqBtn, openBtn], spacing: 10, alignment: NSLayoutConstraint.Attribute.leading)
+        reqBtn.widthAnchor.constraint(equalTo: inner.widthAnchor).isActive = true
+        openBtn.widthAnchor.constraint(equalTo: inner.widthAnchor).isActive = true
+        dragFlow.widthAnchor.constraint(equalTo: inner.widthAnchor).isActive = true
+
+        return glassCard(inner)
+    }
+
+    // MARK: Drag flow visualization
+
+    private func makeDragFlowView() -> NSView {
+        // Large draggable app icon
+        let iconView = DraggableAppIconView(frame: .zero)
+        iconView.image = NSApplication.shared.applicationIconImage
+        iconView.imageScaling = .scaleProportionallyUpOrDown
+        iconView.translatesAutoresizingMaskIntoConstraints = false
+        iconView.widthAnchor.constraint(equalToConstant: 56).isActive = true
+        iconView.heightAnchor.constraint(equalToConstant: 56).isActive = true
+
+        let dragHint = textField("Drag me", size: 10)
+        dragHint.textColor = .tertiaryLabelColor
+        dragHint.alignment = .center
+
+        let iconCol = vstack([iconView, dragHint], spacing: 5, alignment: NSLayoutConstraint.Attribute.centerX)
+
+        let arrow = textField("→", size: 20)
+        arrow.textColor = .tertiaryLabelColor
+
+        let dropZone = PrivacyDropZoneView()
+        dropZone.translatesAutoresizingMaskIntoConstraints = false
+        dropZone.widthAnchor.constraint(equalToConstant: 116).isActive = true
+        dropZone.heightAnchor.constraint(equalToConstant: 60).isActive = true
+
+        let row = hstack([iconCol, arrow, dropZone], spacing: 14)
+        row.alignment = NSLayoutConstraint.Attribute.centerY
+        row.distribution = NSStackView.Distribution.fill
+
+        let hint = wrappingField(
+            "Drag this icon into the Screen Recording list in Privacy & Security, then enable it.",
+            size: 11
+        )
+        hint.textColor = .secondaryLabelColor
+        hint.alignment = .center
+
+        let content = vstack([row, hint], spacing: 8, alignment: NSLayoutConstraint.Attribute.centerX)
+        content.translatesAutoresizingMaskIntoConstraints = false
+
+        let wrapper = InnerTintView()
+        wrapper.translatesAutoresizingMaskIntoConstraints = false
+        wrapper.addSubview(content)
+        NSLayoutConstraint.activate([
+            content.topAnchor.constraint(equalTo: wrapper.topAnchor, constant: 12),
+            content.bottomAnchor.constraint(equalTo: wrapper.bottomAnchor, constant: -12),
+            content.leadingAnchor.constraint(equalTo: wrapper.leadingAnchor, constant: 12),
+            content.trailingAnchor.constraint(equalTo: wrapper.trailingAnchor, constant: -12),
+        ])
+        return wrapper
+    }
+
+    // MARK: Refresh
+
+    private func refresh() {
+        let status = ComputerUsePermissions.status()
+        axBadge.update(granted: status.accessibility)
+        srBadge.update(granted: status.screenRecording)
+        axGrantButton?.isEnabled = !status.accessibility
+        srGrantButton?.isEnabled = !status.screenRecording
+    }
+
+    // MARK: Actions
+
+    @objc private func grantAccessibility() {
+        ComputerUsePermissions.request(.accessibility)
+        DispatchQueue.main.asyncAfter(deadline: .now() + 0.4) { [weak self] in self?.refresh() }
+    }
+
+    @objc private func requestScreenRecording() {
+        ComputerUsePermissions.request(.screenRecording)
+        DispatchQueue.main.asyncAfter(deadline: .now() + 0.4) { [weak self] in self?.refresh() }
+    }
+
+    @objc private func openPrivacySecurity() {
+        guard let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture") else { return }
+        NSWorkspace.shared.open(url)
+    }
+
+    @objc private func done() {
+        NSApplication.shared.terminate(nil)
+    }
+
+    // MARK: Builder helpers
+
+    private func glassCard(_ content: NSView) -> GlassCardView {
+        let card = GlassCardView()
+        card.addContent(content)
+        return card
+    }
+
+    private func textField(_ text: String, size: CGFloat, weight: NSFont.Weight = .regular) -> NSTextField {
+        let f = NSTextField(labelWithString: text)
+        f.font = .systemFont(ofSize: size, weight: weight)
+        return f
+    }
+
+    private func wrappingField(_ text: String, size: CGFloat) -> NSTextField {
+        let f = NSTextField(wrappingLabelWithString: text)
+        f.font = .systemFont(ofSize: size)
+        return f
+    }
+
+    private func vstack(_ views: [NSView], spacing: CGFloat, alignment: NSLayoutConstraint.Attribute = NSLayoutConstraint.Attribute.leading) -> NSStackView {
+        let sv = NSStackView(views: views)
+        sv.orientation = .vertical
+        sv.spacing = spacing
+        sv.alignment = alignment
+        return sv
+    }
+
+    private func hstack(_ views: [NSView], spacing: CGFloat) -> NSStackView {
+        let sv = NSStackView(views: views)
+        sv.orientation = .horizontal
+        sv.spacing = spacing
+        sv.alignment = NSLayoutConstraint.Attribute.centerY
+        return sv
+    }
+
+    private func springy() -> NSView {
+        let v = NSView()
+        v.setContentHuggingPriority(.defaultLow, for: .horizontal)
+        v.setContentCompressionResistancePriority(.defaultLow, for: .horizontal)
+        return v
+    }
+}
+
+// MARK: - Glass card
+
+/// NSView container with an NSVisualEffectView inside for the glass effect.
+/// Uses updateLayer() so the border color responds to appearance changes.
+final class GlassCardView: NSView {
+    private let blur: NSVisualEffectView
+
+    override init(frame: NSRect) {
+        blur = NSVisualEffectView()
+        blur.material = .contentBackground
+        blur.blendingMode = .withinWindow
+        blur.state = .active
+        blur.wantsLayer = true
+        blur.layer?.cornerRadius = 13
+        blur.layer?.masksToBounds = true
+        blur.translatesAutoresizingMaskIntoConstraints = false
+
+        super.init(frame: frame)
+        wantsLayer = true
+        layer?.cornerRadius = 14
+
+        addSubview(blur)
+        NSLayoutConstraint.activate([
+            blur.topAnchor.constraint(equalTo: topAnchor),
+            blur.bottomAnchor.constraint(equalTo: bottomAnchor),
+            blur.leadingAnchor.constraint(equalTo: leadingAnchor),
+            blur.trailingAnchor.constraint(equalTo: trailingAnchor),
+        ])
+    }
+
+    required init?(coder: NSCoder) { nil }
+
+    override var wantsUpdateLayer: Bool { true }
+    override func updateLayer() {
+        layer?.borderWidth = 0.5
+        layer?.borderColor = NSColor.separatorColor.cgColor
+    }
+
+    func addContent(_ content: NSView) {
+        content.translatesAutoresizingMaskIntoConstraints = false
+        blur.addSubview(content)
+        NSLayoutConstraint.activate([
+            content.topAnchor.constraint(equalTo: blur.topAnchor, constant: 16),
+            content.bottomAnchor.constraint(equalTo: blur.bottomAnchor, constant: -16),
+            content.leadingAnchor.constraint(equalTo: blur.leadingAnchor, constant: 16),
+            content.trailingAnchor.constraint(equalTo: blur.trailingAnchor, constant: -16),
+        ])
+    }
+}
+
+// MARK: - Status badge
+
+/// Pill badge showing Granted/Needed with adaptive color.
+@MainActor
+final class StatusBadge: NSView {
+    private let dot = NSView()
+    private let label = NSTextField(labelWithString: "")
+    private var isGranted = false
+
+    override init(frame: NSRect) {
+        super.init(frame: frame)
+        wantsLayer = true
+        layer?.cornerRadius = 10
+
+        dot.wantsLayer = true
+        dot.layer?.cornerRadius = 4
+        dot.translatesAutoresizingMaskIntoConstraints = false
+        dot.widthAnchor.constraint(equalToConstant: 8).isActive = true
+        dot.heightAnchor.constraint(equalToConstant: 8).isActive = true
+
+        label.font = .systemFont(ofSize: 12, weight: .medium)
+        label.translatesAutoresizingMaskIntoConstraints = false
+
+        let row = NSStackView(views: [dot, label])
+        row.orientation = .horizontal
+        row.spacing = 5
+        row.alignment = .centerY
+        row.translatesAutoresizingMaskIntoConstraints = false
+        addSubview(row)
+
+        NSLayoutConstraint.activate([
+            row.topAnchor.constraint(equalTo: topAnchor, constant: 4),
+            row.bottomAnchor.constraint(equalTo: bottomAnchor, constant: -4),
+            row.leadingAnchor.constraint(equalTo: leadingAnchor, constant: 8),
+            row.trailingAnchor.constraint(equalTo: trailingAnchor, constant: -8),
+        ])
+    }
+
+    required init?(coder: NSCoder) { nil }
+
+    func update(granted: Bool) {
+        isGranted = granted
+        applyColors()
+        label.stringValue = granted ? "Granted" : "Needed"
+        label.textColor = granted ? .systemGreen : .systemOrange
+    }
+
+    private func applyColors() {
+        layer?.backgroundColor = isGranted
+            ? NSColor.systemGreen.withAlphaComponent(0.15).cgColor
+            : NSColor.systemOrange.withAlphaComponent(0.12).cgColor
+        dot.layer?.backgroundColor = isGranted
+            ? NSColor.systemGreen.cgColor
+            : NSColor.systemOrange.cgColor
+    }
+
+    override var wantsUpdateLayer: Bool { true }
+    override func updateLayer() { applyColors() }
+}
+
+// MARK: - Step circle
+
+private final class StepCircle: NSView {
+    init(number: String) {
+        super.init(frame: .zero)
+        wantsLayer = true
+        layer?.backgroundColor = NSColor.controlAccentColor.cgColor
+        layer?.cornerRadius = 11
+        translatesAutoresizingMaskIntoConstraints = false
+        widthAnchor.constraint(equalToConstant: 22).isActive = true
+        heightAnchor.constraint(equalToConstant: 22).isActive = true
+
+        let label = NSTextField(labelWithString: number)
+        label.font = .systemFont(ofSize: 12, weight: .semibold)
+        label.textColor = .white
+        label.alignment = .center
+        label.translatesAutoresizingMaskIntoConstraints = false
+        addSubview(label)
+        NSLayoutConstraint.activate([
+            label.centerXAnchor.constraint(equalTo: centerXAnchor),
+            label.centerYAnchor.constraint(equalTo: centerYAnchor),
+        ])
+    }
+
+    required init?(coder: NSCoder) { nil }
+
+    override var wantsUpdateLayer: Bool { true }
+    override func updateLayer() {
+        layer?.backgroundColor = NSColor.controlAccentColor.cgColor
+    }
+}
+
+// MARK: - Inner tint (drag flow background)
+
+/// Subtle tinted background for the drag-flow visualization.
+/// Adapts to light/dark appearance via updateLayer().
+private final class InnerTintView: NSView {
+    override init(frame: NSRect) {
+        super.init(frame: frame)
+        wantsLayer = true
+        layer?.cornerRadius = 10
+    }
+
+    required init?(coder: NSCoder) { nil }
+
+    override var wantsUpdateLayer: Bool { true }
+    override func updateLayer() {
+        let isDark = effectiveAppearance.bestMatch(from: [.darkAqua, .aqua]) == .darkAqua
+        layer?.backgroundColor = isDark
+            ? NSColor.white.withAlphaComponent(0.05).cgColor
+            : NSColor.black.withAlphaComponent(0.04).cgColor
+    }
+}
+
+// MARK: - Privacy drop zone
+
+/// Dashed rounded rect showing where the user should drop the app icon.
+final class PrivacyDropZoneView: NSView {
+    private let label: NSTextField
+
+    override init(frame: NSRect) {
+        label = NSTextField(labelWithString: "Screen Recording\nlist")
+        super.init(frame: frame)
+        label.font = .systemFont(ofSize: 11)
+        label.textColor = .tertiaryLabelColor
+        label.alignment = .center
+        label.maximumNumberOfLines = 2
+        label.translatesAutoresizingMaskIntoConstraints = false
+        addSubview(label)
+        NSLayoutConstraint.activate([
+            label.centerXAnchor.constraint(equalTo: centerXAnchor),
+            label.centerYAnchor.constraint(equalTo: centerYAnchor),
+            label.leadingAnchor.constraint(greaterThanOrEqualTo: leadingAnchor, constant: 4),
+            label.trailingAnchor.constraint(lessThanOrEqualTo: trailingAnchor, constant: -4),
+        ])
+    }
+
+    required init?(coder: NSCoder) { nil }
+
+    override func draw(_ dirtyRect: NSRect) {
+        super.draw(dirtyRect)
+        NSGraphicsContext.saveGraphicsState()
+        let path = NSBezierPath(roundedRect: bounds.insetBy(dx: 1, dy: 1), xRadius: 8, yRadius: 8)
+        path.lineWidth = 1.5
+        path.setLineDash([6.0, 4.0], count: 2, phase: 0)
+        NSColor.tertiaryLabelColor.setStroke()
+        path.stroke()
+        NSGraphicsContext.restoreGraphicsState()
+    }
+}
+
+// MARK: - Draggable app icon
+
+/// NSImageView that initiates a drag session carrying the app bundle URL,
+/// so it can be dropped into System Settings > Privacy & Security > Screen Recording.
+final class DraggableAppIconView: NSImageView, NSDraggingSource {
+    override init(frame: NSRect) {
+        super.init(frame: frame)
+        wantsLayer = true
+        layer?.cornerRadius = 12
+        layer?.masksToBounds = true
+        addTrackingArea(NSTrackingArea(
+            rect: .zero,
+            options: [.mouseEnteredAndExited, .activeAlways, .inVisibleRect],
+            owner: self,
+            userInfo: nil
+        ))
+    }
+
+    required init?(coder: NSCoder) { nil }
+
+    override func mouseEntered(with event: NSEvent) {
+        NSAnimationContext.runAnimationGroup { ctx in
+            ctx.duration = 0.12
+            animator().alphaValue = 0.75
+        }
+    }
+
+    override func mouseExited(with event: NSEvent) {
+        NSAnimationContext.runAnimationGroup { ctx in
+            ctx.duration = 0.12
+            animator().alphaValue = 1.0
+        }
+    }
+
+    override func mouseDown(with event: NSEvent) {
+        guard let image else { return }
+        let appURL = Bundle.main.bundleURL
+        let item = NSPasteboardItem()
+        item.setString(appURL.absoluteString, forType: .fileURL)
+        let dragItem = NSDraggingItem(pasteboardWriter: item)
+        dragItem.setDraggingFrame(bounds, contents: image)
+        beginDraggingSession(with: [dragItem], event: event, source: self)
+    }
+
+    func draggingSession(
+        _ session: NSDraggingSession,
+        sourceOperationMaskFor context: NSDraggingContext
+    ) -> NSDragOperation {
+        .copy
+    }
+}
+
+

packages/handsfree/native/HandsFree/Sources/ComputerUse/Types.swift

@@ -0,0 +1,248 @@
+import AppKit
+import ApplicationServices
+
+struct ElementFrame: Sendable {
+    let x: Int
+    let y: Int
+    let width: Int
+    let height: Int
+
+    var center: CGPoint {
+        CGPoint(x: x + width / 2, y: y + height / 2)
+    }
+
+    var dictionary: [String: Any] {
+        ["x": x, "y": y, "width": width, "height": height]
+    }
+}
+
+struct AXElementState: Sendable {
+    let enabled: Bool?
+    let focused: Bool?
+    let selected: Bool?
+    let expanded: Bool?
+    let checked: Bool?
+
+    var dictionary: [String: Any] {
+        var result: [String: Any] = [:]
+        if let enabled { result["enabled"] = enabled }
+        if let focused { result["focused"] = focused }
+        if let selected { result["selected"] = selected }
+        if let expanded { result["expanded"] = expanded }
+        if let checked { result["checked"] = checked }
+        return result
+    }
+}
+
+struct AXElementCapabilities: Sendable {
+    let canPress: Bool
+    let canFocus: Bool
+    let canScroll: Bool
+    let canAdjust: Bool
+    let canSetValue: Bool
+    let actions: [String]
+
+    var dictionary: [String: Any] {
+        [
+            "press": canPress,
+            "focus": canFocus,
+            "scroll": canScroll,
+            "adjust": canAdjust,
+            "setValue": canSetValue,
+            "actions": actions,
+        ]
+    }
+}
+
+struct SemanticAXElement: Identifiable, Sendable {
+    let id: Int
+    let ref: String
+    let role: String
+    let label: String
+    let value: String?
+    let frame: ElementFrame
+    let state: AXElementState
+    let capabilities: AXElementCapabilities
+
+    var dictionary: [String: Any] {
+        var result: [String: Any] = [
+            "id": id,
+            "ref": ref,
+            "role": role,
+            "label": label,
+            "frame": frame.dictionary,
+            "state": state.dictionary,
+            "capabilities": capabilities.dictionary,
+        ]
+        if let value { result["value"] = value }
+        return result
+    }
+}
+
+struct AXElementRecord: @unchecked Sendable {
+    let element: AXUIElement
+    let semantic: SemanticAXElement
+}
+
+struct ScreenshotMetadata: Sendable {
+    let imageWidth: Int
+    let imageHeight: Int
+    let capturedBounds: CGRect
+
+    var scaleX: CGFloat { capturedBounds.width / CGFloat(imageWidth) }
+    var scaleY: CGFloat { capturedBounds.height / CGFloat(imageHeight) }
+
+    func toScreen(imageX: Double, imageY: Double) -> CGPoint {
+        CGPoint(
+            x: capturedBounds.origin.x + imageX * scaleX,
+            y: capturedBounds.origin.y + imageY * scaleY
+        )
+    }
+
+    func toImage(point: CGPoint) -> CGPoint {
+        CGPoint(
+            x: (point.x - capturedBounds.origin.x) / scaleX,
+            y: (point.y - capturedBounds.origin.y) / scaleY
+        )
+    }
+
+    var dictionary: [String: Any] {
+        [
+            "imageWidth": imageWidth,
+            "imageHeight": imageHeight,
+            "capturedBounds": [
+                "x": Int(capturedBounds.origin.x),
+                "y": Int(capturedBounds.origin.y),
+                "width": Int(capturedBounds.width),
+                "height": Int(capturedBounds.height),
+            ],
+        ]
+    }
+}
+
+struct WindowTarget: @unchecked Sendable {
+    let appName: String
+    let pid: pid_t
+    let windowNumber: Int?
+    let windowTitle: String?
+    let bounds: CGRect
+    let isFrontmost: Bool
+    let axWindow: AXUIElement?
+
+    var center: CGPoint {
+        CGPoint(x: bounds.midX, y: bounds.midY)
+    }
+}
+
+struct AppSnapshot: @unchecked Sendable {
+    let id: String
+    let observation: Int
+    let appName: String
+    let pid: pid_t
+    let windowNumber: Int?
+    let windowTitle: String?
+    let screenshotData: Data
+    let screenshotMimeType: String
+    let screenshotMeta: ScreenshotMetadata
+    let records: [AXElementRecord]
+    let strictMode: Bool
+    let backgroundActivated: Bool
+    let recentActions: [String]
+    let addedLabels: [String]
+    let removedLabels: [String]
+
+    var elements: [SemanticAXElement] {
+        records.map(\.semantic)
+    }
+
+    func withSessionMetadata(id: String, observation: Int, recentActions: [String], addedLabels: [String], removedLabels: [String]) -> AppSnapshot {
+        AppSnapshot(
+            id: id,
+            observation: observation,
+            appName: appName,
+            pid: pid,
+            windowNumber: windowNumber,
+            windowTitle: windowTitle,
+            screenshotData: screenshotData,
+            screenshotMimeType: screenshotMimeType,
+            screenshotMeta: screenshotMeta,
+            records: records,
+            strictMode: strictMode,
+            backgroundActivated: backgroundActivated,
+            recentActions: recentActions,
+            addedLabels: addedLabels,
+            removedLabels: removedLabels
+        )
+    }
+}
+
+enum ExecutionPath: String {
+    case accessibility = "accessibility"
+    case backgroundCGEvent = "background_cgevent"
+    case foregroundCGEvent = "foreground_cgevent"
+    case none = "none"
+}
+
+struct ActionMetadata: Sendable {
+    let ok: Bool
+    let path: ExecutionPath
+    let strictMode: Bool
+    let backgroundSafe: Bool
+    let fallbackUsed: Bool
+    let message: String
+
+    var dictionary: [String: Any] {
+        [
+            "ok": ok,
+            "path": path.rawValue,
+            "strictMode": strictMode,
+            "backgroundSafe": backgroundSafe,
+            "fallbackUsed": fallbackUsed,
+            "message": message,
+        ]
+    }
+}
+
+enum ComputerUseError: LocalizedError {
+    case accessibilityDenied
+    case screenshotFailed
+    case appNotFound(String)
+    case noFrontmostApplication
+    case noWindow(String)
+    case noSnapshot
+    case staleSnapshot(String)
+    case invalidElement(String)
+    case strictModeViolation(String)
+    case eventSourceFailed
+    case eventCreationFailed
+    case unknownKey(String)
+
+    var errorDescription: String? {
+        switch self {
+        case .accessibilityDenied:
+            return "Accessibility permission is not granted."
+        case .screenshotFailed:
+            return "Screenshot capture failed."
+        case .appNotFound(let name):
+            return "App '\(name)' is not running."
+        case .noFrontmostApplication:
+            return "No frontmost application found."
+        case .noWindow(let app):
+            return "No usable window found for \(app)."
+        case .noSnapshot:
+            return "No snapshot is available. Call snapshot first."
+        case .staleSnapshot(let reason):
+            return "The last snapshot is stale. \(reason)"
+        case .invalidElement(let ref):
+            return "Element \(ref) was not found in the last semantic snapshot."
+        case .strictModeViolation(let reason):
+            return "Strict background mode rejected the action: \(reason)"
+        case .eventSourceFailed:
+            return "Failed to create CGEventSource."
+        case .eventCreationFailed:
+            return "Failed to create CGEvent."
+        case .unknownKey(let key):
+            return "Unknown key: \(key)"
+        }
+    }
+}

packages/handsfree/native/HandsFree/Sources/ComputerUse/main.swift

@@ -0,0 +1,65 @@
+/// Computer Use: semantic AX and background-safe macOS computer use.
+///
+/// Four modes:
+///   mcp          — run the MCP server over stdio
+///   --check      — print permission status as JSON to stdout and exit
+///   --list-apps  — print running regular apps as JSON to stdout and exit
+///   (default)    — open the permission setup GUI
+
+import AppKit
+import Foundation
+
+setbuf(stdout, nil)
+
+let args = CommandLine.arguments
+let subcommand = args.count >= 2 ? args[1] : ""
+
+switch subcommand {
+case "mcp":
+    if ProcessInfo.processInfo.environment["OPENWORK_COMPUTER_USE_CURSOR_OVERLAY"] == "0" {
+        let server = MCPServer()
+        await server.run()
+    } else {
+        await runMCPServerWithOverlay()
+    }
+case "--check":
+    // Fresh process → fresh TCC read → always accurate.
+    let status = ComputerUsePermissions.status()
+    let json = "{\"ok\":\(status.ok),\"accessibility\":\(status.accessibility),\"screenRecording\":\(status.screenRecording)}"
+    print(json)
+    exit(0)
+case "--list-apps":
+    // Running regular apps (Dock-visible). Needs no TCC permissions, so this
+    // is safe to call from the composer at any time.
+    let apps = NSWorkspace.shared.runningApplications
+        .filter { $0.activationPolicy == .regular }
+        .compactMap(\.localizedName)
+        .sorted { $0.localizedCaseInsensitiveCompare($1) == .orderedAscending }
+    let payload: [String: Any] = ["ok": true, "apps": apps]
+    let data = (try? JSONSerialization.data(withJSONObject: payload)) ?? Data("{\"ok\":false,\"apps\":[]}".utf8)
+    print(String(decoding: data, as: UTF8.self))
+    exit(0)
+default:
+    await runPermissionSetupApp()
+}
+
+@MainActor
+func runMCPServerWithOverlay() async {
+    NSApplication.shared.setActivationPolicy(.accessory)
+    let server = MCPServer()
+    Task.detached {
+        await server.run()
+        await MainActor.run {
+            NSApplication.shared.terminate(nil)
+        }
+    }
+    NSApplication.shared.run()
+}
+
+@MainActor
+func runPermissionSetupApp() async {
+    NSApplication.shared.setActivationPolicy(.regular)
+    let appDelegate = PermissionSetupAppDelegate()
+    NSApplication.shared.delegate = appDelegate
+    NSApplication.shared.run()
+}

packages/handsfree/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@openwork/handsfree",
+  "private": true,
+  "version": "0.1.0",
+  "description": "macOS semantic AX and background computer-use runtime for OpenWork",
+  "license": "MIT",
+  "type": "module",
+  "bin": {
+    "openwork-handsfree-computer-use": "bin/openwork-handsfree-computer-use.mjs"
+  },
+  "scripts": {
+    "check": "pnpm run check:js && pnpm run check:native",
+    "check:js": "node --check bin/openwork-handsfree-computer-use.mjs && node --check src/cua-runner.mjs && node --check src/realtime-tools.mjs && node --check test/e2e/run.mjs",
+    "build:native": "swift build --package-path native/HandsFree -c release --product HandsFreeComputerUse",
+    "check:native": "swift build --package-path native/HandsFree --product HandsFreeComputerUse",
+    "test:e2e": "node test/e2e/run.mjs"
+  },
+  "files": [
+    "bin",
+    "src",
+    "native",
+    "README.md"
+  ]
+}

packages/handsfree/src/cua-runner.mjs

@@ -0,0 +1,134 @@
+export const CUA_DEFAULT_MODEL = "gpt-5.5";
+export const CUA_MAX_TURNS = 30;
+
+export async function runCuaLoop({
+  task,
+  apiKey,
+  callTool,
+  onProgress,
+  signal,
+  model = CUA_DEFAULT_MODEL,
+  maxTurns = CUA_MAX_TURNS,
+}) {
+  if (!apiKey?.trim()) throw new Error("OpenAI API key required for computer use.");
+  if (typeof callTool !== "function") throw new Error("callTool is required.");
+
+  const display = await callTool("display_info", {});
+  const displayInfo = parseToolText(display) ?? { width: 1440, height: 900 };
+  onProgress?.({ kind: "start", width: displayInfo.width, height: displayInfo.height });
+
+  const items = [{ role: "user", content: String(task ?? "") }];
+  const messages = [];
+
+  for (let turn = 0; turn < maxTurns; turn += 1) {
+    if (signal?.aborted) return { ok: true, messages, turns: turn, aborted: true };
+    onProgress?.({ kind: "turn", turn: turn + 1 });
+
+    const response = await fetch("https://api.openai.com/v1/responses", {
+      method: "POST",
+      headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" },
+      body: JSON.stringify({ model, input: items, tools: [{ type: "computer" }] }),
+      signal,
+    });
+
+    if (!response.ok) {
+      const errorText = await response.text().catch(() => "");
+      throw new Error(`CUA API error ${response.status}: ${errorText.slice(0, 300)}`);
+    }
+
+    const result = await response.json();
+    const output = result.output || [];
+    if (!output.length) throw new Error("No output from CUA model.");
+    items.push(...output);
+
+    let computerCall = null;
+    for (const item of output) {
+      if (item.type === "message") {
+        const text = item.content?.map((part) => part.text || "").join("") || "";
+        if (text) {
+          messages.push(text);
+          onProgress?.({ kind: "message", text });
+        }
+      }
+      if (item.type === "computer_call") computerCall = item;
+    }
+
+    if (!computerCall) return { ok: true, messages, turns: turn + 1 };
+
+    for (const action of computerCall.actions || (computerCall.action ? [computerCall.action] : [])) {
+      if (signal?.aborted) return { ok: true, messages, turns: turn + 1, aborted: true };
+      if (action.type === "screenshot") continue;
+      onProgress?.({ kind: "action", ...summarizeAction(action) });
+      const actionResult = await executeCuaAction(callTool, action);
+      const actionPayload = parseToolText(actionResult);
+      if (actionPayload?.ok === false) {
+        if (actionPayload.requiredNextAction === "snapshot") break;
+        throw new Error(actionPayload.error || `Computer action failed: ${action.type}`);
+      }
+      await delay(150);
+    }
+
+    const screenshot = await callTool("cua_screenshot", {});
+    const image = extractImage(screenshot);
+    if (!image) throw new Error("Could not capture screenshot after action.");
+
+    items.push({
+      type: "computer_call_output",
+      call_id: computerCall.call_id,
+      acknowledged_safety_checks: computerCall.pending_safety_checks || [],
+      output: { type: "input_image", image_url: `data:image/png;base64,${image}` },
+    });
+  }
+
+  return { ok: true, messages, turns: maxTurns, truncated: true };
+}
+
+export async function executeCuaAction(callTool, action) {
+  switch (action.type) {
+    case "click":
+      return callTool("cua_click", { x: action.x, y: action.y, button: action.button || "left", ...(action.keys?.length ? { keys: action.keys } : {}) });
+    case "double_click":
+      return callTool("cua_double_click", { x: action.x, y: action.y });
+    case "scroll":
+      return callTool("cua_scroll", { x: action.x, y: action.y, scroll_x: action.scroll_x || 0, scroll_y: action.scroll_y || 0 });
+    case "type":
+      return callTool("cua_type", { text: action.text });
+    case "keypress":
+      return callTool("cua_keypress", { keys: action.keys || [] });
+    case "drag":
+      return callTool("cua_drag", { path: action.path || [] });
+    case "move":
+      return callTool("cua_move", { x: action.x, y: action.y });
+    case "wait":
+      return callTool("cua_wait", {});
+    default:
+      return null;
+  }
+}
+
+function parseToolText(response) {
+  const text = response?.result?.content?.find?.((item) => item.type === "text")?.text
+    ?? response?.content?.find?.((item) => item.type === "text")?.text;
+  if (!text) return null;
+  try { return JSON.parse(text); } catch { return null; }
+}
+
+function extractImage(response) {
+  return response?.result?.content?.find?.((item) => item.type === "image" && item.data)?.data
+    ?? response?.content?.find?.((item) => item.type === "image" && item.data)?.data
+    ?? null;
+}
+
+function summarizeAction(action) {
+  return {
+    type: action.type,
+    x: action.x,
+    y: action.y,
+    text: action.text?.slice?.(0, 60),
+    desc: `${action.type}${action.x != null ? ` (${action.x},${action.y})` : ""}${action.text ? ` "${action.text.slice(0, 30)}"` : ""}`,
+  };
+}
+
+function delay(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

packages/handsfree/src/realtime-tools.mjs

@@ -0,0 +1,85 @@
+export const HANDSFREE_DEFAULT_MODEL = "gpt-realtime-2";
+export const HANDSFREE_DEFAULT_REASONING_EFFORT = "low";
+export const HANDSFREE_REASONING_EFFORTS = ["minimal", "low", "medium", "high", "xhigh"];
+
+export const HANDSFREE_REALTIME_INSTRUCTIONS = `# Role and Objective
+
+You are Computer Use, a macOS computer-control voice assistant. You control the user's Mac through tools. You respond with voice. You cannot see the screen yourself.
+
+# Personality and Tone
+
+Be concise, calm, and direct. Do not over-explain. Act, then report the result.
+
+# Tool Selection
+
+- Direct typing, keypresses, app launch, clipboard, URLs, and grid clicks are instant tools.
+- Visual or multi-step UI work must use use_computer.
+- Stop/cancel requests must call stop_computer immediately.
+- For MCP servers, list tools before calling unfamiliar tool names.
+
+# Safety
+
+- Type exactly what the user asks; do not paraphrase typed text.
+- Do not use destructive shortcuts unless explicitly requested.
+- For actions that send messages or modify data, confirm content briefly before executing.`;
+
+export function openAIRealtimeTools() {
+  return [
+    functionTool("use_computer", "Control the Mac to complete a visual or UI task using screenshots and native input.", {
+      task: { type: "string", description: "Plain-language task to complete on the computer." },
+    }, ["task"]),
+    functionTool("type_text", "Type exact text into the focused input field.", {
+      text: { type: "string", description: "Exact text to type." },
+    }, ["text"]),
+    functionTool("press_key", "Press a key combo such as return, tab, escape, command+k, or command+shift+a.", {
+      combo: { type: "string", description: "Key combo string." },
+    }, ["combo"]),
+    functionTool("launch_app", "Launch a macOS app by name.", {
+      name: { type: "string", description: "App name." },
+    }, ["name"]),
+    functionTool("activate_app", "Bring a running macOS app to the foreground.", {
+      name: { type: "string", description: "App name." },
+    }, ["name"]),
+    functionTool("list_apps", "List running macOS applications."),
+    functionTool("clipboard_read", "Read the macOS clipboard as text."),
+    functionTool("clipboard_write", "Write text to the macOS clipboard.", {
+      text: { type: "string", description: "Text to copy." },
+    }, ["text"]),
+    functionTool("open_url", "Open a URL in a browser.", {
+      url: { type: "string", description: "URL to open." },
+      app: { type: "string", description: "Optional browser app name." },
+    }, ["url"]),
+    functionTool("mcp_list_servers", "List connected MCP servers."),
+    functionTool("mcp_list_tools", "List tools on a connected MCP server before calling unfamiliar tools.", {
+      serverName: { type: "string", description: "MCP server name." },
+    }, ["serverName"]),
+    functionTool("mcp_call_tool", "Call a tool on a connected MCP server.", {
+      serverName: { type: "string", description: "MCP server name." },
+      toolName: { type: "string", description: "Tool name." },
+      args: { type: "object", description: "Tool arguments.", additionalProperties: true },
+    }, ["serverName", "toolName"]),
+    functionTool("show_grid", "Show a subtle A1-F4 screen grid overlay."),
+    functionTool("hide_grid", "Hide the screen grid overlay."),
+    functionTool("click_grid", "Click the center of a grid zone such as C2.", {
+      zone: { type: "string", description: "Grid zone label." },
+    }, ["zone"]),
+    functionTool("stop_computer", "Stop the current computer-use task."),
+    functionTool("request_permission", "Open System Settings for a macOS permission pane.", {
+      pane: { type: "string", description: "accessibility, screen-recording, or microphone." },
+    }, ["pane"]),
+  ];
+}
+
+function functionTool(name, description, properties = {}, required = []) {
+  return {
+    type: "function",
+    name,
+    description,
+    parameters: {
+      type: "object",
+      properties,
+      required,
+      additionalProperties: false,
+    },
+  };
+}

packages/handsfree/test/e2e/bench.html

@@ -0,0 +1,34 @@
+<!doctype html>
+<html>
+<head>
+<meta charset="utf-8">
+<title>CU-BENCH</title>
+<style>
+  body { font-family: -apple-system, sans-serif; margin: 24px; }
+  .grid { display: grid; grid-template-columns: repeat(4, 160px); gap: 12px; margin: 16px 0; }
+  button { padding: 12px; font-size: 14px; }
+  input, textarea { width: 480px; font-size: 14px; padding: 8px; display: block; margin: 8px 0; }
+</style>
+</head>
+<body>
+<h1>Computer Use Benchmark</h1>
+<div class="grid" id="grid"></div>
+<input id="field" placeholder="type here" aria-label="bench-field">
+<textarea id="area" rows="3" aria-label="bench-area" placeholder="textarea here"></textarea>
+<label><input type="checkbox" id="check" aria-label="bench-check"> bench-check</label>
+<script>
+  window.__events = [];
+  const names = ["alpha","bravo","charlie","delta","echo","foxtrot","golf","hotel","india","juliett","kilo","lima"];
+  const grid = document.getElementById("grid");
+  for (const n of names) {
+    const b = document.createElement("button");
+    b.id = "btn-" + n;
+    b.textContent = "btn-" + n;
+    b.addEventListener("click", () => window.__events.push({ t: Date.now(), type: "click", id: b.id }));
+    grid.appendChild(b);
+  }
+  document.getElementById("check").addEventListener("change", (e) =>
+    window.__events.push({ t: Date.now(), type: "check", checked: e.target.checked }));
+</script>
+</body>
+</html>

packages/handsfree/test/e2e/run.mjs

@@ -0,0 +1,224 @@
+// Computer-use e2e quality benchmark.
+//
+// Measures snapshot/click/type accuracy and latency against a deterministic
+// local HTML page in Chrome, with ground truth verified independently via the
+// Chrome DevTools Protocol (so accuracy is not measured by the system under
+// test).
+//
+// Requirements (macOS only):
+// - The signed helper app with Accessibility + Screen Recording granted:
+//   run `node apps/desktop/scripts/prepare-computer-use-helper.mjs` and grant
+//   permissions once via the helper's setup GUI.
+// - Google Chrome installed. The test launches a disposable instance with a
+//   temp profile and closes it afterwards. No accessibility flags are passed;
+//   the runtime is expected to enable renderer AX itself.
+//
+// Usage: node packages/handsfree/test/e2e/run.mjs
+
+import { spawn, execSync } from "node:child_process";
+import { createReadStream, createWriteStream, mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const repoRoot = resolve(__dirname, "../../../..");
+const HELPER_APP = process.env.OPENWORK_COMPUTER_USE_HELPER
+  ?? join(repoRoot, "apps/desktop/resources/helpers/OpenWork Computer Use.app");
+const PAGE = `file://${join(__dirname, "bench.html")}`;
+const CDP_PORT = Number(process.env.CU_BENCH_CDP_PORT ?? 9224);
+const CHROME_APP = "Google Chrome";
+
+if (process.platform !== "darwin") {
+  console.log("computer-use e2e benchmark is macOS-only; skipping");
+  process.exit(0);
+}
+
+// --- MCP bridge: launch the helper via Launch Services with FIFO stdio so
+// --- TCC permissions attribute to the helper bundle, not this shell.
+const fifoDir = mkdtempSync(join(tmpdir(), "cu-e2e-"));
+const IN = join(fifoDir, "in.fifo");
+const OUT = join(fifoDir, "out.fifo");
+execSync(`mkfifo ${IN} ${OUT}`);
+try { execSync(`pkill -f 'OpenWork Computer Use.app'`); } catch {}
+spawn("open", ["--stdin", IN, "--stdout", OUT, HELPER_APP, "--args", "mcp"], { stdio: "ignore" });
+
+const mcpOut = createReadStream(OUT, { encoding: "utf8" });
+const mcpIn = createWriteStream(IN);
+let mcpBuf = "";
+const mcpPending = new Map();
+mcpOut.on("data", (chunk) => {
+  mcpBuf += chunk;
+  let idx;
+  while ((idx = mcpBuf.indexOf("\n")) >= 0) {
+    const line = mcpBuf.slice(0, idx).trim();
+    mcpBuf = mcpBuf.slice(idx + 1);
+    if (!line) continue;
+    let msg;
+    try { msg = JSON.parse(line); } catch { continue; }
+    if (msg.id != null && mcpPending.has(msg.id)) {
+      mcpPending.get(msg.id)(msg);
+      mcpPending.delete(msg.id);
+    }
+  }
+});
+
+let rpcId = 0;
+function rpc(method, params, timeoutMs = 30000) {
+  const id = ++rpcId;
+  return new Promise((res, rej) => {
+    const t = setTimeout(() => rej(new Error(`mcp timeout: ${method}`)), timeoutMs);
+    mcpPending.set(id, (msg) => { clearTimeout(t); res(msg); });
+    mcpIn.write(JSON.stringify({ jsonrpc: "2.0", id, method, params }) + "\n");
+  });
+}
+
+const results = { latency: {}, accuracy: {}, paths: {} };
+async function cu(name, args = {}) {
+  const t0 = performance.now();
+  const msg = await rpc("tools/call", { name, arguments: args });
+  (results.latency[name] ??= []).push(performance.now() - t0);
+  const text = msg.result?.content?.find((c) => c.type === "text")?.text;
+  const json = JSON.parse(text ?? "{}");
+  if (json.path) (results.paths[json.path] ??= []).push(name);
+  return json;
+}
+
+// --- disposable Chrome with CDP ground truth ---
+const profileDir = mkdtempSync(join(tmpdir(), "cu-e2e-chrome-"));
+spawn("open", ["-na", CHROME_APP, "--args",
+  `--remote-debugging-port=${CDP_PORT}`, `--user-data-dir=${profileDir}`,
+  "--no-first-run", "--no-default-browser-check", "--window-size=1200,900", PAGE,
+], { stdio: "ignore" });
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+async function cdpTargets() {
+  for (let i = 0; i < 30; i++) {
+    try {
+      const r = await fetch(`http://127.0.0.1:${CDP_PORT}/json`);
+      return await r.json();
+    } catch { await sleep(500); }
+  }
+  throw new Error(`Chrome CDP endpoint did not come up on :${CDP_PORT}`);
+}
+
+await rpc("initialize", { protocolVersion: "2024-11-05", capabilities: {}, clientInfo: { name: "cu-e2e", version: "0" } });
+mcpIn.write(JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" }) + "\n");
+
+const targets = await cdpTargets();
+const tab = targets.find((t) => t.type === "page" && t.url.includes("bench.html"));
+if (!tab) throw new Error("bench.html tab not found in disposable Chrome");
+const ws = new WebSocket(tab.webSocketDebuggerUrl);
+await new Promise((res, rej) => { ws.onopen = res; ws.onerror = rej; });
+let wsId = 0;
+const wsPending = new Map();
+ws.onmessage = (ev) => {
+  const m = JSON.parse(ev.data);
+  if (m.id && wsPending.has(m.id)) { wsPending.get(m.id)(m); wsPending.delete(m.id); }
+};
+const evaljs = (expression) => new Promise((res, rej) => {
+  const i = ++wsId;
+  const t = setTimeout(() => rej(new Error("cdp timeout")), 15000);
+  wsPending.set(i, (m) => { clearTimeout(t); res(m.result?.result?.value); });
+  ws.send(JSON.stringify({ id: i, method: "Runtime.evaluate", params: { expression, returnByValue: true } }));
+});
+
+await sleep(2000);
+const findEl = (snap, pred) => snap.elements?.find(pred);
+
+let failures = 0;
+const record = (key, ok, total, note = "") => {
+  results.accuracy[key] = `${ok}/${total}`;
+  if (ok !== total) { failures++; if (note) console.log(note); }
+};
+
+// 1. element recall
+let snap = await cu("snapshot", { app: CHROME_APP });
+const buttons = ["btn-alpha","btn-bravo","btn-charlie","btn-delta","btn-echo","btn-foxtrot","btn-golf","btn-hotel","btn-india","btn-juliett","btn-kilo","btn-lima"];
+const expected = [...buttons, "bench-field", "bench-area", "bench-check"];
+const labels = (snap.elements ?? []).map((e) => `${e.label ?? ""} ${e.value ?? ""}`).join(" | ");
+const found = expected.filter((x) => labels.includes(x));
+record("element_recall", found.length, expected.length, `missing: ${expected.filter((x) => !found.includes(x)).join(", ")}`);
+
+// 2. click accuracy across all three addressing modes, CDP-verified
+async function clickRound(key, makeArgs) {
+  let ok = 0;
+  await evaljs("window.__events = []");
+  for (const name of buttons) {
+    const el = findEl(snap, (e) => e.role === "Button" && (e.label ?? "").includes(name));
+    if (!el) continue;
+    const before = (await evaljs("window.__events.length")) ?? 0;
+    await cu("click", { app: CHROME_APP, ...makeArgs(el) });
+    await sleep(150);
+    const last = await evaljs("window.__events[window.__events.length-1]?.id");
+    const after = await evaljs("window.__events.length");
+    if (after === before + 1 && last === name) ok++;
+    else console.log(`${key} MISS: ${name} -> ${last}`);
+  }
+  record(key, ok, buttons.length);
+}
+await clickRound("click_by_ref", (el) => ({ ref: el.ref }));
+snap = await cu("snapshot", { app: CHROME_APP });
+await clickRound("click_by_image_coord", (el) => ({ x: el.center.imageX, y: el.center.imageY }));
+await clickRound("click_by_screen_coord", (el) => ({ screen_x: el.center.screenX, screen_y: el.center.screenY }));
+
+// 3. pid targeting
+const apps = await cu("list_apps");
+const chrome = (apps.appDetails ?? []).find((a) => a.name === CHROME_APP);
+const pidSnap = chrome ? await cu("snapshot", { pid: chrome.pid }) : {};
+record("snapshot_by_pid", pidSnap.windowTitle?.includes("CU-BENCH") ? 1 : 0, 1);
+
+// 4. typing accuracy (incl. symbols), CDP-verified
+const strings = [
+  "The quick brown fox jumps over the lazy dog 0123456789",
+  "~!@#$%^&*()_+-=[]{}|;':\",./<>?",
+  "CamelCase snake_case kebab-case 42.5%",
+];
+let typeOk = 0;
+for (const text of strings) {
+  await evaljs("document.getElementById('field').value=''; document.getElementById('field').focus()");
+  await sleep(100);
+  await cu("type_text", { app: CHROME_APP, text });
+  await sleep(300);
+  if ((await evaljs("document.getElementById('field').value")) === text) typeOk++;
+}
+record("type_text", typeOk, strings.length);
+
+// 5. set_value with verify + retype fallback
+snap = await cu("snapshot", { app: CHROME_APP });
+const field = findEl(snap, (e) => (e.label ?? "").includes("bench-field"));
+await cu("set_value", { app: CHROME_APP, ref: field.ref, value: "set-value-test-42" });
+await sleep(300);
+record("set_value", (await evaljs("document.getElementById('field').value")) === "set-value-test-42" ? 1 : 0, 1);
+
+// 6. background modifier combo: cmd+a + retype
+await cu("snapshot", { app: CHROME_APP });
+await evaljs("document.getElementById('field').focus()");
+await cu("press_key", { app: CHROME_APP, combo: "command+a" });
+await sleep(100);
+await cu("type_text", { app: CHROME_APP, text: "replaced" });
+await sleep(300);
+record("select_all_replace", (await evaljs("document.getElementById('field').value")) === "replaced" ? 1 : 0, 1);
+
+// --- report ---
+console.log("\n========== RESULTS ==========");
+console.log("\n-- Accuracy --");
+for (const [k, v] of Object.entries(results.accuracy)) console.log(`${k.padEnd(24)} ${v}`);
+console.log("\n-- Latency (ms) --");
+for (const [k, v] of Object.entries(results.latency)) {
+  const s = [...v].sort((a, b) => a - b);
+  const avg = Math.round(s.reduce((a, b) => a + b, 0) / s.length);
+  console.log(`${k.padEnd(24)} n=${String(s.length).padEnd(4)} avg=${String(avg).padEnd(6)} min=${Math.round(s[0]).toString().padEnd(6)} max=${Math.round(s[s.length - 1])}`);
+}
+console.log("\n-- Execution paths --");
+for (const [k, v] of Object.entries(results.paths)) console.log(`${k.padEnd(24)} ${v.length}x`);
+
+// --- cleanup ---
+try { execSync(`pkill -f "user-data-dir=${profileDir}"`); } catch {}
+try { execSync(`pkill -f 'OpenWork Computer Use.app'`); } catch {}
+await sleep(1000);
+try { rmSync(fifoDir, { recursive: true, force: true }); } catch {}
+try { rmSync(profileDir, { recursive: true, force: true }); } catch {}
+
+console.log(failures === 0 ? "\nRESULT: PASS" : `\nRESULT: FAIL (${failures} failing checks)`);
+process.exit(failures === 0 ? 0 : 1);