diff --git a/.gitignore b/.gitignore index 744b61c..3a8a21f 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,10 @@ dist/ .output/ .wxt/ .web-extension-id +.pnpm-store/ + +# Git worktrees +.worktrees/ # Logs *.log diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a7d54f5 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,92 @@ +# Changelog + +All notable changes to BrowserSkill will be documented in this file. +The format is based on [Keep a Changelog](https://keepachangelog.com/). + +Starting from 0.2.0, CLI / Extension / DSH Plugin share the same version number. + +## [Unreleased] + +## [0.2.0] - 2026-09-02 + +### Added +- File transfer: upload and download support across CLI, Extension, and DSH Plugin +- File transfer: drag-and-drop upload (`drop-to-upload`) +- VOM semantic graph, name enrichment, and hover perception modules +- VOM hover probing (opt-in via `observe` parameter) +- DSH Plugin: browser tool parity with CLI commands +- Edge Add-ons automated publishing in CI +- Protocol upgrade reminder when CLI / Extension protocol versions differ +- Browser evaluation harness (`evals/browser/`) +- Unified release script (`scripts/release.mjs`) + +### Changed +- **Version scheme**: all three components now share the same semver +- VOM rendering algorithm optimizations +- Leaner SKILL.md agent instructions +- DSH Plugin: simplified browser commands +- Borrow confirmation UX — proactive focus and longer timeout +- PiP window now has a close button + +### Fixed +- Screenshot media type detection (was hard-coded to `image/png`) +- DSH Plugin session lifecycle stability +- DSH Plugin Cordis package ID mismatch +- Observation thumbnail media type sniffing +- Upload/download race conditions and layout bypass issues +- VOM repeated name and safety policy issues + +--- + +*Previous releases used independent version numbers per component.* + +## CLI 0.1.11 / Extension 0.1.7 / DSH Plugin 0.1.2 — 2026-08-26 ~ 2026-08-29 + +### Added +- DSH Plugin sidebar integration, session lifecycle, and archive cleanup +- Recorder iframe and OOPIF support + +### Changed +- VOM functional refactor + +### Fixed +- Recorder safety policy and bug fixes + +## CLI 0.1.10 / Extension 0.1.6 — 2026-08-08 + +### Added +- VOM observation recording and settled-state detection +- Record overlay timer + +### Fixed +- Browser keepalive disconnect handling + +## CLI 0.1.9 / Extension 0.1.5 — 2026-07-29 + +### Added +- CLI auto-update mechanism +- Trace v3 protocol and recorder + +### Fixed +- MV3 keepalive disconnect + +## CLI 0.1.8 / Extension 0.1.4 — 2026-07-22 + +### Added +- More browser interaction actions + +### Fixed +- Windows named-pipe hash-only path issue + +## CLI 0.1.7 / Extension 0.1.3 — 2026-07-07 + +Initial public release pair. + +## CLI 0.1.6 — 2026-06-30 + +### Fixed +- Minor CLI fixes + +## CLI 0.1.5 / Extension 0.1.2 — 2026-06-22 + +First tagged releases. diff --git a/Cargo.lock b/Cargo.lock index a34b300..d4ff61c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,9 +134,10 @@ dependencies = [ [[package]] name = "bsk" -version = "0.1.11" +version = "0.2.0" dependencies = [ "anyhow", + "base64", "bsk-protocol", "clap", "console", @@ -168,7 +169,7 @@ dependencies = [ [[package]] name = "bsk-protocol" -version = "0.1.11" +version = "0.2.0" dependencies = [ "schemars", "semver", diff --git a/Cargo.toml b/Cargo.toml index d38556a..78ee9cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ members = [ default-members = ["crates/bsk-cli"] [workspace.package] -version = "0.1.11" +version = "0.2.0" edition = "2024" rust-version = "1.85" license = "MIT" @@ -36,13 +36,14 @@ flate2 = "1.1.9" tar = "0.4.46" zip = { version = "7.2.0", default-features = false, features = ["deflate"] } sha2 = "0.11.0" +base64 = "0.22" tempfile = "3" nix = { version = "0.29", features = ["signal", "process"] } rand = "0.8" uuid = { version = "1", features = ["v4", "serde"] } # Workspace crates (version required for `cargo publish` of dependents) -bsk-protocol = { version = "0.1.9", path = "crates/bsk-protocol" } +bsk-protocol = { version = "0.2.0", path = "crates/bsk-protocol" } [profile.release] strip = "symbols" diff --git a/apps/extension/PRIVACY.md b/apps/extension/PRIVACY.md index 9517a3c..bfe4675 100644 --- a/apps/extension/PRIVACY.md +++ b/apps/extension/PRIVACY.md @@ -1,6 +1,6 @@ # BrowserSkill — Privacy Policy -**Last updated:** May 25, 2026 +**Last updated:** August 20, 2026 This Privacy Policy describes how the **BrowserSkill** browser extension (the "Extension") handles information when you install and use it. BrowserSkill is published as part of the open-source [BrowserSkill](https://github.com/Tencent/BrowserSkill) project. The source code is publicly auditable. @@ -14,7 +14,7 @@ BrowserSkill is a local automation bridge that lets AI coding agents (such as Cu ## 2. Single Purpose -The Extension's single purpose is to expose browser automation primitives (navigation, DOM observation, screenshots, clicks, form filling, tab management) to a locally running BrowserSkill daemon over a WebSocket connection on `127.0.0.1`, so that an AI agent invoked by the user can interact with web pages on the user's behalf. +The Extension's single purpose is to expose browser automation primitives (navigation, DOM observation, screenshots, clicks, form filling, task-scoped file transfer, and tab management) to a locally running BrowserSkill daemon over a WebSocket connection on `127.0.0.1`, so that an AI agent invoked by the user can interact with web pages on the user's behalf. ## 3. Data the Extension Accesses @@ -26,6 +26,7 @@ Depending on the commands the user (via their AI agent) sends to the local daemo | **User input simulated by the agent** | Mouse clicks, keystrokes, and form values that the AI agent dispatches through the Chrome DevTools Protocol (CDP). | Required to perform automation actions the user has asked the agent to do. | | **Tab and window metadata** | Tab IDs, URLs, titles, window IDs of the Agent Window and any tabs the user explicitly authorizes. | Required to target automation commands at the correct tab/window. | | **Local extension storage** | A randomly generated 8-character instance ID and an optional user-supplied label. | Used so the local daemon can recognize this browser instance across reconnects. No personal data is stored. | +| **File transfers requested by the agent** | Local files explicitly supplied to `bsk upload`, and the file created by a single `bsk download` action. | Required to attach a task file to a web page or return a browser-generated download to the invoking local agent. | | **OS notifications** | Permission to display a system notification when the agent requests to "borrow" one of the user's existing tabs. | Required to obtain explicit, per-tab user consent before the agent touches any pre-existing tab. | ## 4. Data the Extension Does **Not** Collect @@ -34,7 +35,7 @@ BrowserSkill does **not**: - Send any data to remote servers, the Extension's authors, or any third party. - Call any LLM, AI, or cloud API. The Extension contains no API keys, model identifiers, or remote endpoints. -- Read or transmit cookies, browsing history, bookmarks, downloads, saved passwords, or autofill data. +- Read or transmit cookies, browsing history, bookmarks, saved passwords, or autofill data. It observes only the download initiated by an active `bsk download` call, not download history generally. - Use webcam, microphone, geolocation, or any device sensor. - Include analytics, telemetry, crash reporting, advertising SDKs, or fingerprinting code. - Track users across websites or across sessions. @@ -50,6 +51,7 @@ The Extension requests the following Chrome permissions. Each is used solely for - **`alarms`** — Periodically wake the service worker to keep the local WebSocket connection alive. - **`idle`** — Detect when the device returns from idle/locked so the Extension can promptly re-establish the local WebSocket connection after the machine wakes. No idle data is stored or transmitted. - **`notifications`** — Show a system notification to obtain user approval before the agent borrows a user-owned tab. +- **`downloads`** — Correlate and route the one browser download initiated by an active `bsk download` command. If that claimed transaction fails, BrowserSkill cancels an in-progress file or removes its completed temporary browser file. It is not used to enumerate download history or alter unclaimed downloads. - **`storage`** — Persist a random instance ID and optional label in `chrome.storage.local`. - **Host permission ``** — Inject a small status overlay (showing "Agent Active") on pages controlled by the agent, and enable automation across whatever sites the user directs the agent to. The Extension does **not** read or transmit page content from sites the agent is not actively driving. @@ -61,6 +63,7 @@ All Extension activity stays on the user's local device. The only network traffi - The instance ID and optional label persist in `chrome.storage.local` until the user uninstalls the Extension or clears extension storage. - Page content, screenshots, DOM snapshots, and other observed data are returned to the local daemon in response to commands and are **not retained by the Extension**. They live only as long as the agent's tool call. +- Upload and download bytes are staged by the local daemon in a private, session-scoped directory. Download staging is removed after it is copied to the requested destination. Upload staging is retained until the session ends so a later form submission can still read the attached file. Remaining staging is removed when the session ends or disconnects, or when the daemon next starts after a crash. ## 8. User Control diff --git a/apps/extension/e2e/fixtures/file-transfer.html b/apps/extension/e2e/fixtures/file-transfer.html new file mode 100644 index 0000000..47e64f8 --- /dev/null +++ b/apps/extension/e2e/fixtures/file-transfer.html @@ -0,0 +1,81 @@ + + + + + BrowserSkill file-transfer fixture + + + +

Upload then download the same image

+

No image selected

+
+ + + Download uploaded image +
+
Drop an image here
+ + + + diff --git a/apps/extension/package.json b/apps/extension/package.json index 546a019..fdd9d4d 100644 --- a/apps/extension/package.json +++ b/apps/extension/package.json @@ -1,6 +1,6 @@ { "name": "@browser-skill/extension", - "version": "0.1.7", + "version": "0.2.0", "private": true, "type": "module", "scripts": { diff --git a/apps/extension/src/content/__tests__/BorrowConfirmationOverlay.test.tsx b/apps/extension/src/content/__tests__/BorrowConfirmationOverlay.test.tsx index d273ef8..f1a8368 100644 --- a/apps/extension/src/content/__tests__/BorrowConfirmationOverlay.test.tsx +++ b/apps/extension/src/content/__tests__/BorrowConfirmationOverlay.test.tsx @@ -23,7 +23,7 @@ describe("BorrowConfirmationOverlay", () => { id: "req-1", isActiveTab: true, tabTitle: "Example", - timeoutMs: 5000, + timeoutMs: 60_000, onAllow, onDeny, }, @@ -32,7 +32,7 @@ describe("BorrowConfirmationOverlay", () => { ); expect(screen.getByText("允许借用标签页？")).toBeTruthy(); - for (let i = 0; i < 5; i++) { + for (let i = 0; i < 60; i++) { await act(async () => { await vi.advanceTimersByTimeAsync(1000); }); @@ -66,7 +66,7 @@ describe("BorrowConfirmationOverlay", () => { id: "req-fallback", isActiveTab: true, tabTitle: "Example", - timeoutMs: 5000, + timeoutMs: 60_000, onAllow, onDeny, }, @@ -74,7 +74,7 @@ describe("BorrowConfirmationOverlay", () => { />, ); - for (let i = 0; i < 5; i++) { + for (let i = 0; i < 60; i++) { await act(async () => { await vi.advanceTimersByTimeAsync(1000); }); diff --git a/apps/extension/src/content/__tests__/record-frame-agent.test.ts b/apps/extension/src/content/__tests__/record-frame-agent.test.ts index 0c0a651..7c55de3 100644 --- a/apps/extension/src/content/__tests__/record-frame-agent.test.ts +++ b/apps/extension/src/content/__tests__/record-frame-agent.test.ts @@ -45,10 +45,32 @@ function portHarness() { describe("RecordFrameAgent", () => { afterEach(() => { + vi.unstubAllGlobals(); document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE); document.body.replaceChildren(); }); + it("starts when crypto.randomUUID is unavailable", async () => { + const harness = portHarness(); + const getRandomValues = vi.fn((bytes: Uint8Array) => { + bytes.fill(0x11); + return bytes; + }); + vi.stubGlobal("crypto", { getRandomValues }); + vi.stubGlobal("chrome", { + runtime: { + connect: vi.fn(() => harness.port), + sendMessage: vi.fn(), + }, + }); + const agent = new RecordFrameAgent(); + + await expect( + agent.start({ type: RECORD_FRAME_START, requestId: "rec-http", startedAtMs: 10 }), + ).resolves.toEqual({ ok: true }); + expect(getRandomValues).toHaveBeenCalled(); + }); + it("keeps a failed stop retryable and flushes the final dirty fill", async () => { const harness = portHarness(); const sendMessage = vi diff --git a/apps/extension/src/content/record-step-delivery.ts b/apps/extension/src/content/record-step-delivery.ts index 5099b32..fc55138 100644 --- a/apps/extension/src/content/record-step-delivery.ts +++ b/apps/extension/src/content/record-step-delivery.ts @@ -1,3 +1,4 @@ +import { createRandomUuid } from "@/lib/random-uuid"; import { isAcceptedRecordStepAck, RECORD_STEP, @@ -18,7 +19,7 @@ export class RecordStepDelivery { constructor( requestId: string, send: SendRecordStepMessage = (message) => chrome.runtime.sendMessage(message), - producerId: string = crypto.randomUUID(), + producerId: string = createRandomUuid(), ) { this.#requestId = requestId; this.#producerId = producerId; diff --git a/apps/extension/src/content/recording/frame-agent.ts b/apps/extension/src/content/recording/frame-agent.ts index 68e7b41..5d1b64f 100644 --- a/apps/extension/src/content/recording/frame-agent.ts +++ b/apps/extension/src/content/recording/frame-agent.ts @@ -1,3 +1,4 @@ +import { createRandomUuid } from "@/lib/random-uuid"; import { markRecordingDocument, type RecordingDocumentMarker, @@ -67,7 +68,7 @@ export class RecordFrameAgent { async #start( message: RecordFrameStartMessage, ): Promise<{ ok: true } | { ok: false; error: string }> { - const producerId = crypto.randomUUID(); + const producerId = createRandomUuid(); const root = await waitForDocumentElement(); if (this.#disposed) return { ok: false, error: "recording document is disposed" }; const marker = markRecordingDocument(producerId, root); diff --git a/apps/extension/src/entrypoints/popup/App.test.tsx b/apps/extension/src/entrypoints/popup/App.test.tsx index daec637..2d71235 100644 --- a/apps/extension/src/entrypoints/popup/App.test.tsx +++ b/apps/extension/src/entrypoints/popup/App.test.tsx @@ -122,8 +122,8 @@ describe("App", () => { expect(screen.queryByText(/^扩展 v/)).toBeNull(); expect(screen.queryByText(/^daemon v/)).toBeNull(); - expect(screen.getByTitle("扩展版本").textContent).toBe(EXTENSION_VERSION); - expect(screen.getByTitle("bsk 版本").textContent).toBe(mockDaemonVersion); + expect(screen.getByTitle("扩展版本").textContent).toBe(`Ext ${EXTENSION_VERSION}`); + expect(screen.getByTitle("bsk 版本").textContent).toBe(`CLI ${mockDaemonVersion}`); expect(screen.getByText("03c3e47f")).toBeTruthy(); const copyButton = screen.getByRole("button", { name: "复制实例 ID" }); diff --git a/apps/extension/src/entrypoints/popup/App.tsx b/apps/extension/src/entrypoints/popup/App.tsx index 0756e9c..cc9287b 100644 --- a/apps/extension/src/entrypoints/popup/App.tsx +++ b/apps/extension/src/entrypoints/popup/App.tsx @@ -267,9 +267,9 @@ export function App() { data-slot="popup-meta" >
- {extensionVersion} + CLI {daemonVersion} / - {daemonVersion} + Ext {extensionVersion}
{t("popup.instanceTitle")} diff --git a/apps/extension/src/lib/__tests__/capture-suppress-bridge.test.ts b/apps/extension/src/lib/__tests__/capture-suppress-bridge.test.ts index 1baf524..7e5fe11 100644 --- a/apps/extension/src/lib/__tests__/capture-suppress-bridge.test.ts +++ b/apps/extension/src/lib/__tests__/capture-suppress-bridge.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it, vi } from "vitest"; import { CAPTURE_SUPPRESS, type CaptureSuppressMessage, - withOverlaysHiddenForCapture, + withExtensionOverlayHidden, } from "../capture-suppress-bridge"; function recordingSendToTab(events: string[]) { @@ -12,12 +12,12 @@ function recordingSendToTab(events: string[]) { }); } -describe("withOverlaysHiddenForCapture", () => { +describe("withExtensionOverlayHidden", () => { it("sends begin → fn → end and returns fn's result", async () => { const events: string[] = []; const sendToTab = recordingSendToTab(events); - const result = await withOverlaysHiddenForCapture( + const result = await withExtensionOverlayHidden( 7, async () => { events.push("capture"); @@ -37,7 +37,7 @@ describe("withOverlaysHiddenForCapture", () => { const sendToTab = recordingSendToTab(events); await expect( - withOverlaysHiddenForCapture( + withExtensionOverlayHidden( 7, async () => { events.push("capture"); @@ -55,7 +55,7 @@ describe("withOverlaysHiddenForCapture", () => { }); const fn = vi.fn(async () => "shot"); - const result = await withOverlaysHiddenForCapture(7, fn, sendToTab); + const result = await withExtensionOverlayHidden(7, fn, sendToTab); expect(result).toBe("shot"); expect(fn).toHaveBeenCalledTimes(1); @@ -69,7 +69,7 @@ describe("withOverlaysHiddenForCapture", () => { return { type: CAPTURE_SUPPRESS, ok: true }; }); - const result = await withOverlaysHiddenForCapture(7, async () => "shot", sendToTab); + const result = await withExtensionOverlayHidden(7, async () => "shot", sendToTab); expect(result).toBe("shot"); expect(sendToTab).toHaveBeenCalledTimes(2); diff --git a/apps/extension/src/lib/__tests__/random-uuid.test.ts b/apps/extension/src/lib/__tests__/random-uuid.test.ts new file mode 100644 index 0000000..db65ead --- /dev/null +++ b/apps/extension/src/lib/__tests__/random-uuid.test.ts @@ -0,0 +1,30 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createRandomUuid } from "../random-uuid"; + +describe("createRandomUuid", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("uses crypto.randomUUID when it is available", () => { + const randomUUID = vi.fn(() => "00000000-0000-4000-8000-000000000000"); + vi.stubGlobal("crypto", { randomUUID, getRandomValues: vi.fn() }); + + expect(createRandomUuid()).toBe("00000000-0000-4000-8000-000000000000"); + expect(randomUUID).toHaveBeenCalledOnce(); + }); + + it("falls back to getRandomValues and sets UUID v4 bits", () => { + const bytes = new Uint8Array(16).fill(0xff); + const getRandomValues = vi.fn((target: Uint8Array) => { + target.set(bytes); + return target; + }); + vi.stubGlobal("crypto", { getRandomValues }); + + const uuid = createRandomUuid(); + + expect(uuid).toBe("ffffffff-ffff-4fff-bfff-ffffffffffff"); + expect(getRandomValues).toHaveBeenCalledOnce(); + }); +}); diff --git a/apps/extension/src/lib/capture-suppress-bridge.ts b/apps/extension/src/lib/capture-suppress-bridge.ts index 03e05b3..07096f9 100644 --- a/apps/extension/src/lib/capture-suppress-bridge.ts +++ b/apps/extension/src/lib/capture-suppress-bridge.ts @@ -1,9 +1,9 @@ /** - * Wire protocol and background-side helper for hiding the extension's - * in-page overlay while a screenshot is captured, so captured frames only - * contain the page itself. + * Wire protocol and background-side helper for temporarily excluding the + * extension's in-page overlay. Screenshots use it to avoid captured chrome; + * coordinate-driven native input uses it so Chrome hit-tests the page. * - * The background wraps every capture in `withOverlaysHiddenForCapture`, + * The background wraps each bounded operation in `withExtensionOverlayHidden`, * which sends `begin` (the content script hides the overlay host and only * acks once the compositor has produced an overlay-free frame) and always * follows up with `end`, even when the capture throws. Tabs without the @@ -46,7 +46,7 @@ const defaultSendToTab: CaptureSuppressSendToTab = (tabId, message) => * (or any other `begin` failure) simply skips suppression — there is no * overlay to hide in that tab. */ -export async function withOverlaysHiddenForCapture( +export async function withExtensionOverlayHidden( tabId: number, fn: () => Promise, sendToTab: CaptureSuppressSendToTab = defaultSendToTab, diff --git a/apps/extension/src/lib/random-uuid.ts b/apps/extension/src/lib/random-uuid.ts new file mode 100644 index 0000000..ae18e2f --- /dev/null +++ b/apps/extension/src/lib/random-uuid.ts @@ -0,0 +1,26 @@ +/** + * Generate a UUID v4 in browser contexts where randomUUID() is unavailable. + * + * Crypto.randomUUID() is restricted to secure contexts, while + * crypto.getRandomValues() is available to content scripts on ordinary HTTP + * origins. BrowserSkill records many intranet pages served over HTTP, so the + * fallback is required for recording to work on those pages. + */ +export function createRandomUuid(): string { + const cryptoApi = globalThis.crypto; + if (typeof cryptoApi.randomUUID === "function") return cryptoApi.randomUUID(); + + const bytes = new Uint8Array(16); + cryptoApi.getRandomValues(bytes); + bytes[6] = (bytes[6]! & 0x0f) | 0x40; + bytes[8] = (bytes[8]! & 0x3f) | 0x80; + + const hex = Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(""); + return [ + hex.slice(0, 8), + hex.slice(8, 12), + hex.slice(12, 16), + hex.slice(16, 20), + hex.slice(20), + ].join("-"); +} diff --git a/apps/extension/src/lib/recording/frame-coordinator.ts b/apps/extension/src/lib/recording/frame-coordinator.ts index 52ca0ad..00dd17c 100644 --- a/apps/extension/src/lib/recording/frame-coordinator.ts +++ b/apps/extension/src/lib/recording/frame-coordinator.ts @@ -1,3 +1,4 @@ +import { createRandomUuid } from "../random-uuid"; import { isRecordFrameQueryMessage, RECORD_FRAME_PORT, @@ -188,7 +189,7 @@ export class RecordFrameCoordinator { const recording = this.#recordings.get(requestId); if (!recording) return true; recording.finishing = true; - const commandId = crypto.randomUUID(); + const commandId = createRandomUuid(); const results = await Promise.all( [...recording.agents.values()].map((agent) => this.#stopAgent(agent, requestId, commandId)), ); diff --git a/apps/extension/src/tools/__tests__/borrow-confirmation.test.ts b/apps/extension/src/tools/__tests__/borrow-confirmation.test.ts index 0cb106b..784fa80 100644 --- a/apps/extension/src/tools/__tests__/borrow-confirmation.test.ts +++ b/apps/extension/src/tools/__tests__/borrow-confirmation.test.ts @@ -127,6 +127,38 @@ describe("requestBorrowConfirmation", () => { expect(message.requestId.length).toBeGreaterThan(0); }); + it("proactively focuses the user window before sending the overlay message", async () => { + tabs.get.mockResolvedValueOnce({ id: 42, title: "Tab" }); + windows.getLastFocused.mockResolvedValueOnce( + userWindowWithActiveTab({ windowId: 77, tabId: 7, url: "https://app.example/" }), + ); + tabs.sendMessage.mockResolvedValueOnce({ type: "borrow-response", allowed: true }); + + const pending = requestBorrowConfirmation(42, { + deps: { tabs, windows, notifications }, + }); + await vi.runAllTimersAsync(); + await pending; + + expect(windows.update).toHaveBeenCalledWith(77, { focused: true }); + }); + + it("still proceeds when proactive window focus fails", async () => { + tabs.get.mockResolvedValueOnce({ id: 42, title: "Tab" }); + windows.getLastFocused.mockResolvedValueOnce( + userWindowWithActiveTab({ windowId: 77, tabId: 7, url: "https://app.example/" }), + ); + windows.update.mockRejectedValueOnce(new Error("window gone")); + tabs.sendMessage.mockResolvedValueOnce({ type: "borrow-response", allowed: true }); + + const pending = requestBorrowConfirmation(42, { + deps: { tabs, windows, notifications }, + }); + await vi.runAllTimersAsync(); + + expect(await pending).toBe(true); + }); + it("sends confirmation to the borrow target when it is the active tab", async () => { tabs.get.mockResolvedValueOnce({ id: 42, title: "Active Tab" }); windows.getLastFocused.mockResolvedValueOnce( diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index 8509187..53a529a 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -273,6 +273,107 @@ describe("ToolDispatcher", () => { }); }); + it("bypasses and restores the control overlay for an upload trigger click", async () => { + const sendMessage = vi.fn(async () => undefined); + vi.stubGlobal("chrome", { + tabs: { + get: vi.fn(async () => ({ id: 7, windowId: 4242, active: true })), + query: vi.fn(async () => [{ id: 7, windowId: 4242, active: true }]), + sendMessage, + }, + }); + const { transport, sent, deliver } = fakeTransport(); + const sessions = new SessionManager({ + agentWindow: { + create: vi.fn(async () => 4242), + remove: vi.fn(async () => {}), + ensureActiveTab: vi.fn(async () => {}), + }, + }); + const ctx = await sessions.start("aa11"); + ctx.refStore.set("e1", 123, { tabId: 7 }); + const send = vi.fn(async (_tabId: number, method: string, params?: object) => { + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } } as T; + } + if (method === "DOM.getContentQuads") { + return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] } as T; + } + if (method === "DOM.resolveNode") { + return { object: { objectId: "trigger-object" } } as T; + } + if (method === "DOM.describeNode") return { node: { backendNodeId: 456 } } as T; + if (method === "Runtime.callFunctionOn") { + const declaration = (params as { functionDeclaration?: string }).functionDeclaration ?? ""; + if (declaration.includes("count: state.inputs.length")) { + return { result: { value: { count: 1, multiple: false } } } as T; + } + if (declaration.includes("inputs[0]")) { + return { result: { objectId: "input-object" } } as T; + } + return { result: { value: true } } as T; + } + if (method === "Runtime.evaluate") { + const expression = (params as { expression?: string }).expression ?? ""; + if (expression.includes("overlayDetails")) { + return { result: { value: { hitIndex: 0 } } } as T; + } + if (expression.includes("overlayHostPresent")) { + return { + result: { + value: { overlayHostPresent: true, overlayHostConnected: true }, + }, + } as T; + } + if (expression.includes("count:")) { + return { result: { value: { count: 1, multiple: false } } } as T; + } + if (expression.includes("?.inputs[0]")) { + return { result: { objectId: "input-object" } } as T; + } + return { result: { value: true } } as T; + } + return {} as T; + }); + const cdp = { + send, + detachSession: vi.fn(async () => {}), + ensureNetworkCapture: vi.fn(async () => {}), + networkEntriesSince: vi.fn(() => ({ + tab_id: 7, + entries: [], + next_since: 0, + truncated: false, + })), + setDeviceMetricsOverride: vi.fn(async () => {}), + clearDeviceMetricsOverride: vi.fn(async () => {}), + setUserAgentOverride: vi.fn(async () => {}), + setTouchEmulationEnabled: vi.fn(async () => {}), + }; + const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp }); + dispatcher.start(); + + deliver( + makeRequest("tool.upload", { + session_id: "aa11", + ref: "@e1", + files: [{ transfer_id: "tr_1", name: "test.png", staged_path: "/stage/test.png" }], + }), + ); + await flushMicrotasks(); + await vi.waitFor(() => expect(sent).toHaveLength(1)); + + expect(sent[0]).toMatchObject({ result: { tab_id: 7, file_names: ["test.png"] } }); + expect(sendMessage).toHaveBeenNthCalledWith(1, 7, { + type: "bh-automation-bypass", + enabled: true, + }); + expect(sendMessage).toHaveBeenNthCalledWith(2, 7, { + type: "bh-automation-bypass", + enabled: false, + }); + }); + it("detaches CDP state before stopping a session", async () => { const { transport, sent, deliver } = fakeTransport(); const sessions = new SessionManager({ diff --git a/apps/extension/src/tools/__tests__/file-transfer.test.ts b/apps/extension/src/tools/__tests__/file-transfer.test.ts new file mode 100644 index 0000000..3ef1986 --- /dev/null +++ b/apps/extension/src/tools/__tests__/file-transfer.test.ts @@ -0,0 +1,988 @@ +import { describe, expect, it, vi } from "vitest"; +import { SessionManager } from "@/session-manager/manager"; +import { type DownloadsApi, handleDownload } from "../download"; +import { captureBrowserDownload } from "../download-capture"; +import { uploadThroughFileDrop } from "../file-drop-transaction"; +import { uploadThroughActivatedFileInput } from "../file-input-transaction"; +import type { ResolvedActionTarget } from "../interaction"; +import type { CdpRunner } from "../shared"; +import { handleUpload } from "../upload"; + +function sessions() { + return new SessionManager({ + agentWindow: { + create: vi.fn(async () => 100), + remove: vi.fn(async () => {}), + ensureActiveTab: vi.fn(async () => {}), + }, + }); +} + +function tabsApi() { + return { + get: vi.fn( + async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, + ), + query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), + }; +} + +function fakeEvent unknown>() { + const listeners = new Set(); + return { + addListener: (listener: T) => listeners.add(listener), + removeListener: (listener: T) => listeners.delete(listener), + emit: (...args: Parameters) => { + for (const listener of listeners) listener(...args); + }, + }; +} + +function actionTarget(frameId?: string, sessionId?: string): ResolvedActionTarget { + return { + tab: { tabId: 4, windowId: 100, active: true }, + backendNodeId: 123, + cdpTarget: { tabId: 4, ...(sessionId ? { sessionId } : {}) }, + ...(frameId ? { frameId } : {}), + usedRef: "e3", + }; +} + +function withoutOverlay(operation: () => Promise): Promise { + return operation(); +} + +function uploadCdp( + options: { + inputCount?: number; + multiple?: boolean; + chooser?: { frameId?: string; backendNodeId?: number; mode?: string }; + pendingResolve?: boolean; + } = {}, +) { + const calls: Array<{ method: string; params?: object }> = []; + let cdpEvent: Parameters>[0] | undefined; + const send = vi.fn(async (_tabId: number, method: string, params?: object) => { + calls.push({ method, params }); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "Page.setInterceptFileChooserDialog") return {}; + if (method === "Page.getLayoutMetrics") + return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; + if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] }; + if (method === "DOM.resolveNode") { + if (options.pendingResolve) return new Promise(() => {}); + return { object: { objectId: "trigger-object" } }; + } + if (method === "Runtime.callFunctionOn") { + const declaration = (params as { functionDeclaration?: string }).functionDeclaration ?? ""; + if (declaration.includes("Object.defineProperty")) return { result: { value: true } }; + if (declaration.includes("count: state.inputs.length")) { + return { + result: { + value: { + count: options.inputCount ?? 1, + multiple: options.multiple ?? true, + }, + }, + }; + } + if (declaration.includes("inputs[0]")) { + return { result: { objectId: "input-object" } }; + } + return { result: { value: true } }; + } + if (method === "DOM.describeNode") return { node: { backendNodeId: 456 } }; + if ( + method === "Input.dispatchMouseEvent" && + (params as { type?: string }).type === "mousePressed" && + options.chooser + ) { + cdpEvent?.({ tabId: 4 }, "Page.fileChooserOpened", options.chooser); + } + return {}; + }); + const cdp: CdpRunner = { + send: send as unknown as CdpRunner["send"], + onEvent: (handler) => { + cdpEvent = handler; + return { dispose: vi.fn() }; + }, + }; + return { + calls, + emitChooser: (event: { frameId?: string; backendNodeId?: number; mode?: string }) => + cdpEvent?.({ tabId: 4 }, "Page.fileChooserOpened", event), + cdp, + }; +} + +function dropCdp( + options: { + targetContainsPoint?: boolean; + dispatchError?: (type: string) => Error | Promise | undefined; + onCall?: (method: string) => void; + } = {}, +) { + const calls: Array<{ method: string; params?: object }> = []; + const detach = vi.fn(async () => {}); + const send = vi.fn(async (_tabId: number, method: string, params?: object) => { + calls.push({ method, params }); + options.onCall?.(method); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; + } + if (method === "DOM.getContentQuads") { + return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] }; + } + if (method === "DOM.getNodeForLocation") return { backendNodeId: 124 }; + if (method === "DOM.resolveNode") { + const id = (params as { backendNodeId?: number }).backendNodeId; + return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } }; + } + if (method === "Runtime.callFunctionOn") { + return { result: { value: options.targetContainsPoint ?? true } }; + } + if (method === "Input.dispatchDragEvent") { + const type = (params as { type?: string }).type ?? ""; + const error = options.dispatchError?.(type); + if (error instanceof Promise) return error; + if (error) throw error; + return {}; + } + if (method === "Runtime.releaseObjectGroup") return {}; + throw new Error(`unexpected drop CDP command ${method}`); + }); + const cdp: CdpRunner = { + send: send as unknown as CdpRunner["send"], + detach, + }; + return { calls, cdp, detach }; +} + +describe("file transfer tools", () => { + it("captures the file input activated by the requested click and injects only staged paths", async () => { + const manager = sessions(); + const ctx = await manager.start("s1"); + ctx.refStore.set("e3", 123, { tabId: 4 }); + const { cdp, calls } = uploadCdp(); + + const result = await handleUpload( + manager, + { + session_id: "s1", + ref: "@e3", + files: [ + { transfer_id: "tr_1", name: "one.png", staged_path: "/private/stage/one" }, + { transfer_id: "tr_2", name: "two.png", staged_path: "/private/stage/two" }, + ], + }, + { cdp, tabsApi: tabsApi() }, + ); + + expect(result).toMatchObject({ tab_id: 4, file_names: ["one.png", "two.png"] }); + expect(calls[0]).toEqual({ + method: "Page.setInterceptFileChooserDialog", + params: { enabled: true }, + }); + expect(calls).toContainEqual({ + method: "Page.setInterceptFileChooserDialog", + params: { enabled: false, cancel: true }, + }); + expect(calls).toContainEqual({ + method: "DOM.setFileInputFiles", + params: { files: ["/private/stage/one", "/private/stage/two"], backendNodeId: 456 }, + }); + }); + + it("fails immediately when the trigger does not activate a file input", async () => { + const { cdp } = uploadCdp({ inputCount: 0 }); + const result = await uploadThroughActivatedFileInput({ + cdp, + actionTarget: actionTarget(), + files: ["/private/stage/one"], + timeoutMs: 100, + trigger: async () => ({ tab_id: 4, x: 10, y: 10 }), + }); + + expect(result).toMatchObject({ + code: "unsupported", + data: { reason: "file_input_not_activated", phase: "resolve_input" }, + }); + }); + + it("fails before clicking when chooser interception is unavailable", async () => { + const trigger = vi.fn(async () => ({ tab_id: 4, x: 10, y: 10 })); + const cdp: CdpRunner = { + send: vi.fn(async (_tabId: number, method: string) => { + if (method === "Page.setInterceptFileChooserDialog") { + throw new Error("method unavailable"); + } + return {}; + }) as CdpRunner["send"], + }; + + const result = await uploadThroughActivatedFileInput({ + cdp, + actionTarget: actionTarget(), + files: ["/private/stage/one"], + timeoutMs: 100, + trigger, + }); + + expect(result).toMatchObject({ + code: "cdp_failed", + data: { effect_state: "none", phase: "arm_interception" }, + }); + expect(trigger).not.toHaveBeenCalled(); + }); + + it("uses an exact chooser event as an independent input-location signal", async () => { + const { cdp, calls, emitChooser } = uploadCdp({ inputCount: 0 }); + const result = await uploadThroughActivatedFileInput({ + cdp, + actionTarget: actionTarget("f1"), + files: ["/private/stage/one"], + timeoutMs: 100, + trigger: async () => { + emitChooser({ frameId: "f1", backendNodeId: 789, mode: "selectSingle" }); + return { tab_id: 4, x: 10, y: 10 }; + }, + }); + + expect(result).toMatchObject({ multiple: false }); + expect(calls).toContainEqual({ + method: "DOM.setFileInputFiles", + params: { files: ["/private/stage/one"], backendNodeId: 789 }, + }); + }); + + it("reports a File System Access picker without waiting for a timeout", async () => { + const { cdp, emitChooser } = uploadCdp({ + inputCount: 0, + }); + const result = await uploadThroughActivatedFileInput({ + cdp, + actionTarget: actionTarget("f1"), + files: ["/private/stage/one"], + timeoutMs: 100, + trigger: async () => { + emitChooser({ frameId: "f1", mode: "selectSingle" }); + return { tab_id: 4, x: 10, y: 10 }; + }, + }); + + expect(result).toMatchObject({ + code: "unsupported", + message: "upload trigger invoked a non-input file picker", + data: { reason: "file_input_not_activated", phase: "resolve_input" }, + }); + }); + + it("bounds a stuck file-input probe", async () => { + const { cdp } = uploadCdp({ pendingResolve: true }); + + const result = await uploadThroughActivatedFileInput({ + cdp, + actionTarget: actionTarget(), + files: ["/private/stage/one"], + timeoutMs: 5, + trigger: vi.fn(), + }); + + expect(result).toMatchObject({ + code: "timeout", + data: { reason: "file_input_probe_failed", phase: "arm_input_probe" }, + }); + }); + + it("marks a timed-out file assignment unknown and detaches browser state", async () => { + const fixture = uploadCdp(); + const originalSend = fixture.cdp.send; + const detach = vi.fn(async () => {}); + fixture.cdp.detach = detach; + fixture.cdp.send = vi.fn((tabId: number, method: string, params?: object) => { + if (method === "DOM.setFileInputFiles") return new Promise(() => {}); + return originalSend(tabId, method, params); + }) as CdpRunner["send"]; + + const result = await uploadThroughActivatedFileInput({ + cdp: fixture.cdp, + actionTarget: actionTarget(), + files: ["/private/stage/one"], + timeoutMs: 10, + trigger: async () => ({ tab_id: 4, x: 10, y: 10 }), + }); + + expect(result).toMatchObject({ + code: "timeout", + data: { effect_state: "unknown", phase: "set_files" }, + }); + expect(detach).toHaveBeenCalledWith(4); + }); + + it("delivers staged files through one explicit native drop without clicking", async () => { + const manager = sessions(); + const ctx = await manager.start("s1"); + ctx.refStore.set("e3", 123, { tabId: 4 }); + const lifecycle: string[] = []; + const { cdp, calls } = dropCdp({ onCall: (method) => lifecycle.push(method) }); + const overlayPhases: string[] = []; + + const result = await handleUpload( + manager, + { + session_id: "s1", + ref: "@e3", + mode: "drop", + files: [{ transfer_id: "tr_1", name: "one.png", staged_path: "/stage/one.png" }], + }, + { + cdp, + tabsApi: tabsApi(), + sendToTab: vi.fn(async (_tabId, message) => { + overlayPhases.push(message.phase); + lifecycle.push(`overlay:${message.phase}`); + }), + }, + ); + + expect(result).toMatchObject({ tab_id: 4, used_ref: "e3", file_names: ["one.png"] }); + expect(calls.filter((call) => call.method === "Input.dispatchDragEvent")).toEqual([ + { + method: "Input.dispatchDragEvent", + params: { + type: "dragEnter", + x: 110, + y: 70, + data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 }, + }, + }, + { + method: "Input.dispatchDragEvent", + params: { + type: "dragOver", + x: 110, + y: 70, + data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 }, + }, + }, + { + method: "Input.dispatchDragEvent", + params: { + type: "drop", + x: 110, + y: 70, + data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 }, + }, + }, + ]); + expect(calls.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false); + expect(overlayPhases).toEqual(["begin", "end"]); + expect(lifecycle.indexOf("DOM.getContentQuads")).toBeLessThan( + lifecycle.indexOf("overlay:begin"), + ); + expect(lifecycle.indexOf("overlay:begin")).toBeLessThan( + lifecycle.indexOf("DOM.getNodeForLocation"), + ); + expect(lifecycle.lastIndexOf("Input.dispatchDragEvent")).toBeLessThan( + lifecycle.indexOf("overlay:end"), + ); + expect(lifecycle.indexOf("overlay:end")).toBeLessThan( + lifecycle.indexOf("Runtime.releaseObjectGroup"), + ); + }); + + it("fails closed before dragging when another element owns the action point", async () => { + const { cdp, calls } = dropCdp({ targetContainsPoint: false }); + const result = await uploadThroughFileDrop({ + cdp, + actionTarget: actionTarget(), + files: ["/stage/one.png"], + timeoutMs: 100, + withOverlayHidden: withoutOverlay, + }); + + expect(result).toMatchObject({ + code: "permission_denied", + data: { + reason: "file_drop_target_unavailable", + mechanism: "drop", + effect_state: "none", + phase: "resolve_target", + }, + }); + expect(calls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false); + }); + + it("dispatches an OOPIF file drop in the child target's local coordinates", async () => { + const rootCalls: Array<{ method: string; params?: object }> = []; + const childCalls: Array<{ method: string; params?: object }> = []; + const cdp: CdpRunner = { + send: vi.fn(async (_tabId, method, params) => { + rootCalls.push({ method, params }); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "DOM.getBoxModel") { + return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } }; + } + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } }; + } + throw new Error(`unexpected root command ${method}`); + }) as CdpRunner["send"], + sendToTarget: vi.fn(async (_target, method, params) => { + childCalls.push({ method, params }); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "DOM.getContentQuads") { + return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] }; + } + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } }; + } + if (method === "DOM.getNodeForLocation") { + return { backendNodeId: 124, frameId: "child" }; + } + if (method === "DOM.resolveNode") { + const id = (params as { backendNodeId?: number }).backendNodeId; + return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } }; + } + if (method === "Runtime.callFunctionOn") return { result: { value: true } }; + if (method === "Input.dispatchDragEvent" || method === "Runtime.releaseObjectGroup") { + return {}; + } + throw new Error(`unexpected child command ${method}`); + }) as CdpRunner["sendToTarget"], + getFrameGraph: vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 10, + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })), + }; + + const result = await uploadThroughFileDrop({ + cdp, + actionTarget: actionTarget("child", "child-session"), + files: ["/stage/one.png"], + timeoutMs: 100, + withOverlayHidden: withoutOverlay, + }); + + expect(result).toMatchObject({ x: 110, y: 70 }); + expect( + childCalls + .filter((call) => call.method === "Input.dispatchDragEvent") + .map((call) => call.params), + ).toEqual([ + expect.objectContaining({ type: "dragEnter", x: 110, y: 70 }), + expect.objectContaining({ type: "dragOver", x: 110, y: 70 }), + expect.objectContaining({ type: "drop", x: 110, y: 70 }), + ]); + expect(rootCalls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false); + }); + + it("reports a missing native drag command as unsupported before commit", async () => { + const { cdp } = dropCdp({ + dispatchError: (type) => + type === "dragEnter" ? new Error("'Input.dispatchDragEvent' wasn't found") : undefined, + }); + const result = await uploadThroughFileDrop({ + cdp, + actionTarget: actionTarget(), + files: ["/stage/one.png"], + timeoutMs: 100, + withOverlayHidden: withoutOverlay, + }); + + expect(result).toMatchObject({ + code: "unsupported", + data: { + reason: "upload_mechanism_unsupported", + effect_state: "none", + phase: "drag_enter", + }, + }); + }); + + it("cancels an unresolved drag before returning a pre-commit timeout", async () => { + const { cdp, calls } = dropCdp({ + dispatchError: (type) => (type === "dragEnter" ? new Promise(() => {}) : undefined), + }); + const result = await uploadThroughFileDrop({ + cdp, + actionTarget: actionTarget(), + files: ["/stage/one.png"], + timeoutMs: 10, + withOverlayHidden: withoutOverlay, + }); + + expect(result).toMatchObject({ + code: "timeout", + data: { reason: "file_drop_failed", effect_state: "none", phase: "drag_enter" }, + }); + expect(calls).toContainEqual({ + method: "Input.dispatchDragEvent", + params: expect.objectContaining({ type: "dragCancel" }), + }); + }); + + it("marks an unresolved drop unknown and detaches browser state", async () => { + const { cdp, detach } = dropCdp({ + dispatchError: (type) => (type === "drop" ? new Promise(() => {}) : undefined), + }); + const result = await uploadThroughFileDrop({ + cdp, + actionTarget: actionTarget(), + files: ["/stage/one.png"], + timeoutMs: 10, + withOverlayHidden: withoutOverlay, + }); + + expect(result).toMatchObject({ + code: "timeout", + data: { reason: "file_drop_failed", effect_state: "unknown", phase: "drop" }, + }); + expect(detach).toHaveBeenCalledWith(4); + }); + + it("routes one exact-target download through a browser-relative capability", async () => { + const manager = sessions(); + const ctx = await manager.start("s1"); + ctx.refStore.set("e3", 123, { tabId: 4 }); + const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>(); + const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>(); + const onDeterminingFilename = + fakeEvent< + ( + item: chrome.downloads.DownloadItem, + suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void, + ) => void | true + >(); + const initial = { + id: 9, + url: "https://example.test/result.zip", + finalUrl: "https://example.test/result.zip", + filename: "result.zip", + state: "in_progress", + fileSize: -1, + totalBytes: 12, + mime: "application/zip", + danger: "safe", + } as chrome.downloads.DownloadItem; + const completed = { + ...initial, + filename: "/profile/Downloads/BrowserSkill/tr_1/result.zip", + state: "complete", + fileSize: 12, + } as chrome.downloads.DownloadItem; + const downloads: DownloadsApi = { + onCreated, + onChanged, + onDeterminingFilename, + search: vi.fn(async () => [completed]), + cancel: vi.fn(async () => {}), + removeFile: vi.fn(async () => {}), + }; + let cdpEvent: Parameters>[0] | undefined; + let suggested: chrome.downloads.DownloadFilenameSuggestion | undefined; + const send = vi.fn(async (_tabId: number, method: string, params?: object) => { + if (method === "Page.getLayoutMetrics") + return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; + if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] }; + if ( + method === "Input.dispatchMouseEvent" && + (params as { type?: string }).type === "mousePressed" + ) { + cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", { + url: initial.url, + suggestedFilename: "result.zip", + }); + await new Promise((resolve) => { + onDeterminingFilename.emit(initial, (value) => { + suggested = value; + resolve(); + }); + }); + onCreated.emit(completed); + } + return {}; + }); + const cdp: CdpRunner = { + send: send as unknown as CdpRunner["send"], + onEvent: (handler) => { + cdpEvent = handler; + return { dispose: vi.fn() }; + }, + }; + + const result = await handleDownload( + manager, + { session_id: "s1", ref: "@e3", browser_relative_dir: "BrowserSkill/tr_1" }, + { cdp, tabsApi: tabsApi(), downloads }, + ); + + expect(suggested).toEqual({ + filename: "BrowserSkill/tr_1/result.zip", + conflictAction: "overwrite", + }); + expect(result).toMatchObject({ + tab_id: 4, + suggested_filename: "result.zip", + byte_size: 12, + browser_path: completed.filename, + }); + }); + + it("does not claim a download without an intent from the exact target", async () => { + const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>(); + const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>(); + const onDeterminingFilename = + fakeEvent< + ( + item: chrome.downloads.DownloadItem, + suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void, + ) => void | true + >(); + const downloads: DownloadsApi = { + onCreated, + onChanged, + onDeterminingFilename, + search: vi.fn(async () => []), + cancel: vi.fn(async () => {}), + removeFile: vi.fn(async () => {}), + }; + let cdpEvent: Parameters>[0] | undefined; + const cdp: CdpRunner = { + send: vi.fn(async () => ({})) as unknown as CdpRunner["send"], + onEvent: (handler) => { + cdpEvent = handler; + return { dispose: vi.fn() }; + }, + }; + const unrelated = { + id: 17, + url: "https://example.test/unrelated.zip", + finalUrl: "https://example.test/unrelated.zip", + filename: "unrelated.zip", + state: "in_progress", + } as chrome.downloads.DownloadItem; + let defaultSuggestionCalled = false; + + const result = await captureBrowserDownload({ + cdp, + target: { tabId: 4, sessionId: "expected-child" }, + downloads, + browserRelativeDir: "BrowserSkill/tr_1", + timeoutMs: 5, + trigger: async () => { + cdpEvent?.({ tabId: 4, sessionId: "other-child" }, "Page.downloadWillBegin", { + url: unrelated.url, + suggestedFilename: unrelated.filename, + }); + onDeterminingFilename.emit(unrelated, (suggestion) => { + defaultSuggestionCalled = suggestion === undefined; + }); + return { tab_id: 4, x: 10, y: 10 }; + }, + }); + + expect(defaultSuggestionCalled).toBe(true); + expect(downloads.cancel).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + code: "cdp_failed", + data: { reason: "download_capture_failed" }, + }); + }); + + it("correlates a filename candidate that arrives before the CDP intent", async () => { + const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>(); + const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>(); + const onDeterminingFilename = + fakeEvent< + ( + item: chrome.downloads.DownloadItem, + suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void, + ) => void | true + >(); + const initial = { + id: 21, + url: "https://example.test/candidate-first.bin", + finalUrl: "https://example.test/candidate-first.bin", + filename: "candidate-first.bin", + state: "in_progress", + fileSize: -1, + totalBytes: 4, + bytesReceived: 0, + } as chrome.downloads.DownloadItem; + const complete = { + ...initial, + state: "complete", + fileSize: 4, + } as chrome.downloads.DownloadItem; + const downloads: DownloadsApi = { + onCreated, + onChanged, + onDeterminingFilename, + search: vi.fn(async () => [complete]), + cancel: vi.fn(async () => {}), + removeFile: vi.fn(async () => {}), + }; + let cdpEvent: Parameters>[0] | undefined; + const cdp: CdpRunner = { + send: vi.fn(async () => ({})) as CdpRunner["send"], + onEvent: (handler) => { + cdpEvent = handler; + return { dispose: vi.fn() }; + }, + }; + let suggestion: chrome.downloads.DownloadFilenameSuggestion | undefined; + + const result = await captureBrowserDownload({ + cdp, + target: { tabId: 4 }, + downloads, + browserRelativeDir: "BrowserSkill/tr_21", + timeoutMs: 1_000, + trigger: async () => { + const suggested = new Promise((resolve) => { + onDeterminingFilename.emit(initial, (value) => { + suggestion = value; + resolve(); + }); + }); + cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", { + url: initial.url, + suggestedFilename: initial.filename, + }); + await suggested; + onCreated.emit(complete); + return { tab_id: 4, x: 10, y: 10 }; + }, + }); + + expect(suggestion).toEqual({ + filename: "BrowserSkill/tr_21/candidate-first.bin", + conflictAction: "overwrite", + }); + expect(result).toMatchObject({ item: { id: 21, state: "complete" } }); + }); + + it("removes a completed download when the final size exceeds the limit", async () => { + const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>(); + const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>(); + const onDeterminingFilename = + fakeEvent< + ( + item: chrome.downloads.DownloadItem, + suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void, + ) => void | true + >(); + const initial = { + id: 22, + url: "https://example.test/oversized.bin", + finalUrl: "https://example.test/oversized.bin", + filename: "oversized.bin", + state: "in_progress", + fileSize: -1, + totalBytes: -1, + bytesReceived: 0, + } as chrome.downloads.DownloadItem; + const complete = { + ...initial, + filename: "/profile/Downloads/BrowserSkill/tr_22/oversized.bin", + state: "complete", + fileSize: 8, + totalBytes: 8, + bytesReceived: 8, + } as chrome.downloads.DownloadItem; + const downloads: DownloadsApi = { + onCreated, + onChanged, + onDeterminingFilename, + search: vi.fn(async () => [complete]), + cancel: vi.fn(async () => {}), + removeFile: vi.fn(async () => {}), + }; + let cdpEvent: Parameters>[0] | undefined; + const cdp: CdpRunner = { + send: vi.fn(async () => ({})) as CdpRunner["send"], + onEvent: (handler) => { + cdpEvent = handler; + return { dispose: vi.fn() }; + }, + }; + + const result = await captureBrowserDownload({ + cdp, + target: { tabId: 4 }, + downloads, + browserRelativeDir: "BrowserSkill/tr_22", + maxByteSize: 4, + timeoutMs: 1_000, + trigger: async () => { + cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", { + url: initial.url, + suggestedFilename: initial.filename, + }); + await new Promise((resolve) => { + onDeterminingFilename.emit(initial, () => resolve()); + }); + onCreated.emit(initial); + onChanged.emit({ id: initial.id, state: { current: "complete" } }); + return { tab_id: 4, x: 10, y: 10 }; + }, + }); + + expect(result).toMatchObject({ + code: "cdp_failed", + data: { reason: "download_capture_failed", effect_state: "committed" }, + }); + expect(downloads.removeFile).toHaveBeenCalledWith(initial.id); + expect(downloads.cancel).not.toHaveBeenCalled(); + }); + + it("reconciles a download that completes while cancellation is being requested", async () => { + const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>(); + const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>(); + const onDeterminingFilename = + fakeEvent< + ( + item: chrome.downloads.DownloadItem, + suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void, + ) => void | true + >(); + const initial = { + id: 23, + url: "https://example.test/racing.bin", + finalUrl: "https://example.test/racing.bin", + filename: "racing.bin", + state: "in_progress", + fileSize: -1, + totalBytes: -1, + bytesReceived: 0, + } as chrome.downloads.DownloadItem; + const complete = { + ...initial, + filename: "/profile/Downloads/BrowserSkill/tr_23/racing.bin", + state: "complete", + fileSize: 4, + totalBytes: 4, + bytesReceived: 4, + } as chrome.downloads.DownloadItem; + const downloads: DownloadsApi = { + onCreated, + onChanged, + onDeterminingFilename, + search: vi.fn().mockResolvedValueOnce([initial]).mockResolvedValueOnce([complete]), + cancel: vi.fn(async () => { + throw new Error("download already complete"); + }), + removeFile: vi.fn(async () => {}), + }; + let cdpEvent: Parameters>[0] | undefined; + const cdp: CdpRunner = { + send: vi.fn(async () => ({})) as CdpRunner["send"], + onEvent: (handler) => { + cdpEvent = handler; + return { dispose: vi.fn() }; + }, + }; + + const result = await captureBrowserDownload({ + cdp, + target: { tabId: 4 }, + downloads, + browserRelativeDir: "BrowserSkill/tr_23", + timeoutMs: 80, + trigger: async () => { + cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", { + url: initial.url, + suggestedFilename: initial.filename, + }); + await new Promise((resolve) => { + onDeterminingFilename.emit(initial, () => resolve()); + }); + onCreated.emit(initial); + return { tab_id: 4, x: 10, y: 10 }; + }, + }); + + expect(result).toMatchObject({ + code: "cdp_failed", + data: { reason: "download_capture_failed", effect_state: "committed" }, + }); + expect(downloads.cancel).toHaveBeenCalledWith(initial.id); + expect(downloads.removeFile).toHaveBeenCalledWith(initial.id); + expect(result).not.toMatchObject({ data: { cleanup_state: "failed" } }); + }); + + it("rejects ambiguous attribution without cancelling either unclaimed download", async () => { + const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>(); + const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>(); + const onDeterminingFilename = + fakeEvent< + ( + item: chrome.downloads.DownloadItem, + suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void, + ) => void | true + >(); + const downloads: DownloadsApi = { + onCreated, + onChanged, + onDeterminingFilename, + search: vi.fn(async () => []), + cancel: vi.fn(async () => {}), + removeFile: vi.fn(async () => {}), + }; + let cdpEvent: Parameters>[0] | undefined; + const cdp: CdpRunner = { + send: vi.fn(async () => ({})) as CdpRunner["send"], + onEvent: (handler) => { + cdpEvent = handler; + return { dispose: vi.fn() }; + }, + }; + const defaults: number[] = []; + const candidate = (id: number) => + ({ + id, + url: "https://example.test/same.bin", + finalUrl: "https://example.test/same.bin", + filename: "same.bin", + state: "in_progress", + }) as chrome.downloads.DownloadItem; + + const result = await captureBrowserDownload({ + cdp, + target: { tabId: 4 }, + downloads, + browserRelativeDir: "BrowserSkill/tr_ambiguous", + timeoutMs: 100, + trigger: async () => { + cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", { + url: "https://example.test/same.bin", + suggestedFilename: "same.bin", + }); + for (const id of [31, 32]) { + onDeterminingFilename.emit(candidate(id), (value) => { + if (value === undefined) defaults.push(id); + }); + } + return { tab_id: 4, x: 10, y: 10 }; + }, + }); + + expect(defaults.sort()).toEqual([31, 32]); + expect(downloads.cancel).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + code: "cdp_failed", + data: { effect_state: "unknown", phase: "attribution" }, + }); + }); +}); diff --git a/apps/extension/src/tools/__tests__/frame-geometry.test.ts b/apps/extension/src/tools/__tests__/frame-geometry.test.ts index c7f82b4..9116f6a 100644 --- a/apps/extension/src/tools/__tests__/frame-geometry.test.ts +++ b/apps/extension/src/tools/__tests__/frame-geometry.test.ts @@ -169,6 +169,7 @@ describe("frame geometry projection", () => { expect(geometry).toMatchObject({ topBounds: { x: 100, y: 100, width: 200, height: 100 }, actionPoint: { x: 200, y: 150 }, + targetActionPoint: { x: 100, y: 50 }, }); }); @@ -261,6 +262,7 @@ describe("frame geometry projection", () => { expect(geometry).toMatchObject({ topBounds: { x: 100, y: 100, width: 200, height: 100 }, actionPoint: { x: 200, y: 150 }, + targetActionPoint: { x: 200, y: 150 }, }); }); diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index 0763262..655e30c 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -1904,36 +1904,41 @@ describe("buildVomScene", () => { backendDOMNodeId: 20, }, ]; - const scene = buildVomScene(axNodes, { - viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), - excludedBackendNodeIds: new Set(), - surfaceProbes: [ - { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] }, - ], - nodes: [ - { - backendNodeId: 10, - parentBackendNodeId: null, - tag: "body", - attrs: {}, - rect: { x: 0, y: 0, w: 1000, h: 800 }, - paintOrder: 0, - position: "static", - pointerEvents: "auto", - }, - { - backendNodeId: 20, - parentBackendNodeId: 10, - tag: "button", - attrs: {}, - rect: { x: 20, y: 20, w: 120, h: 40 }, - paintOrder: 1, - position: "static", - pointerEvents: "auto", - }, - ], - }); + const scene = buildVomScene( + axNodes, + { + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + nodes: [ + { + backendNodeId: 10, + parentBackendNodeId: null, + tag: "body", + attrs: {}, + rect: { x: 0, y: 0, w: 1000, h: 800 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 20, + parentBackendNodeId: 10, + tag: "button", + attrs: {}, + rect: { x: 20, y: 20, w: 120, h: 40 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + ], + }, + { + surfaceProbes: [ + { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] }, + ], + }, + ); expect(scene.surfaces).toEqual([ { triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] }, @@ -1957,46 +1962,51 @@ describe("buildVomScene", () => { backendDOMNodeId: 21, }, ]; - const scene = buildVomScene(axNodes, { - viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), - excludedBackendNodeIds: new Set(), - surfaceProbes: [ - { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] }, - ], - nodes: [ - { - backendNodeId: 10, - parentBackendNodeId: null, - tag: "body", - attrs: {}, - rect: { x: 0, y: 0, w: 1000, h: 800 }, - paintOrder: 0, - position: "static", - pointerEvents: "auto", - }, - { - backendNodeId: 20, - parentBackendNodeId: 10, - tag: "div", - attrs: { class: "tg-avatar" }, - rect: { x: 900, y: 10, w: 30, h: 30 }, - paintOrder: 1, - position: "static", - pointerEvents: "auto", - }, - { - backendNodeId: 21, - parentBackendNodeId: 20, - tag: "div", - attrs: { class: "tg-avatar__inner" }, - rect: { x: 902, y: 12, w: 26, h: 26 }, - paintOrder: 2, - position: "static", - pointerEvents: "auto", - }, - ], - }); + const scene = buildVomScene( + axNodes, + { + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + nodes: [ + { + backendNodeId: 10, + parentBackendNodeId: null, + tag: "body", + attrs: {}, + rect: { x: 0, y: 0, w: 1000, h: 800 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 20, + parentBackendNodeId: 10, + tag: "div", + attrs: { class: "tg-avatar" }, + rect: { x: 900, y: 10, w: 30, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 21, + parentBackendNodeId: 20, + tag: "div", + attrs: { class: "tg-avatar__inner" }, + rect: { x: 902, y: 12, w: 26, h: 26 }, + paintOrder: 2, + position: "static", + pointerEvents: "auto", + }, + ], + }, + { + surfaceProbes: [ + { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] }, + ], + }, + ); expect(renderVom(scene).text).toContain('@e1 button "image" [hover first: My profile]'); }); @@ -2017,48 +2027,53 @@ describe("buildVomScene", () => { backendDOMNodeId: 21, }, ]; - const scene = buildVomScene(axNodes, { - viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), - excludedBackendNodeIds: new Set(), - surfaceProbes: [ - { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] }, - { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] }, - { triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] }, - ], - nodes: [ - { - backendNodeId: 10, - parentBackendNodeId: null, - tag: "body", - attrs: {}, - rect: { x: 0, y: 0, w: 1000, h: 800 }, - paintOrder: 0, - position: "static", - pointerEvents: "auto", - }, - { - backendNodeId: 20, - parentBackendNodeId: 10, - tag: "div", - attrs: { class: "tg-avatar" }, - rect: { x: 900, y: 10, w: 30, h: 30 }, - paintOrder: 1, - position: "static", - pointerEvents: "auto", - }, - { - backendNodeId: 21, - parentBackendNodeId: 20, - tag: "div", - attrs: { class: "tg-avatar__inner" }, - rect: { x: 902, y: 12, w: 26, h: 26 }, - paintOrder: 2, - position: "static", - pointerEvents: "auto", - }, - ], - }); + const scene = buildVomScene( + axNodes, + { + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + nodes: [ + { + backendNodeId: 10, + parentBackendNodeId: null, + tag: "body", + attrs: {}, + rect: { x: 0, y: 0, w: 1000, h: 800 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 20, + parentBackendNodeId: 10, + tag: "div", + attrs: { class: "tg-avatar" }, + rect: { x: 900, y: 10, w: 30, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 21, + parentBackendNodeId: 20, + tag: "div", + attrs: { class: "tg-avatar__inner" }, + rect: { x: 902, y: 12, w: 26, h: 26 }, + paintOrder: 2, + position: "static", + pointerEvents: "auto", + }, + ], + }, + { + surfaceProbes: [ + { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] }, + { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] }, + { triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] }, + ], + }, + ); expect(scene.surfaces).toEqual([ { triggerId: 21, triggerAction: "hover", subItems: ["My profile"] }, @@ -2081,54 +2096,59 @@ describe("buildVomScene", () => { backendDOMNodeId: 21, }, ]; - const scene = buildVomScene(axNodes, { - viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), - excludedBackendNodeIds: new Set(), - surfaceProbes: [ - { - triggerBackendNodeId: 20, - triggerPoint: { x: 915, y: 25 }, - triggerAction: "hover", - subItems: ["My profile", "Sign out"], - }, - ], - nodes: [ - { - backendNodeId: 10, - parentBackendNodeId: null, - tag: "body", - attrs: {}, - rect: { x: 0, y: 0, w: 1000, h: 800 }, - paintOrder: 0, - position: "static", - pointerEvents: "auto", - cursor: "auto", - }, - { - backendNodeId: 20, - parentBackendNodeId: 10, - tag: "div", - attrs: { class: "tg-avatar" }, - rect: { x: 900, y: 10, w: 30, h: 30 }, - paintOrder: 1, - position: "static", - pointerEvents: "auto", - cursor: "pointer", - }, - { - backendNodeId: 21, - parentBackendNodeId: 20, - tag: "div", - attrs: { class: "tg-avatar__inner" }, - rect: { x: 902, y: 12, w: 26, h: 26 }, - paintOrder: 2, - position: "static", - pointerEvents: "auto", - cursor: "pointer", - }, - ], - }); + const scene = buildVomScene( + axNodes, + { + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + nodes: [ + { + backendNodeId: 10, + parentBackendNodeId: null, + tag: "body", + attrs: {}, + rect: { x: 0, y: 0, w: 1000, h: 800 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + cursor: "auto", + }, + { + backendNodeId: 20, + parentBackendNodeId: 10, + tag: "div", + attrs: { class: "tg-avatar" }, + rect: { x: 900, y: 10, w: 30, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + cursor: "pointer", + }, + { + backendNodeId: 21, + parentBackendNodeId: 20, + tag: "div", + attrs: { class: "tg-avatar__inner" }, + rect: { x: 902, y: 12, w: 26, h: 26 }, + paintOrder: 2, + position: "static", + pointerEvents: "auto", + cursor: "pointer", + }, + ], + }, + { + surfaceProbes: [ + { + triggerBackendNodeId: 20, + triggerPoint: { x: 915, y: 25 }, + triggerAction: "hover", + subItems: ["My profile", "Sign out"], + }, + ], + }, + ); expect(scene.surfaces).toEqual([ { triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] }, @@ -2154,41 +2174,46 @@ describe("buildVomScene", () => { backendDOMNodeId: 42, }, ]; - const scene = buildVomScene(axNodes, { - viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), - excludedBackendNodeIds: new Set(), - surfaceProbes: [ - { - triggerBackendNodeId: 999, - triggerPoint: { x: 900, y: 20 }, - triggerAction: "hover", - subItems: ["My profile"], - }, - ], - nodes: [ - { - backendNodeId: 10, - parentBackendNodeId: null, - tag: "body", - attrs: {}, - rect: { x: 0, y: 0, w: 1000, h: 800 }, - paintOrder: 0, - position: "static", - pointerEvents: "auto", - }, - { - backendNodeId: 42, - parentBackendNodeId: 10, - tag: "a", - attrs: {}, - rect: { x: 860, y: 120, w: 40, h: 30 }, - paintOrder: 1, - position: "static", - pointerEvents: "auto", - }, - ], - }); + const scene = buildVomScene( + axNodes, + { + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + nodes: [ + { + backendNodeId: 10, + parentBackendNodeId: null, + tag: "body", + attrs: {}, + rect: { x: 0, y: 0, w: 1000, h: 800 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 42, + parentBackendNodeId: 10, + tag: "a", + attrs: {}, + rect: { x: 860, y: 120, w: 40, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + ], + }, + { + surfaceProbes: [ + { + triggerBackendNodeId: 999, + triggerPoint: { x: 900, y: 20 }, + triggerAction: "hover", + subItems: ["My profile"], + }, + ], + }, + ); expect(scene.surfaces).toBeUndefined(); expect(renderVom(scene).text).not.toContain("[hover first:"); @@ -2606,7 +2631,106 @@ describe("handleSnapshot", () => { ); }); - it("enables conditional surface probing for semantic observe", async () => { + it("hovers the page only after the accessibility tree has been captured", async () => { + // Hovering can open menus and reflow the page. Probing between the DOM and + // AX captures would leave the two halves of one observation describing the + // page on either side of that change. + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + await sm.start("aa11"); + const root: CdpAxNode = { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + name: { type: "computedString", value: "Example" }, + backendDOMNodeId: 100, + childIds: ["2"], + }; + const button: CdpAxNode = { + nodeId: "2", + parentId: "1", + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Products" }, + backendDOMNodeId: 200, + }; + const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"]; + const i = (s: string) => strings.indexOf(s); + const methodOrder: string[] = []; + const send = vi.fn(async (_tabId: number, method: string, params?: object) => { + methodOrder.push(method); + if (method === "Accessibility.enable") return {}; + if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] }; + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } }; + } + if (method === "DOMSnapshot.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") { + return { + strings, + documents: [ + { + nodes: { + parentIndex: [-1, 0], + nodeName: [i("body"), i("button")], + backendNodeId: [100, 200], + attributes: [[], []], + }, + layout: { + nodeIndex: [0, 1], + styles: [ + [i("static"), i("auto"), i("auto")], + [i("static"), i("auto"), i("auto")], + ], + bounds: [ + [0, 0, 1000, 800], + [20, 20, 120, 40], + ], + paintOrders: [0, 1], + }, + }, + ], + }; + } + if (method === "Input.dispatchMouseEvent") return {}; + if (method === "Runtime.evaluate") { + const expression = (params as { expression?: string } | undefined)?.expression ?? ""; + // Report the button as a `:hover` rule target so a real hover fires. + if (expression.includes("document.styleSheets")) { + return { result: { value: [{ x: 80, y: 40 }] } }; + } + return { result: { value: [] } }; + } + throw new Error(`unexpected CDP method ${method}`); + }); + + const res = await handleObserve( + sm, + { session_id: "aa11", probe_hover: true }, + { + cdp: { + send: send as unknown as ( + tabId: number, + method: string, + params?: object, + ) => Promise, + trackSessionTab: vi.fn(), + }, + tabsApi: { + get: vi.fn( + async (tabId: number) => + ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, + ), + query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), + }, + }, + ); + + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + const firstHover = methodOrder.indexOf("Input.dispatchMouseEvent"); + expect(firstHover).toBeGreaterThan(-1); + expect(methodOrder.lastIndexOf("DOMSnapshot.captureSnapshot")).toBeLessThan(firstHover); + expect(methodOrder.lastIndexOf("Accessibility.getFullAXTree")).toBeLessThan(firstHover); + }); + + it("runs conditional surface probing only when observe opts in", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); await sm.start("aa11"); const root: CdpAxNode = { @@ -2662,30 +2786,46 @@ describe("handleSnapshot", () => { if (method === "Runtime.evaluate") return { result: { value: [] } }; throw new Error(`unexpected CDP method ${method}`); }); - const res = await handleObserve( + const deps = { + cdp: { + send: send as unknown as ( + tabId: number, + method: string, + params?: object, + ) => Promise, + trackSessionTab: vi.fn(), + }, + tabsApi: { + get: vi.fn( + async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, + ), + query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), + }, + }; + + const withoutProbe = await handleObserve( sm, { session_id: "aa11", debug_surfaces: true }, - { - cdp: { - send: send as unknown as ( - tabId: number, - method: string, - params?: object, - ) => Promise, - trackSessionTab: vi.fn(), - }, - tabsApi: { - get: vi.fn( - async (tabId: number) => - ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, - ), - query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), - }, - }, + deps, + ); + if ("code" in withoutProbe) { + throw new Error(`unexpected error: ${JSON.stringify(withoutProbe)}`); + } + expect(withoutProbe.hover_probe).toBeUndefined(); + expect(send).not.toHaveBeenCalledWith( + 4, + "Runtime.evaluate", + expect.objectContaining({ returnByValue: true }), ); - if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); - expect(res.debug).toEqual({ surface_probes: [] }); + const withProbe = await handleObserve( + sm, + { session_id: "aa11", debug_surfaces: true, probe_hover: true }, + deps, + ); + if ("code" in withProbe) throw new Error(`unexpected error: ${JSON.stringify(withProbe)}`); + expect(withProbe.debug).toEqual({ surface_probes: [] }); + expect(withProbe.hover_probe).toEqual({ performed: true, revealed_content: false }); expect(send).toHaveBeenCalledWith( 4, "Runtime.evaluate", diff --git a/apps/extension/src/tools/borrow-confirmation.ts b/apps/extension/src/tools/borrow-confirmation.ts index 68a889f..b575084 100644 --- a/apps/extension/src/tools/borrow-confirmation.ts +++ b/apps/extension/src/tools/borrow-confirmation.ts @@ -24,7 +24,7 @@ export interface BorrowResponseMessage { allowed: boolean; } -export const CONFIRMATION_TIMEOUT_MS = 5000; +export const CONFIRMATION_TIMEOUT_MS = 60_000; const EXIT_ANIMATION_MS = 150; /** Matches BorrowConfirmationOverlay progress ring/bar transition (duration-1000). */ export const PROGRESS_TRANSITION_MS = 1000; @@ -480,6 +480,13 @@ export async function requestBorrowConfirmation( settle(false); }, BACKGROUND_TIMEOUT_MS); + // Bring the user window to the front so the overlay is visible even + // when the Agent Window has stolen focus. This is fire-and-forget: if + // it fails the overlay / OS notification still give the user a path. + void windowsApi.update(notificationAnchor.windowId, { focused: true }).catch((err) => { + console.debug("[bsk borrow] proactive focus of user window failed", err); + }); + // Surface the OS notification *before* messaging any candidate so the // user has a parallel signal even if every content script is missing. // The Allow / Deny buttons let the user authorize explicitly when the diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index 2752791..8202186 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -4,6 +4,7 @@ import type { Transport } from "@/transport/transport"; import type { ClickParams, ConsoleParams, + DownloadParams, EmulateParams, EvaluateParams, FillParams, @@ -28,10 +29,12 @@ import type { ScreenshotParams, SelectParams, SnapshotParams, + UploadParams, WaitForNavigationParams, } from "@/transport/types"; import { isRequestFrame } from "@/transport/types"; import { handleConsole } from "./console"; +import { handleDownload } from "./download"; import { type EmulateCdpRunner, handleEmulate } from "./emulate"; import { handleEvaluate } from "./evaluate"; import { handleRequestHelp } from "./human-loop"; @@ -80,6 +83,7 @@ import { type TabReturnParams, type TabSelectParams, } from "./tabs"; +import { handleUpload } from "./upload"; import { handleWaitForNavigation } from "./waits"; import { handleWindowResize, type WindowResizeParams } from "./window"; @@ -405,7 +409,11 @@ export class ToolDispatcher { ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi, - conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope), + // Active hover probing is opt-in. A held hover latch still + // suppresses it, because probing would move the cursor off + // the element the caller is deliberately holding. + conditionalSurfaceProbe: + params.probe_hover === true && !this.hasHoverLatchForScope(hoverScope), hoverProbeBypassOverlay: bypassOverlay, } : undefined, @@ -535,6 +543,40 @@ export class ToolDispatcher { ), signal, ); + case "tool.upload": + return this.withHoverReleaseForRequest( + req.params as UploadParams, + () => + this.cdp + ? handleUpload(this.sessions, req.params as UploadParams, { + cdp: this.cdp, + tabsApi: chromeTabsApi, + signal, + bypassOverlay, + }) + : Promise.resolve({ + code: "unsupported", + message: "upload requires CDP", + } satisfies RpcError), + signal, + ); + case "tool.download": + return this.withHoverReleaseForRequest( + req.params as DownloadParams, + () => + this.cdp + ? handleDownload(this.sessions, req.params as DownloadParams, { + cdp: this.cdp, + tabsApi: chromeTabsApi, + signal, + bypassOverlay, + }) + : Promise.resolve({ + code: "unsupported", + message: "download requires CDP", + } satisfies RpcError), + signal, + ); case "tool.evaluate": return handleEvaluate( this.sessions, @@ -741,6 +783,8 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { case "tool.fill": case "tool.press": case "tool.select": + case "tool.upload": + case "tool.download": case "tool.evaluate": case "tool.observe": case "tool.request_help": diff --git a/apps/extension/src/tools/download-capture.ts b/apps/extension/src/tools/download-capture.ts new file mode 100644 index 0000000..6e75bff --- /dev/null +++ b/apps/extension/src/tools/download-capture.ts @@ -0,0 +1,364 @@ +// Order-independent coordinator for one browser download. CDP supplies the +// exact target/frame intent while chrome.downloads supplies the download id +// and filename routing hook; neither event is assumed to arrive first. + +import type { CdpTarget } from "@/browser-driver/frame-graph"; +import type { ClickResult, RpcError, TransferEffectState } from "@/transport/types"; +import { transferError } from "./errors"; +import { type CdpRunner, isRpcError } from "./shared"; + +const CORRELATION_GRACE_MS = 750; +const UNIQUE_SETTLE_MS = 50; +const SIZE_POLL_MS = 250; + +type DeterminingFilenameListener = ( + item: chrome.downloads.DownloadItem, + suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void, +) => void | true; + +interface ListenerEvent { + addListener(listener: T): void; + removeListener(listener: T): void; +} + +export interface DownloadsApi { + onCreated: ListenerEvent<(item: chrome.downloads.DownloadItem) => void>; + onChanged: ListenerEvent<(delta: chrome.downloads.DownloadDelta) => void>; + onDeterminingFilename: ListenerEvent; + search(query: chrome.downloads.DownloadQuery): Promise; + cancel(downloadId: number): Promise; + removeFile(downloadId: number): Promise; +} + +export const chromeDownloadsApi: DownloadsApi = { + get onCreated() { + return chrome.downloads.onCreated; + }, + get onChanged() { + return chrome.downloads.onChanged; + }, + get onDeterminingFilename() { + return chrome.downloads.onDeterminingFilename; + }, + search: (query) => chrome.downloads.search(query), + cancel: (id) => chrome.downloads.cancel(id), + removeFile: (id) => chrome.downloads.removeFile(id), +}; + +export interface DownloadCaptureOptions { + cdp: CdpRunner; + target: CdpTarget; + expectedFrameId?: string; + downloads: DownloadsApi; + browserRelativeDir: string; + maxByteSize?: number; + timeoutMs: number; + signal?: AbortSignal; + trigger(): Promise; +} + +export interface DownloadCaptureResult { + click: ClickResult; + item: chrome.downloads.DownloadItem; +} + +interface DownloadIntent { + url: string; + suggestedFilename: string; + frameId?: string; +} + +interface DownloadCandidate { + item: chrome.downloads.DownloadItem; + suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void; + suggested: boolean; + graceTimer: ReturnType; +} + +function safeBasename(filename: string): string { + const basename = filename.split(/[\\/]/).pop()?.trim(); + return basename && basename !== "." && basename !== ".." ? basename : "download"; +} + +function sameTarget(source: { tabId?: number; sessionId?: string }, target: CdpTarget): boolean { + return source.tabId === target.tabId && source.sessionId === target.sessionId; +} + +function matchesIntent(item: chrome.downloads.DownloadItem, intent: DownloadIntent): boolean { + const urlMatches = item.url === intent.url || item.finalUrl === intent.url; + return urlMatches && safeBasename(item.filename) === safeBasename(intent.suggestedFilename); +} + +function knownSize(item: chrome.downloads.DownloadItem): number | undefined { + if (item.fileSize >= 0) return item.fileSize; + if (item.totalBytes >= 0) return item.totalBytes; + return undefined; +} + +function captureError( + message: string, + effectState: TransferEffectState, + phase: string, + cleanupFailed = false, +): RpcError { + return transferError("cdp_failed", "download_capture_failed", message, { + effectState, + phase, + ...(cleanupFailed ? { cleanupState: "failed" } : {}), + }); +} + +async function cleanupClaimedDownload(downloads: DownloadsApi, downloadId: number): Promise { + const lookup = async () => (await downloads.search({ id: downloadId }))[0]; + const item = await lookup(); + if (!item || item.state === "interrupted") return; + if (item.state === "complete") { + await downloads.removeFile(downloadId); + return; + } + + try { + await downloads.cancel(downloadId); + } catch (cancelError) { + // Completion can win the race after the lookup but before cancellation. + // Reconcile against Chrome's authoritative state before declaring cleanup + // failed so every terminal state has one explicit cleanup path. + const reconciled = await lookup(); + if (!reconciled || reconciled.state === "interrupted") return; + if (reconciled.state === "complete") { + await downloads.removeFile(downloadId); + return; + } + throw cancelError; + } +} + +export async function captureBrowserDownload( + options: DownloadCaptureOptions, +): Promise { + let click: ClickResult | undefined; + let intent: DownloadIntent | undefined; + let capturedId: number | undefined; + let settled = false; + let succeeded = false; + let failureResult: RpcError | undefined; + let uniquenessTimer: ReturnType | undefined; + let operationTimer: ReturnType | undefined; + let sizePoll: ReturnType | undefined; + const candidates = new Map(); + const createdItems = new Map(); + + let resolveCompletion!: (item: chrome.downloads.DownloadItem) => void; + let rejectCompletion!: (error: Error) => void; + const completion = new Promise((resolve, reject) => { + resolveCompletion = resolve; + rejectCompletion = reject; + }); + const fail = (error: Error) => { + if (settled) return; + settled = true; + rejectCompletion(error); + }; + const complete = (item: chrome.downloads.DownloadItem) => { + if (settled) return; + const size = knownSize(item); + if (size !== undefined && options.maxByteSize !== undefined && size > options.maxByteSize) { + fail(new Error(`download exceeds transfer limit ${options.maxByteSize}`)); + return; + } + settled = true; + resolveCompletion(item); + }; + const suggestDefault = (candidate: DownloadCandidate) => { + if (candidate.suggested) return; + candidate.suggested = true; + candidate.suggest(); + }; + const matchingCandidates = (): DownloadCandidate[] => { + const currentIntent = intent; + return currentIntent + ? [...candidates.values()].filter( + (candidate) => !candidate.suggested && matchesIntent(candidate.item, currentIntent), + ) + : []; + }; + + const claimUnique = () => { + uniquenessTimer = undefined; + if (settled || capturedId !== undefined || !intent) return; + const matches = matchingCandidates(); + if (matches.length !== 1) { + if (matches.length > 1) { + for (const candidate of matches) suggestDefault(candidate); + fail(new Error("download attribution is ambiguous")); + } + return; + } + const candidate = matches[0]; + candidate.suggested = true; + clearTimeout(candidate.graceTimer); + capturedId = candidate.item.id; + candidate.suggest({ + filename: `${options.browserRelativeDir}/${safeBasename(intent.suggestedFilename)}`, + conflictAction: "overwrite", + }); + const size = knownSize(candidate.item); + if (size !== undefined && options.maxByteSize !== undefined && size > options.maxByteSize) { + fail(new Error(`download exceeds transfer limit ${options.maxByteSize}`)); + return; + } + const created = createdItems.get(candidate.item.id); + if (created?.state === "interrupted") { + fail(new Error(created.error ?? "download interrupted")); + } else if (created?.state === "complete") { + complete(created); + } + }; + const reconcile = () => { + if (settled || capturedId !== undefined || !intent) return; + const matches = matchingCandidates(); + if (matches.length > 1) { + for (const candidate of matches) suggestDefault(candidate); + fail(new Error("download attribution is ambiguous")); + return; + } + if (matches.length === 1 && !uniquenessTimer) { + uniquenessTimer = setTimeout(claimUnique, UNIQUE_SETTLE_MS); + } + }; + + const determiningListener: DeterminingFilenameListener = (item, suggest) => { + const candidate: DownloadCandidate = { + item, + suggest, + suggested: false, + graceTimer: setTimeout(() => { + suggestDefault(candidate); + candidates.delete(item.id); + if (intent && matchesIntent(item, intent) && capturedId === undefined) { + fail(new Error("download correlation grace elapsed before unique attribution")); + } + }, CORRELATION_GRACE_MS), + }; + candidates.set(item.id, candidate); + reconcile(); + return true; + }; + const createdListener = (item: chrome.downloads.DownloadItem) => { + createdItems.set(item.id, item); + if (capturedId !== item.id) return; + if (item.state === "interrupted") { + fail(new Error(item.error ?? "download interrupted")); + } else if (item.state === "complete") { + complete(item); + } + }; + const changedListener = async (delta: chrome.downloads.DownloadDelta) => { + if (capturedId === undefined || delta.id !== capturedId || settled) return; + if (delta.state?.current === "interrupted" || delta.error?.current) { + fail(new Error(delta.error?.current ?? "download interrupted")); + return; + } + if (delta.state?.current === "complete") { + try { + const [item] = await options.downloads.search({ id: delta.id }); + if (item) complete(item); + else fail(new Error("completed download disappeared")); + } catch (err) { + fail(err instanceof Error ? err : new Error(String(err))); + } + } + }; + const onAbort = () => fail(new DOMException("aborted", "AbortError")); + const cdpSubscription = options.cdp.onEvent?.((source, method, raw) => { + if (method !== "Page.downloadWillBegin" || !sameTarget(source, options.target)) return; + const event = raw as { url?: unknown; suggestedFilename?: unknown; frameId?: unknown }; + if (typeof event.url !== "string" || typeof event.suggestedFilename !== "string") return; + if (options.expectedFrameId && event.frameId !== options.expectedFrameId) { + fail(new Error("download originated from a different frame")); + return; + } + if (intent) { + fail(new Error("download trigger produced more than one browser download intent")); + return; + } + intent = { + url: event.url, + suggestedFilename: event.suggestedFilename, + ...(typeof event.frameId === "string" ? { frameId: event.frameId } : {}), + }; + reconcile(); + }); + if (!cdpSubscription) { + return captureError("CDP download intent subscription unavailable", "none", "arm"); + } + + options.downloads.onDeterminingFilename.addListener(determiningListener); + options.downloads.onCreated.addListener(createdListener); + options.downloads.onChanged.addListener(changedListener); + options.signal?.addEventListener("abort", onAbort, { once: true }); + operationTimer = setTimeout( + () => fail(new Error("download did not complete before timeout")), + options.timeoutMs, + ); + sizePoll = setInterval(() => { + if (capturedId === undefined || settled || options.maxByteSize === undefined) return; + void options.downloads + .search({ id: capturedId }) + .then(([item]) => { + if (!item || settled) return; + if (item.bytesReceived > (options.maxByteSize as number)) { + fail(new Error(`download exceeds transfer limit ${options.maxByteSize}`)); + } + }) + .catch((err) => fail(err instanceof Error ? err : new Error(String(err)))); + }, SIZE_POLL_MS); + + try { + const triggered = await options.trigger(); + if (isRpcError(triggered)) { + void completion.catch(() => undefined); + const effect: TransferEffectState = + capturedId !== undefined ? "committed" : intent ? "unknown" : "none"; + failureResult = { + ...triggered, + data: { ...triggered.data, effect_state: effect, phase: "trigger" }, + }; + return failureResult; + } + click = triggered; + const item = await completion; + succeeded = true; + return { click, item }; + } catch (err) { + const effect: TransferEffectState = + capturedId !== undefined ? "committed" : click ? "unknown" : "none"; + failureResult = captureError( + err instanceof Error ? err.message : String(err), + effect, + capturedId !== undefined ? "download" : "attribution", + ); + return failureResult; + } finally { + settled = true; + if (operationTimer) clearTimeout(operationTimer); + if (uniquenessTimer) clearTimeout(uniquenessTimer); + if (sizePoll) clearInterval(sizePoll); + options.signal?.removeEventListener("abort", onAbort); + options.downloads.onDeterminingFilename.removeListener(determiningListener); + options.downloads.onCreated.removeListener(createdListener); + options.downloads.onChanged.removeListener(changedListener); + cdpSubscription.dispose(); + for (const candidate of candidates.values()) { + clearTimeout(candidate.graceTimer); + if (candidate.item.id !== capturedId) suggestDefault(candidate); + } + if (!succeeded && capturedId !== undefined) { + try { + await cleanupClaimedDownload(options.downloads, capturedId); + } catch { + if (failureResult?.data) failureResult.data.cleanup_state = "failed"; + } + } + } +} diff --git a/apps/extension/src/tools/download.ts b/apps/extension/src/tools/download.ts new file mode 100644 index 0000000..047e263 --- /dev/null +++ b/apps/extension/src/tools/download.ts @@ -0,0 +1,64 @@ +// Download orchestration: validate session/tab ownership, then delegate one +// browser-global chrome.downloads transaction to download-capture.ts. + +import type { SessionManager } from "@/session-manager/manager"; +import type { DownloadParams, DownloadResult, RpcError } from "@/transport/types"; +import { captureBrowserDownload, chromeDownloadsApi, type DownloadsApi } from "./download-capture"; +import { clickResolvedTarget, type InteractionDeps, resolveActionTarget } from "./interaction"; +import { enforceAgentWindow, isRpcError, lookupSession, resolveTargetTab } from "./shared"; + +let downloadActive = false; + +export type { DownloadsApi } from "./download-capture"; + +export interface DownloadDeps extends InteractionDeps { + downloads?: DownloadsApi; +} + +export async function handleDownload( + manager: SessionManager, + params: DownloadParams, + deps: DownloadDeps, +): Promise { + if (downloadActive) return { code: "invalid_params", message: "another bsk download is active" }; + downloadActive = true; + try { + const ctx = lookupSession(manager, params, "download"); + if (isRpcError(ctx)) return ctx; + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, "download"); + if (denied) return denied; + if (!params.browser_relative_dir) { + return { code: "invalid_params", message: "download requires a daemon capability directory" }; + } + const address = await resolveActionTarget(deps.cdp, ctx, target, params, "download"); + if (isRpcError(address)) return address; + + const capture = await captureBrowserDownload({ + cdp: deps.cdp, + target: address.cdpTarget, + downloads: deps.downloads ?? chromeDownloadsApi, + browserRelativeDir: params.browser_relative_dir, + maxByteSize: params.max_byte_size, + timeoutMs: params.timeout_ms ?? 120_000, + signal: deps.signal, + expectedFrameId: address.frameId, + trigger: () => clickResolvedTarget(ctx, address, {}, deps), + }); + if (isRpcError(capture)) return capture; + const { click, item } = capture; + return { + tab_id: target.tabId, + used_ref: click.used_ref, + used_selector: click.used_selector, + suggested_filename: item.filename.split(/[\\/]/).pop() ?? "download", + byte_size: item.fileSize >= 0 ? item.fileSize : item.totalBytes, + mime: item.mime || undefined, + danger: item.danger, + browser_path: item.filename, + }; + } finally { + downloadActive = false; + } +} diff --git a/apps/extension/src/tools/errors.ts b/apps/extension/src/tools/errors.ts index 88cfd78..800a765 100644 --- a/apps/extension/src/tools/errors.ts +++ b/apps/extension/src/tools/errors.ts @@ -2,10 +2,23 @@ // rendering. Extension handlers attach reasons here; human-facing copy // lives in bsk-cli `render_error.rs`. -import type { ErrorCode, RpcError, RpcErrorData, RpcErrorReason } from "@/transport/types"; +import type { + ErrorCode, + RpcError, + RpcErrorData, + RpcErrorReason, + TransferCleanupState, + TransferEffectState, +} from "@/transport/types"; export type { RpcErrorData, RpcErrorReason }; +export interface TransferErrorOptions { + effectState: TransferEffectState; + phase: string; + cleanupState?: TransferCleanupState; +} + export function rpcError( code: ErrorCode, reason: RpcErrorReason, @@ -15,3 +28,16 @@ export function rpcError( const data: RpcErrorData = { reason, ...extra }; return { code, message, data }; } + +export function transferError( + code: ErrorCode, + reason: RpcErrorReason, + message: string, + options: TransferErrorOptions, +): RpcError { + return rpcError(code, reason, message, { + effect_state: options.effectState, + phase: options.phase, + ...(options.cleanupState ? { cleanup_state: options.cleanupState } : {}), + }); +} diff --git a/apps/extension/src/tools/file-drop-transaction.ts b/apps/extension/src/tools/file-drop-transaction.ts new file mode 100644 index 0000000..f806e84 --- /dev/null +++ b/apps/extension/src/tools/file-drop-transaction.ts @@ -0,0 +1,319 @@ +// One native file-drop transaction bound to an explicit page target. CDP drag +// events are sent to the target that owns the node: root targets use top-level +// coordinates, while OOPIF targets require coordinates local to their session. + +import type { RpcError, TransferEffectState } from "@/transport/types"; +import { transferError } from "./errors"; +import { resolveNodeGeometry } from "./frame-geometry"; +import type { ResolvedActionTarget } from "./interaction"; +import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared"; +import { BoundedWaitError, waitBounded } from "./transfer-transaction"; + +const CLEANUP_TIMEOUT_MS = 1_000; + +type FileDropPhase = "resolve_target" | "drag_enter" | "drag_over" | "drop" | "cleanup"; + +interface RuntimeReply { + result?: { value?: unknown; objectId?: string }; + exceptionDetails?: { text?: string; exception?: { description?: string } }; +} + +interface DragData { + items: []; + files: string[]; + dragOperationsMask: number; +} + +export interface FileDropTransactionOptions { + cdp: CdpRunner; + actionTarget: ResolvedActionTarget; + files: string[]; + timeoutMs: number; + signal?: AbortSignal; + withOverlayHidden: (operation: () => Promise) => Promise; +} + +export interface FileDropTransactionResult { + x: number; + y: number; +} + +function failure( + code: RpcError["code"], + reason: "upload_mechanism_unsupported" | "file_drop_target_unavailable" | "file_drop_failed", + message: string, + effectState: TransferEffectState, + phase: FileDropPhase, +): RpcError { + const error = transferError(code, reason, message, { effectState, phase }); + error.data = { ...error.data, mechanism: "drop" }; + return error; +} + +function enrichFailure( + error: RpcError, + effectState: TransferEffectState, + phase: FileDropPhase, +): RpcError { + return { + ...error, + data: { + ...error.data, + mechanism: "drop", + effect_state: effectState, + phase, + }, + }; +} + +function messageOf(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function methodUnavailable(error: unknown): boolean { + return /(?:method|command).*(?:not found|wasn't found|unsupported)|wasn't found|-32601/i.test( + messageOf(error), + ); +} + +function runtimeError(reply: RuntimeReply, fallback: string): Error | null { + if (!reply.exceptionDetails) return null; + return new Error( + reply.exceptionDetails.exception?.description ?? reply.exceptionDetails.text ?? fallback, + ); +} + +async function pointBelongsToTarget( + options: FileDropTransactionOptions, + x: number, + y: number, + objectGroup: string, +): Promise { + const target = options.actionTarget.cdpTarget; + const hit = await sendToCdpTarget<{ backendNodeId?: number; frameId?: string }>( + options.cdp, + target, + "DOM.getNodeForLocation", + { x: Math.round(x), y: Math.round(y), includeUserAgentShadowDOM: true }, + ); + if (typeof hit.backendNodeId !== "number") return false; + if (options.actionTarget.frameId && hit.frameId !== options.actionTarget.frameId) return false; + + const [resolvedTarget, resolvedHit] = await Promise.all([ + sendToCdpTarget<{ object?: { objectId?: string } }>(options.cdp, target, "DOM.resolveNode", { + backendNodeId: options.actionTarget.backendNodeId, + objectGroup, + }), + sendToCdpTarget<{ object?: { objectId?: string } }>(options.cdp, target, "DOM.resolveNode", { + backendNodeId: hit.backendNodeId, + objectGroup, + }), + ]); + const targetObjectId = resolvedTarget.object?.objectId; + const hitObjectId = resolvedHit.object?.objectId; + if (!targetObjectId || !hitObjectId) return false; + + const reply = await sendToCdpTarget(options.cdp, target, "Runtime.callFunctionOn", { + objectId: targetObjectId, + functionDeclaration: `function(hit) { + for (let node = hit; node; ) { + if (node === this) return true; + if (this.contains?.(node)) return true; + const root = node.getRootNode?.(); + node = node.parentNode || (root && root !== node ? root.host : null); + } + return false; + }`, + arguments: [{ objectId: hitObjectId }], + returnByValue: true, + }); + const error = runtimeError(reply, "failed to verify file-drop target"); + if (error) throw error; + return reply.result?.value === true; +} + +export async function uploadThroughFileDrop( + options: FileDropTransactionOptions, +): Promise { + const deadline = Date.now() + options.timeoutMs; + const target = options.actionTarget.cdpTarget; + const objectGroup = `bsk-file-drop-${crypto.randomUUID()}`; + const dragData: DragData = { + items: [], + files: options.files, + dragOperationsMask: 1, + }; + let dragStarted = false; + let dropSent = false; + let dragPoint = { x: 0, y: 0 }; + let cleanupFailed = false; + let outcome: FileDropTransactionResult | RpcError = failure( + "protocol_error", + "file_drop_failed", + "file-drop transaction ended without an outcome", + "none", + "cleanup", + ); + + try { + const geometry = await waitBounded( + resolveNodeGeometry( + options.cdp, + options.actionTarget.tab.tabId, + { + target, + backendNodeId: options.actionTarget.backendNodeId, + ...(options.actionTarget.frameId ? { frameId: options.actionTarget.frameId } : {}), + }, + { scrollIntoView: true }, + ), + deadline, + options.signal, + "resolving file-drop target timed out", + ); + if (isRpcError(geometry)) { + outcome = enrichFailure(geometry, "none", "resolve_target"); + return outcome; + } + const point = geometry.targetActionPoint; + dragPoint = point; + + outcome = await options.withOverlayHidden(async () => { + try { + try { + const belongs = await waitBounded( + pointBelongsToTarget(options, point.x, point.y, objectGroup), + deadline, + options.signal, + "verifying file-drop target timed out", + ); + if (!belongs) { + return failure( + "permission_denied", + "file_drop_target_unavailable", + "file-drop target is not the topmost element at its action point", + "none", + "resolve_target", + ); + } + } catch (error) { + return failure( + error instanceof BoundedWaitError ? "timeout" : "cdp_failed", + "file_drop_target_unavailable", + messageOf(error), + "none", + "resolve_target", + ); + } + + for (const [type, phase] of [ + ["dragEnter", "drag_enter"], + ["dragOver", "drag_over"], + ] as const) { + try { + if (type === "dragEnter") dragStarted = true; + await waitBounded( + sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", { + type, + x: point.x, + y: point.y, + data: dragData, + }), + deadline, + options.signal, + `${type} timed out`, + ); + } catch (error) { + const unsupported = methodUnavailable(error); + if (type === "dragEnter" && unsupported) dragStarted = false; + return failure( + unsupported + ? "unsupported" + : error instanceof BoundedWaitError + ? "timeout" + : "cdp_failed", + unsupported ? "upload_mechanism_unsupported" : "file_drop_failed", + messageOf(error), + "none", + phase, + ); + } + } + + try { + dropSent = true; + await waitBounded( + sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", { + type: "drop", + x: point.x, + y: point.y, + data: dragData, + }), + deadline, + options.signal, + "file drop timed out", + ); + return { x: point.x, y: point.y }; + } catch (error) { + outcome = failure( + error instanceof BoundedWaitError ? "timeout" : "cdp_failed", + "file_drop_failed", + messageOf(error), + "unknown", + "drop", + ); + return outcome; + } + } finally { + if (dragStarted && !dropSent) { + try { + await waitBounded( + sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", { + type: "dragCancel", + x: dragPoint.x, + y: dragPoint.y, + data: dragData, + }), + Date.now() + CLEANUP_TIMEOUT_MS, + undefined, + "cancelling file drag timed out", + ); + } catch { + cleanupFailed = true; + } + } + } + }); + return outcome; + } catch (error) { + outcome = failure( + error instanceof BoundedWaitError ? "timeout" : "cdp_failed", + "file_drop_target_unavailable", + messageOf(error), + "none", + "resolve_target", + ); + return outcome; + } finally { + try { + await waitBounded( + sendToCdpTarget(options.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }), + Date.now() + CLEANUP_TIMEOUT_MS, + undefined, + "releasing file-drop object group timed out", + ); + } catch { + cleanupFailed = true; + } + + const effect = isRpcError(outcome) + ? (outcome.data?.effect_state as TransferEffectState | undefined) + : "committed"; + if (effect === "unknown" || cleanupFailed) { + await options.cdp.detach?.(target.tabId); + } + if (cleanupFailed && isRpcError(outcome)) { + outcome.data = { ...outcome.data, cleanup_state: "failed" }; + } + } +} diff --git a/apps/extension/src/tools/file-input-transaction.ts b/apps/extension/src/tools/file-input-transaction.ts new file mode 100644 index 0000000..c37dce2 --- /dev/null +++ b/apps/extension/src/tools/file-input-transaction.ts @@ -0,0 +1,445 @@ +// One upload transaction with an explicit browser-side commit boundary. +// Chrome interception prevents a native chooser from escaping automation; +// chooser events and a frame-scoped DOM probe are independent input-location +// signals, so event delivery is not required for standard file inputs. + +import type { ClickResult, RpcError, TransferEffectState } from "@/transport/types"; +import { transferError } from "./errors"; +import type { ResolvedActionTarget } from "./interaction"; +import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared"; +import { BoundedWaitError, remainingMs, waitBounded } from "./transfer-transaction"; + +const CLEANUP_TIMEOUT_MS = 1_000; +const ACTIVATION_GRACE_MS = 1_000; +const PROBE_INTERVAL_MS = 20; + +type UploadPhase = + | "arm_interception" + | "arm_input_probe" + | "trigger" + | "resolve_input" + | "set_files" + | "cleanup"; + +interface RuntimeReply { + result?: { value?: unknown; objectId?: string }; + exceptionDetails?: { text?: string; exception?: { description?: string } }; +} + +interface ChooserEvent { + frameId?: string; + backendNodeId?: number; + mode?: "selectSingle" | "selectMultiple"; +} + +export interface FileInputTransactionOptions { + cdp: CdpRunner; + actionTarget: ResolvedActionTarget; + files: string[]; + timeoutMs: number; + signal?: AbortSignal; + trigger(): Promise; +} + +export interface FileInputTransactionResult { + click: ClickResult; + multiple: boolean; +} + +function runtimeError(reply: RuntimeReply, fallback: string): Error | null { + if (!reply.exceptionDetails) return null; + return new Error( + reply.exceptionDetails.exception?.description ?? reply.exceptionDetails.text ?? fallback, + ); +} + +function transferFailure( + code: RpcError["code"], + reason: "file_input_probe_failed" | "file_input_not_activated" | "set_file_input_failed", + message: string, + effectState: TransferEffectState, + phase: UploadPhase, +): RpcError { + return transferError(code, reason, message, { effectState, phase }); +} + +function enrichFailure( + error: RpcError, + effectState: TransferEffectState, + phase: UploadPhase, +): RpcError { + return { + ...error, + data: { ...error.data, effect_state: effectState, phase }, + }; +} + +function sameTarget( + source: { tabId?: number; sessionId?: string }, + target: { tabId: number; sessionId?: string }, +): boolean { + return source.tabId === target.tabId && source.sessionId === target.sessionId; +} + +async function delay(ms: number, signal?: AbortSignal): Promise { + await waitBounded( + new Promise((resolve) => setTimeout(resolve, ms)), + Date.now() + ms + 1, + signal, + "upload activation probe timed out", + ); +} + +async function callOnTrigger( + options: FileInputTransactionOptions, + objectId: string, + functionDeclaration: string, + args: unknown[], + returnByValue: boolean, +): Promise { + return sendToCdpTarget(options.cdp, options.actionTarget.cdpTarget, "Runtime.callFunctionOn", { + objectId, + functionDeclaration, + arguments: args.map((value) => ({ value })), + returnByValue, + awaitPromise: false, + }); +} + +export async function uploadThroughActivatedFileInput( + options: FileInputTransactionOptions, +): Promise { + const deadline = Date.now() + options.timeoutMs; + const target = options.actionTarget.cdpTarget; + const objectGroup = `bsk-upload-${crypto.randomUUID()}`; + const stateKey = `__bskUpload_${crypto.randomUUID().replaceAll("-", "")}`; + const chooserEvents: ChooserEvent[] = []; + let interceptionArmed = false; + let probeArmed = false; + let triggerObjectId: string | undefined; + let outcome: FileInputTransactionResult | RpcError = transferFailure( + "protocol_error", + "file_input_probe_failed", + "upload transaction ended without an outcome", + "none", + "cleanup", + ); + + const chooserSubscription = options.cdp.onEvent?.((source, method, raw) => { + if (method !== "Page.fileChooserOpened" || !sameTarget(source, target)) return; + const event = raw as ChooserEvent; + chooserEvents.push(event); + }); + + try { + try { + await waitBounded( + sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", { + enabled: true, + }), + deadline, + options.signal, + "arming native file chooser interception timed out", + ); + interceptionArmed = true; + } catch (err) { + outcome = transferFailure( + err instanceof BoundedWaitError ? "timeout" : "cdp_failed", + "file_input_probe_failed", + err instanceof Error ? err.message : String(err), + "none", + "arm_interception", + ); + return outcome; + } + + try { + const resolved = await waitBounded( + sendToCdpTarget<{ object?: { objectId?: string } }>( + options.cdp, + target, + "DOM.resolveNode", + { + backendNodeId: options.actionTarget.backendNodeId, + objectGroup, + }, + ), + deadline, + options.signal, + "resolving upload trigger timed out", + ); + triggerObjectId = resolved.object?.objectId; + if (!triggerObjectId) throw new Error("DOM.resolveNode returned no trigger objectId"); + + const armed = await waitBounded( + callOnTrigger( + options, + triggerObjectId, + `function(key) { + const doc = this.ownerDocument; + const owner = doc.defaultView; + if (!owner) return false; + const state = { inputs: [], listener: null }; + Object.defineProperty(owner, key, { value: state, configurable: true }); + state.listener = event => { + const path = typeof event.composedPath === "function" ? event.composedPath() : []; + const candidate = path[0] || event.target; + if (candidate && candidate.nodeType === 1 && + candidate.localName === "input" && candidate.type === "file") { + if (!state.inputs.includes(candidate)) state.inputs.push(candidate); + event.preventDefault(); + } + }; + doc.addEventListener("click", state.listener, true); + return true; + }`, + [stateKey], + true, + ), + deadline, + options.signal, + "arming frame-scoped file input probe timed out", + ); + const armError = runtimeError(armed, "failed to arm file input probe"); + if (armError || armed.result?.value !== true) { + throw armError ?? new Error("file input probe did not arm"); + } + probeArmed = true; + } catch (err) { + outcome = transferFailure( + err instanceof BoundedWaitError ? "timeout" : "cdp_failed", + "file_input_probe_failed", + err instanceof Error ? err.message : String(err), + "none", + "arm_input_probe", + ); + return outcome; + } + + let click: ClickResult | RpcError; + try { + click = await waitBounded( + options.trigger(), + deadline, + options.signal, + "upload trigger timed out", + ); + } catch (err) { + outcome = transferFailure( + err instanceof BoundedWaitError ? "timeout" : "cdp_failed", + "file_input_probe_failed", + err instanceof Error ? err.message : String(err), + "none", + "trigger", + ); + return outcome; + } + if (isRpcError(click)) { + outcome = enrichFailure(click, "none", "trigger"); + return outcome; + } + + try { + const activationDeadline = Math.min(deadline, Date.now() + ACTIVATION_GRACE_MS); + let summary: { count: number; multiple: boolean } = { count: 0, multiple: false }; + while (Date.now() < activationDeadline) { + if (chooserEvents.length > 0) break; + const reply = await callOnTrigger( + options, + triggerObjectId, + `function(key) { + const state = this.ownerDocument.defaultView?.[key]; + return state + ? { count: state.inputs.length, multiple: state.inputs[0]?.multiple === true } + : { count: 0, multiple: false }; + }`, + [stateKey], + true, + ); + const summaryError = runtimeError(reply, "failed to inspect activated file input"); + if (summaryError) throw summaryError; + const value = reply.result?.value as { count?: unknown; multiple?: unknown } | undefined; + summary = { + count: typeof value?.count === "number" ? value.count : 0, + multiple: value?.multiple === true, + }; + if (summary.count > 0) break; + await delay(Math.min(PROBE_INTERVAL_MS, remainingMs(activationDeadline)), options.signal); + } + + if (chooserEvents.length > 1) { + outcome = transferFailure( + "unsupported", + "file_input_not_activated", + "upload trigger activated more than one file chooser", + "none", + "resolve_input", + ); + return outcome; + } + + const chooser = chooserEvents[0]; + let backendNodeId: number | undefined; + let multiple = summary.multiple; + if (chooser) { + if (options.actionTarget.frameId && chooser.frameId !== options.actionTarget.frameId) { + outcome = transferFailure( + "unsupported", + "file_input_not_activated", + "upload trigger activated a file chooser in a different frame", + "none", + "resolve_input", + ); + return outcome; + } + if (typeof chooser.backendNodeId !== "number") { + outcome = transferFailure( + "unsupported", + "file_input_not_activated", + "upload trigger invoked a non-input file picker", + "none", + "resolve_input", + ); + return outcome; + } + backendNodeId = chooser.backendNodeId; + multiple = chooser.mode === "selectMultiple"; + } else { + if (summary.count !== 1) { + outcome = transferFailure( + "unsupported", + "file_input_not_activated", + summary.count === 0 + ? "upload trigger did not activate an input[type=file]" + : "upload trigger activated more than one input[type=file]", + "none", + "resolve_input", + ); + return outcome; + } + const input = await callOnTrigger( + options, + triggerObjectId, + `function(key) { return this.ownerDocument.defaultView?.[key]?.inputs[0]; }`, + [stateKey], + false, + ); + const inputError = runtimeError(input, "failed to resolve activated file input object"); + if (inputError) throw inputError; + if (!input.result?.objectId) throw new Error("activated file input returned no objectId"); + const described = await sendToCdpTarget<{ node?: { backendNodeId?: number } }>( + options.cdp, + target, + "DOM.describeNode", + { objectId: input.result.objectId }, + ); + backendNodeId = described.node?.backendNodeId; + if (typeof backendNodeId !== "number") { + throw new Error("DOM.describeNode returned no file input backendNodeId"); + } + } + + if (!multiple && options.files.length !== 1) { + outcome = enrichFailure( + { code: "invalid_params", message: "file input accepts exactly one file" }, + "none", + "resolve_input", + ); + return outcome; + } + + try { + await waitBounded( + sendToCdpTarget(options.cdp, target, "DOM.setFileInputFiles", { + files: options.files, + backendNodeId, + }), + deadline, + options.signal, + "setting file input files timed out", + ); + } catch (err) { + outcome = transferFailure( + err instanceof BoundedWaitError ? "timeout" : "cdp_failed", + "set_file_input_failed", + err instanceof Error ? err.message : String(err), + "unknown", + "set_files", + ); + return outcome; + } + outcome = { click, multiple }; + return outcome; + } catch (err) { + outcome = transferFailure( + err instanceof BoundedWaitError ? "timeout" : "cdp_failed", + "file_input_probe_failed", + err instanceof Error ? err.message : String(err), + "none", + "resolve_input", + ); + return outcome; + } + } finally { + chooserSubscription?.dispose(); + let cleanupFailed = false; + if (probeArmed && triggerObjectId) { + try { + await waitBounded( + callOnTrigger( + options, + triggerObjectId, + `function(key) { + const owner = this.ownerDocument.defaultView; + const state = owner?.[key]; + if (state?.listener) this.ownerDocument.removeEventListener("click", state.listener, true); + if (owner) delete owner[key]; + }`, + [stateKey], + true, + ), + Date.now() + CLEANUP_TIMEOUT_MS, + undefined, + "cleaning file input probe timed out", + ); + } catch { + cleanupFailed = true; + } + } + if (interceptionArmed) { + try { + await waitBounded( + sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", { + enabled: false, + cancel: true, + }), + Date.now() + CLEANUP_TIMEOUT_MS, + undefined, + "disabling file chooser interception timed out", + ); + } catch { + cleanupFailed = true; + } + } + try { + await waitBounded( + sendToCdpTarget(options.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }), + Date.now() + CLEANUP_TIMEOUT_MS, + undefined, + "releasing upload object group timed out", + ); + } catch { + cleanupFailed = true; + } + + const effect = isRpcError(outcome) + ? (outcome.data?.effect_state as TransferEffectState | undefined) + : "committed"; + if (effect === "unknown" || cleanupFailed) { + await options.cdp.detach?.(target.tabId); + } + if (cleanupFailed && isRpcError(outcome)) { + outcome.data = { ...outcome.data, cleanup_state: "failed" }; + } + } +} diff --git a/apps/extension/src/tools/frame-geometry.ts b/apps/extension/src/tools/frame-geometry.ts index ba2be42..0c788c9 100644 --- a/apps/extension/src/tools/frame-geometry.ts +++ b/apps/extension/src/tools/frame-geometry.ts @@ -29,7 +29,10 @@ export interface NodeAddress { export interface ResolvedNodeGeometry { topVisibleRegions: Region; topBounds: ViewportRect; + /** Point in the top-level tab viewport, used by root-target input events. */ actionPoint: Point; + /** Point in the addressed CDP target's viewport, used by OOPIF-local input events. */ + targetActionPoint: Point; } function geometryError(message: string): RpcError { @@ -262,11 +265,30 @@ export async function resolveNodeGeometry( if (isRpcError(localRegion)) return localRegion; let topVisibleRegions: Region; + let targetActionPoint: Point | null = null; if (address.frameId && graph) { const projection = await resolveFrameProjection(cdp, graph, address.frameId); if (!projection) return geometryError(`could not resolve frame geometry for ${address.frameId}`); topVisibleRegions = projectRegionToViewport(localRegion, projection); + if (address.target.sessionId) { + const localViewport = projection.edges[0]?.sourceViewport ?? projection.topViewport; + const localVisibleRegions = localRegion + .map((polygon) => + clipPolygon( + polygon, + rectPolygon({ + x: 0, + y: 0, + w: localViewport.width, + h: localViewport.height, + }), + ), + ) + .filter((polygon) => polygon.length >= 3); + const localActionRegion = largestRegion(localVisibleRegions); + targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null; + } } else { const viewport = await targetViewport(cdp, address.target); if (!viewport) return geometryError("could not resolve top viewport geometry"); @@ -283,7 +305,11 @@ export async function resolveNodeGeometry( if (!topBounds || !actionPoint) { return { code: "permission_denied", message: "element not visible" }; } - return { topVisibleRegions, topBounds, actionPoint }; + if (address.target.sessionId && !targetActionPoint) { + return { code: "permission_denied", message: "element not visible in its target" }; + } + targetActionPoint ??= actionPoint; + return { topVisibleRegions, topBounds, actionPoint, targetActionPoint }; } catch (error) { return geometryError(error instanceof Error ? error.message : String(error)); } diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index 182cabd..54646ed 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -40,6 +40,7 @@ import { enforceAgentWindow, isRpcError, lookupSession, + type ResolvedTargetTab, resolveTargetTab, } from "./shared"; import { resolveSnapshotRef } from "./snapshot-ref"; @@ -56,6 +57,15 @@ export interface InteractionDeps { keepOverlayBypassAfterHover?: boolean; } +export interface ResolvedActionTarget { + tab: ResolvedTargetTab; + backendNodeId: number; + cdpTarget: CdpTarget; + frameId?: string; + usedRef?: string; + usedSelector?: string; +} + const DEFAULT_TIMEOUT_MS = 30_000; const DEFAULT_HOVER_SETTLE_MS = 200; @@ -128,7 +138,7 @@ async function wait(ms: number, signal?: AbortSignal): Promise { * `RpcError` if the caller supplied neither (or both), or if neither * lookup matched. */ -async function resolveBackendNode( +export async function resolveBackendNode( cdp: CdpRunner, ctx: SessionContext, target: { tabId: number }, @@ -215,6 +225,17 @@ async function resolveBackendNode( } } +export async function resolveActionTarget( + cdp: CdpRunner, + ctx: SessionContext, + target: ResolvedTargetTab, + params: { ref?: string; selector?: string }, + toolName: string, +): Promise { + const node = await resolveBackendNode(cdp, ctx, target, params, toolName); + return isRpcError(node) ? node : { tab: target, ...node }; +} + // --------------------------------------------------------------------------- // tool.click // --------------------------------------------------------------------------- @@ -233,10 +254,19 @@ export async function handleClick( if (isRpcError(target)) return target; const denied = enforceAgentWindow(ctx, target, "click"); if (denied) return denied; - const dialogCursor = markDialogCursor(deps.cdp, target.tabId); + const resolved = await resolveActionTarget(deps.cdp, ctx, target, params, "click"); + if (isRpcError(resolved)) return resolved; + return clickResolvedTarget(ctx, resolved, params, deps); +} - const node = await resolveBackendNode(deps.cdp, ctx, target, params, "click"); - if (isRpcError(node)) return node; +export async function clickResolvedTarget( + ctx: SessionContext, + resolved: ResolvedActionTarget, + params: Pick, + deps: InteractionDeps, +): Promise { + const { tab: target } = resolved; + const dialogCursor = markDialogCursor(deps.cdp, target.tabId); if (throwIfAborted(deps.signal)) { return { code: "cancelled", message: "click aborted" }; @@ -247,9 +277,9 @@ export async function handleClick( deps.cdp, target.tabId, { - target: node.cdpTarget, - backendNodeId: node.backendNodeId, - ...(node.frameId ? { frameId: node.frameId } : {}), + target: resolved.cdpTarget, + backendNodeId: resolved.backendNodeId, + ...(resolved.frameId ? { frameId: resolved.frameId } : {}), }, { scrollIntoView: true }, ); @@ -337,8 +367,8 @@ export async function handleClick( return attachDialogs(deps.cdp, target.tabId, dialogCursor, { tab_id: target.tabId, - used_ref: node.usedRef, - used_selector: node.usedSelector, + used_ref: resolved.usedRef, + used_selector: resolved.usedSelector, x: centre.x, y: centre.y, }); diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index ec78d18..9c944ab 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -17,7 +17,7 @@ import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; import type { CdpTarget } from "@/browser-driver/frame-graph"; import { type CaptureSuppressSendToTab, - withOverlaysHiddenForCapture, + withExtensionOverlayHidden, } from "@/lib/capture-suppress-bridge"; import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { @@ -49,11 +49,14 @@ import { import { resolveSnapshotRef } from "./snapshot-ref"; import { type CapturedNode, + type CapturedSurfaceProbe, type CapturedViewModel, captureViewModel, collectOverlayExcludedBackendIds, + probeHoverSurfaces, } from "./vom/capture"; import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture"; +import { withOverlayBypass } from "./vom/hover-perception"; import { probeTooltipNames } from "./vom/name-enrichment"; import { type CaptureVomObservationResult, @@ -321,7 +324,7 @@ export async function handleScreenshot( tabId: target.tabId, ...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}), }; - const captured = await withOverlaysHiddenForCapture( + const captured = await withExtensionOverlayHidden( target.tabId, () => captureElementScreenshot( @@ -353,7 +356,7 @@ export async function handleScreenshot( ); } - const captured = await withOverlaysHiddenForCapture( + const captured = await withExtensionOverlayHidden( target.tabId, () => captureFullTabPng(deps, ctx, target, signal), deps.sendToTab, @@ -525,8 +528,11 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A return blocks; } -function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] { - const probes = captured.surfaceProbes ?? []; +function buildConditionalSurfaces( + nodes: VomNode[], + captured: CapturedViewModel, + probes: CapturedSurfaceProbe[], +): CondSurface[] { if (probes.length === 0) return []; const surfaces: CondSurface[] = []; @@ -649,6 +655,7 @@ function findSurfaceNodeByPoint( export interface BuildVomSceneOptions { pageUrl?: string; supplementalNames?: ReadonlyMap; + surfaceProbes?: CapturedSurfaceProbe[]; } export type VomFrameDocument = CapturedFrameDocument; @@ -817,18 +824,19 @@ export function buildFrameVomScene( excludedBackendNodeIds: captured.excludedBackendNodeIds, supplementalNames: options.supplementalNames, }); - return attachCapturedSceneAnnotations(scene, documents, captured); + return attachCapturedSceneAnnotations(scene, documents, captured, options.surfaceProbes ?? []); } function attachCapturedSceneAnnotations( scene: VomScene, documents: VomFrameDocument[], captured: CapturedViewModel, + surfaceProbes: CapturedSurfaceProbe[], ): VomScene { const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId); const signals = capturedOnlySignals(rootDocument?.domNodes ?? captured.nodes); const activeScopeBlocks = buildActiveScopeBlocks(scene.nodes, signals); - const surfaces = buildConditionalSurfaces(scene.nodes, captured); + const surfaces = buildConditionalSurfaces(scene.nodes, captured, surfaceProbes); return { ...scene, ...(surfaces.length > 0 ? { surfaces } : {}), @@ -1010,17 +1018,69 @@ async function captureForVom( options: CaptureVomObservationOptions, ): Promise { try { - return await captureViewModel(cdp, tabId, { - conditionalSurfaceProbe: options.conditionalSurfaceProbe ?? false, - hoverProbeBypassOverlay: options.hoverProbeBypassOverlay, - signal: options.signal, - }); + return await captureViewModel(cdp, tabId, { signal: options.signal }); } catch (error) { if (isAbortError(error)) throw error; return fallbackCapturedViewModel(cdp, tabId, options.signal); } } +export interface HoverProbeOutcome { + /** Whether any active hover was dispatched during this observation. */ + performed: boolean; + /** + * Whether hovering surfaced content absent from the static snapshot. Cached + * tooltip names count, so this may over-report on repeat observations — it + * errs towards telling the caller the snapshot is less fresh than it looks. + */ + revealedContent: boolean; + surfaceProbes: CapturedSurfaceProbe[]; + tooltipNames: Map; +} + +const NO_HOVER_PROBES: HoverProbeOutcome = { + performed: false, + revealedContent: false, + surfaceProbes: [], + tooltipNames: new Map(), +}; + +/** + * Runs both active-hover chains back to back. + * + * Ordering matters: this happens after DOM *and* accessibility capture so a + * hover that opens a menu cannot leave one half of the observation describing + * the page before the change and the other half after it. Sharing one overlay + * bypass span also means the agent overlay toggles once per observation + * instead of once per chain. + */ +async function runHoverProbes( + cdp: CdpRunner, + tabId: number, + captured: CapturedViewModel, + documents: VomFrameDocument[], + staticSemantics: ReturnType, + options: CaptureVomObservationOptions, +): Promise { + if (!options.conditionalSurfaceProbe) return NO_HOVER_PROBES; + + return withOverlayBypass(options.hoverProbeBypassOverlay, tabId, async () => { + const surfaceProbes = await probeHoverSurfaces(cdp, tabId, captured.nodes, { + signal: options.signal, + }); + throwIfAborted(options.signal, "observation"); + const tooltipNames = await probeTooltipNames(cdp, tabId, documents, staticSemantics, { + signal: options.signal, + }); + return { + performed: true, + revealedContent: surfaceProbes.length > 0 || tooltipNames.size > 0, + surfaceProbes, + tooltipNames, + }; + }); +} + export interface CaptureVomObservationOptions extends VomOptions { conditionalSurfaceProbe?: boolean; hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise; @@ -1051,19 +1111,26 @@ export async function captureVomObservation( excludedBackendNodeIds: captured.excludedBackendNodeIds, }); const staticSemantics = resolveSemanticGraph(semanticGraph, { identifierFallback: false }); - const tooltipNames = options.conditionalSurfaceProbe - ? await probeTooltipNames(cdp, tabId, normalizedDocuments, staticSemantics, { - signal: options.signal, - hoverProbeBypassOverlay: options.hoverProbeBypassOverlay, - }) - : new Map(); + const hoverProbes = await runHoverProbes( + cdp, + tabId, + captured, + normalizedDocuments, + staticSemantics, + options, + ); throwIfAborted(options.signal, "observation"); const scene = projectSemanticGraph( normalizeSemanticStructure( - resolveSemanticGraph(semanticGraph, { supplementalNames: tooltipNames }), + resolveSemanticGraph(semanticGraph, { supplementalNames: hoverProbes.tooltipNames }), ), ); - const decoratedScene = attachCapturedSceneAnnotations(scene, normalizedDocuments, captured); + const decoratedScene = attachCapturedSceneAnnotations( + scene, + normalizedDocuments, + captured, + hoverProbes.surfaceProbes, + ); const rendered = renderVom(decoratedScene, { maxDepth: options.maxDepth, maxTokens: options.maxTokens, @@ -1075,7 +1142,11 @@ export async function captureVomObservation( rootFrameId: captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root", frameDocuments: normalizedDocuments, rendered, - surfaceProbes: captured.surfaceProbes, + surfaceProbes: hoverProbes.surfaceProbes, + hoverProbe: { + performed: hoverProbes.performed, + revealedContent: hoverProbes.revealedContent, + }, }); } @@ -1141,6 +1212,14 @@ async function handleVomObservation( ref_count: observation.refs.length, tab_id: target.tabId, truncated: observation.truncated, + ...(toolName === "observe" && observation.hoverProbe?.performed + ? { + hover_probe: { + performed: true, + revealed_content: observation.hoverProbe.revealedContent, + }, + } + : {}), ...(toolName === "observe" && (params as ObserveParams).debug_surfaces ? { debug: { @@ -1179,5 +1258,13 @@ export async function handleObserve( deps: SnapshotDeps = getDefaultDeps(), signal?: AbortSignal, ): Promise { - return handleVomObservation(manager, params, "observe", "transient_input", true, deps, signal); + return handleVomObservation( + manager, + params, + "observe", + "transient_input", + params.probe_hover === true, + deps, + signal, + ); } diff --git a/apps/extension/src/tools/shared.ts b/apps/extension/src/tools/shared.ts index 3b2cb79..9483abf 100644 --- a/apps/extension/src/tools/shared.ts +++ b/apps/extension/src/tools/shared.ts @@ -53,6 +53,7 @@ export type { DialogCursor }; export interface CdpRunner { send(tabId: number, method: string, params?: object): Promise; sendToTarget?(target: CdpTarget, method: string, params?: object): Promise; + detach?(tabId: number): Promise; getFrameGraph?(tabId: number): Promise; ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise; trackSessionTab?(sessionId: string, tabId: number): void; diff --git a/apps/extension/src/tools/transfer-transaction.ts b/apps/extension/src/tools/transfer-transaction.ts new file mode 100644 index 0000000..6debaeb --- /dev/null +++ b/apps/extension/src/tools/transfer-transaction.ts @@ -0,0 +1,42 @@ +// Shared time boundary for browser-side file-transfer transactions. Concrete +// upload/download mechanisms own their state machines and effect semantics. + +export class BoundedWaitError extends Error { + constructor( + readonly kind: "timeout" | "aborted", + message: string, + ) { + super(message); + } +} + +export function remainingMs(deadline: number): number { + return Math.max(0, deadline - Date.now()); +} + +export async function waitBounded( + promise: Promise, + deadline: number, + signal: AbortSignal | undefined, + timeoutMessage: string, +): Promise { + const remaining = remainingMs(deadline); + if (signal?.aborted) throw new BoundedWaitError("aborted", "file transfer aborted"); + if (remaining === 0) throw new BoundedWaitError("timeout", timeoutMessage); + + let timer: ReturnType | undefined; + let onAbort: (() => void) | undefined; + const boundary = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new BoundedWaitError("timeout", timeoutMessage)), remaining); + if (signal) { + onAbort = () => reject(new BoundedWaitError("aborted", "file transfer aborted")); + signal.addEventListener("abort", onAbort, { once: true }); + } + }); + try { + return await Promise.race([promise, boundary]); + } finally { + if (timer) clearTimeout(timer); + if (signal && onAbort) signal.removeEventListener("abort", onAbort); + } +} diff --git a/apps/extension/src/tools/upload.ts b/apps/extension/src/tools/upload.ts new file mode 100644 index 0000000..9203dd2 --- /dev/null +++ b/apps/extension/src/tools/upload.ts @@ -0,0 +1,85 @@ +// Upload orchestration: validate the session-scoped request, resolve its click +// target, then delegate the browser protocol transaction to the file-input +// transaction module. + +import { + type CaptureSuppressSendToTab, + withExtensionOverlayHidden, +} from "@/lib/capture-suppress-bridge"; +import type { SessionManager } from "@/session-manager/manager"; +import type { RpcError, UploadParams, UploadResult } from "@/transport/types"; +import { uploadThroughFileDrop } from "./file-drop-transaction"; +import { uploadThroughActivatedFileInput } from "./file-input-transaction"; +import { clickResolvedTarget, type InteractionDeps, resolveActionTarget } from "./interaction"; +import { + type CdpRunner, + enforceAgentWindow, + isRpcError, + lookupSession, + resolveTargetTab, +} from "./shared"; + +const DEFAULT_TIMEOUT_MS = 120_000; + +export interface UploadDeps extends InteractionDeps { + cdp: CdpRunner; + sendToTab?: CaptureSuppressSendToTab; +} + +export async function handleUpload( + manager: SessionManager, + params: UploadParams, + deps: UploadDeps, +): Promise { + const ctx = lookupSession(manager, params, "upload"); + if (isRpcError(ctx)) return ctx; + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, "upload"); + if (denied) return denied; + if ( + params.files.length === 0 || + params.files.length > 20 || + params.files.some((file) => !file.staged_path) + ) { + return { code: "invalid_params", message: "upload requires daemon-staged files" }; + } + const address = await resolveActionTarget(deps.cdp, ctx, target, params, "upload"); + if (isRpcError(address)) return address; + const timeoutMs = params.timeout_ms ?? DEFAULT_TIMEOUT_MS; + const files = params.files.map((file) => file.staged_path as string); + const mode = params.mode ?? "input"; + let usedRef = address.usedRef; + let usedSelector = address.usedSelector; + if (mode === "drop") { + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const transaction = await uploadThroughFileDrop({ + cdp: deps.cdp, + actionTarget: address, + files, + timeoutMs, + signal: deps.signal, + withOverlayHidden: (operation) => + withExtensionOverlayHidden(target.tabId, operation, deps.sendToTab), + }); + if (isRpcError(transaction)) return transaction; + } else { + const transaction = await uploadThroughActivatedFileInput({ + cdp: deps.cdp, + actionTarget: address, + files, + timeoutMs, + signal: deps.signal, + trigger: () => clickResolvedTarget(ctx, address, {}, deps), + }); + if (isRpcError(transaction)) return transaction; + usedRef = transaction.click.used_ref; + usedSelector = transaction.click.used_selector; + } + return { + tab_id: target.tabId, + used_ref: usedRef, + used_selector: usedSelector, + file_names: params.files.map((file) => file.name), + }; +} diff --git a/apps/extension/src/tools/vom/__tests__/capture.test.ts b/apps/extension/src/tools/vom/__tests__/capture.test.ts index 9463383..86a2f00 100644 --- a/apps/extension/src/tools/vom/__tests__/capture.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from "vitest"; import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge"; -import { captureViewModel, collectOverlayExcludedBackendIds } from "../capture"; +import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture"; // Minimal but format-accurate captureSnapshot reply: a body with one // fixed full-screen overlay div carrying a password input. @@ -251,11 +251,10 @@ describe("captureViewModel", () => { expect(input?.attrs).toEqual({ type: "password" }); }); - it("does not run conditional surface probes by default", async () => { + it("never hovers the page while capturing", async () => { const cdp = makeCdp(fakeSnapshotReply()); - const result = await captureViewModel(cdp, 4); + await captureViewModel(cdp, 4); - expect(result.surfaceProbes).toEqual([]); expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything()); }); @@ -319,9 +318,10 @@ describe("captureViewModel", () => { }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true }); + const captured = await captureViewModel(cdp, 4); + const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes); - expect(result.surfaceProbes).toEqual([ + expect(surfaceProbes).toEqual([ { triggerBackendNodeId: 12, triggerPoint: { x: 956, y: 32 }, @@ -378,9 +378,10 @@ describe("captureViewModel", () => { }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true }); + const captured = await captureViewModel(cdp, 4); + const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes); - expect(result.surfaceProbes).toEqual([ + expect(surfaceProbes).toEqual([ expect.objectContaining({ triggerBackendNodeId: 12, subItems: ["My profile", "Sign out"], @@ -422,7 +423,8 @@ describe("captureViewModel", () => { }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true }); + const captured = await captureViewModel(cdp, 4); + await probeHoverSurfaces(cdp, 4, captured.nodes); expect(hoverMoves).toBe(1); }); diff --git a/apps/extension/src/tools/vom/__tests__/hover-perception.test.ts b/apps/extension/src/tools/vom/__tests__/hover-perception.test.ts new file mode 100644 index 0000000..32c662b --- /dev/null +++ b/apps/extension/src/tools/vom/__tests__/hover-perception.test.ts @@ -0,0 +1,84 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { ProbeBudget, resetOverlayBypassDepth, withOverlayBypass } from "../hover-perception"; + +afterEach(() => { + resetOverlayBypassDepth(); +}); + +describe("ProbeBudget", () => { + it("refuses a candidate that cannot finish inside the budget", () => { + const budget = new ProbeBudget(500); + + expect(budget.canAfford(400)).toBe(true); + expect(budget.canAfford(600)).toBe(false); + }); + + it("stops admitting candidates once the remaining budget is smaller than one", async () => { + const budget = new ProbeBudget(60); + await new Promise((resolve) => setTimeout(resolve, 40)); + + // A top-of-loop `elapsed > total` check would still admit this candidate + // and then overshoot by its full cost. + expect(budget.canAfford(50)).toBe(false); + }); +}); + +describe("withOverlayBypass", () => { + it("toggles the overlay once around nested probe phases", async () => { + const bypass = vi.fn(async () => {}); + + await withOverlayBypass(bypass, 4, async () => { + await withOverlayBypass(bypass, 4, async () => { + expect(bypass).toHaveBeenCalledTimes(1); + }); + expect(bypass).toHaveBeenCalledTimes(1); + }); + + expect(bypass.mock.calls).toEqual([ + [4, true], + [4, false], + ]); + }); + + it("restores the overlay when the probe throws", async () => { + const bypass = vi.fn(async () => {}); + + await expect( + withOverlayBypass(bypass, 4, async () => { + throw new Error("probe exploded"); + }), + ).rejects.toThrow("probe exploded"); + + expect(bypass).toHaveBeenLastCalledWith(4, false); + }); + + it("reports a failed restore instead of swallowing it", async () => { + const onRestoreFailure = vi.fn(); + const bypass = vi.fn(async (_tabId: number, enabled: boolean) => { + if (!enabled) throw new Error("tab is gone"); + }); + + await withOverlayBypass(bypass, 4, async () => undefined, { onRestoreFailure }); + + expect(onRestoreFailure).toHaveBeenCalledTimes(1); + }); + + it("does not pin the overlay off for the tab when a restore fails", async () => { + const failing = vi.fn(async (_tabId: number, enabled: boolean) => { + if (!enabled) throw new Error("tab is gone"); + }); + await withOverlayBypass(failing, 4, async () => undefined, { onRestoreFailure: () => {} }); + + const bypass = vi.fn(async () => {}); + await withOverlayBypass(bypass, 4, async () => undefined); + + expect(bypass.mock.calls).toEqual([ + [4, true], + [4, false], + ]); + }); + + it("runs the probe untouched when no bypass is wired up", async () => { + await expect(withOverlayBypass(undefined, 4, async () => "done")).resolves.toBe("done"); + }); +}); diff --git a/apps/extension/src/tools/vom/__tests__/name-enrichment.test.ts b/apps/extension/src/tools/vom/__tests__/name-enrichment.test.ts index 425517e..b4b2e4d 100644 --- a/apps/extension/src/tools/vom/__tests__/name-enrichment.test.ts +++ b/apps/extension/src/tools/vom/__tests__/name-enrichment.test.ts @@ -132,6 +132,37 @@ describe("tooltip name enrichment", () => { expect(cdp.sendToTarget).not.toHaveBeenCalled(); }); + it("remembers controls that revealed no tooltip so the miss is paid once", async () => { + const cdp: CdpRunner = { + send: vi.fn(async () => ({})) as CdpRunner["send"], + sendToTarget: vi.fn(async (_target, method) => { + if (method === "Page.createIsolatedWorld") return { executionContextId: 31 }; + if (method === "DOM.resolveNode") return { object: { objectId: "button-801" } }; + if (method === "Runtime.callFunctionOn") return { result: { value: [] } }; + return {}; + }) as CdpRunner["sendToTarget"], + }; + + const documents = [ + document({ tabId: 11, sessionId: "miss-session" }, 801, undefined, "miss-frame"), + ]; + const graph = semantics(documents); + + const first = await probeTooltipNames(cdp, 11, documents, graph); + expect(first.size).toBe(0); + const hoversAfterFirst = vi + .mocked(cdp.send) + .mock.calls.filter(([, method]) => method === "Input.dispatchMouseEvent").length; + expect(hoversAfterFirst).toBeGreaterThan(0); + + const second = await probeTooltipNames(cdp, 11, documents, graph); + expect(second.size).toBe(0); + expect( + vi.mocked(cdp.send).mock.calls.filter(([, method]) => method === "Input.dispatchMouseEvent") + .length, + ).toBe(hoversAfterFirst); + }); + it("keeps tooltip evidence isolated across sibling frame sessions", async () => { let hoveredX = -10; const cdp: CdpRunner = { diff --git a/apps/extension/src/tools/vom/capture.ts b/apps/extension/src/tools/vom/capture.ts index 1db0e29..6c7de38 100644 --- a/apps/extension/src/tools/vom/capture.ts +++ b/apps/extension/src/tools/vom/capture.ts @@ -9,7 +9,7 @@ import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy"; import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge"; import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry"; import type { CdpRunner } from "../shared"; -import { clearHover, waitForHover } from "./hover-perception"; +import { clearHover, ProbeBudget, waitForHover } from "./hover-perception"; const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const; const STYLE_COL = Object.fromEntries( @@ -71,14 +71,11 @@ export interface CapturedViewModel { /** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */ frameParentIds?: Map; rootFrameId?: string; - surfaceProbes?: CapturedSurfaceProbe[]; /** Backend node ids belonging to the agent overlay host + its shadow subtree. */ excludedBackendNodeIds: Set; } export interface CaptureViewModelOptions { - conditionalSurfaceProbe?: boolean; - hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise; signal?: AbortSignal; } @@ -368,7 +365,16 @@ function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set { - const started = Date.now(); + const budget = new ProbeBudget(MAX_HOVER_PROBE_MS); try { throwIfAborted(options.signal); const cssScan = await cdp.send(tabId, "Runtime.evaluate", { @@ -635,60 +663,54 @@ async function probeHoverSurfaces( const results: CapturedSurfaceProbe[] = []; const seen = new Set(); throwIfAborted(options.signal); - await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined); - try { + for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) { throwIfAborted(options.signal); - for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) { + if (!budget.canAfford(HOVER_CANDIDATE_COST_MS)) break; + if (results.length >= MAX_HOVER_SURFACES) break; + if (seen.has(candidate.backendNodeId)) continue; + try { + await clearHover(cdp, tabId); throwIfAborted(options.signal); - if (Date.now() - started > MAX_HOVER_PROBE_MS) break; - if (results.length >= MAX_HOVER_SURFACES) break; - if (seen.has(candidate.backendNodeId)) continue; - try { - await clearHover(cdp, tabId); - throwIfAborted(options.signal); - await waitForHover(HOVER_SETTLE_MS, options.signal); - const baselineReply = await cdp.send(tabId, "Runtime.evaluate", { - expression: hoverStateExpression(), - returnByValue: true, - }); - throwIfAborted(options.signal); - const baselineItems = runtimeValue(baselineReply) ?? []; + await waitForHover(HOVER_SETTLE_MS, options.signal); + const baselineReply = await cdp.send(tabId, "Runtime.evaluate", { + expression: hoverStateExpression(), + returnByValue: true, + }); + throwIfAborted(options.signal); + const baselineItems = runtimeValue(baselineReply) ?? []; - throwIfAborted(options.signal); - await cdp.send(tabId, "Input.dispatchMouseEvent", { - type: "mouseMoved", - x: candidate.x, - y: candidate.y, - }); - throwIfAborted(options.signal); - await waitForHover(HOVER_SETTLE_MS, options.signal); - const collected = await cdp.send(tabId, "Runtime.evaluate", { - expression: hoverStateExpression(), - returnByValue: true, - }); - throwIfAborted(options.signal); - const subItems = diffHoverItems( - baselineItems, - runtimeValue(collected) ?? [], - ); - if (subItems.length === 0) continue; - seen.add(candidate.backendNodeId); - results.push({ - triggerBackendNodeId: candidate.backendNodeId, - triggerPoint: { x: candidate.x, y: candidate.y }, - triggerAction: "hover", - subItems, - confidence: confidenceForHover(candidate, subItems), - }); - } catch (error) { - if (isAbortError(error)) throw error; - continue; - } finally { - await clearHover(cdp, tabId); - } + throwIfAborted(options.signal); + await cdp.send(tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: candidate.x, + y: candidate.y, + }); + throwIfAborted(options.signal); + await waitForHover(HOVER_SETTLE_MS, options.signal); + const collected = await cdp.send(tabId, "Runtime.evaluate", { + expression: hoverStateExpression(), + returnByValue: true, + }); + throwIfAborted(options.signal); + const subItems = diffHoverItems( + baselineItems, + runtimeValue(collected) ?? [], + ); + if (subItems.length === 0) continue; + seen.add(candidate.backendNodeId); + results.push({ + triggerBackendNodeId: candidate.backendNodeId, + triggerPoint: { x: candidate.x, y: candidate.y }, + triggerAction: "hover", + subItems, + confidence: confidenceForHover(candidate, subItems), + }); + } catch (error) { + if (isAbortError(error)) throw error; + continue; + } finally { + await clearHover(cdp, tabId); } - } finally { - await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined); } return results; } catch (err) { @@ -1042,7 +1064,6 @@ export async function captureViewModel( nodes: [], viewport, iframeNodes: new Map(), - surfaceProbes: [], excludedBackendNodeIds: new Set(), }; } @@ -1071,11 +1092,6 @@ export async function captureViewModel( await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()], options.signal); throwIfAborted(options.signal); - const surfaceProbes = options.conditionalSurfaceProbe - ? await probeHoverSurfaces(cdp, tabId, nodes, options) - : []; - throwIfAborted(options.signal); - const frameNodes = new Map(); if (topContext.frameId) frameNodes.set(topContext.frameId, nodes); for (const iframeFrameNodes of iframeNodes.values()) { @@ -1096,7 +1112,6 @@ export async function captureViewModel( frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds, frameParentIds: frameParsed.frameParentIds, ...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}), - surfaceProbes, excludedBackendNodeIds, }; } diff --git a/apps/extension/src/tools/vom/hover-perception.ts b/apps/extension/src/tools/vom/hover-perception.ts index 253f792..c177daf 100644 --- a/apps/extension/src/tools/vom/hover-perception.ts +++ b/apps/extension/src/tools/vom/hover-perception.ts @@ -35,3 +35,91 @@ export async function clearHover(cdp: CdpRunner, tabId: number): Promise { }), ); } + +/** + * Wall-clock budget for a probe phase. + * + * Callers ask whether the *next* candidate still fits before starting it, + * rather than checking elapsed time at the top of the loop. A top-of-loop + * check lets a candidate start with 1ms of budget left and then run to + * completion, overshooting by a full settle window. + */ +export class ProbeBudget { + private readonly startedAt = Date.now(); + + constructor(private readonly totalMs: number) {} + + get elapsedMs(): number { + return Date.now() - this.startedAt; + } + + /** True when a candidate costing `estimatedMs` still fits in the budget. */ + canAfford(estimatedMs: number): boolean { + return this.elapsedMs + estimatedMs <= this.totalMs; + } +} + +export type OverlayBypass = (tabId: number, enabled: boolean) => Promise; + +export interface OverlayBypassScopeOptions { + /** + * Called when the overlay could not be restored. Defaults to `console.error` + * so a stuck-bypassed overlay is never silent; tests inject a spy. + */ + onRestoreFailure?: (error: unknown) => void; +} + +const overlayBypassDepth = new Map(); + +/** + * Runs `body` with the agent overlay bypassed, restoring it afterwards. + * + * Reference counted per tab so nested probe phases share a single bypass span + * instead of toggling the overlay once per phase. Restore failures are + * reported rather than swallowed, and the depth is released even when restore + * throws so a failure cannot pin the overlay off for the tab's lifetime. + */ +export async function withOverlayBypass( + bypass: OverlayBypass | undefined, + tabId: number, + body: () => Promise, + options: OverlayBypassScopeOptions = {}, +): Promise { + if (!bypass) return body(); + + const depth = overlayBypassDepth.get(tabId) ?? 0; + overlayBypassDepth.set(tabId, depth + 1); + if (depth === 0) { + try { + await bypass(tabId, true); + } catch (error) { + // Failing to hide the overlay only risks probing our own UI, so continue. + console.debug("[bsk hover] overlay bypass could not be enabled", error); + } + } + + try { + return await body(); + } finally { + const current = overlayBypassDepth.get(tabId) ?? 1; + if (current <= 1) { + overlayBypassDepth.delete(tabId); + try { + await bypass(tabId, false); + } catch (error) { + (options.onRestoreFailure ?? defaultRestoreFailure)(error); + } + } else { + overlayBypassDepth.set(tabId, current - 1); + } + } +} + +function defaultRestoreFailure(error: unknown): void { + console.error("[bsk hover] agent overlay could not be restored after probing", error); +} + +/** Test seam: drops any bypass depth left behind by a crashed probe. */ +export function resetOverlayBypassDepth(): void { + overlayBypassDepth.clear(); +} diff --git a/apps/extension/src/tools/vom/name-enrichment.ts b/apps/extension/src/tools/vom/name-enrichment.ts index 1f42ee3..acff86a 100644 --- a/apps/extension/src/tools/vom/name-enrichment.ts +++ b/apps/extension/src/tools/vom/name-enrichment.ts @@ -1,7 +1,7 @@ import { isVomReferenceNode } from "@browser-skill/vom"; import { type CdpRunner, sendToCdpTarget } from "../shared"; import type { FrameDocument, FrameOwnedAxNode } from "./frame-document"; -import { clearHover, waitForHover } from "./hover-perception"; +import { clearHover, ProbeBudget, waitForHover } from "./hover-perception"; import { stableIdentifierName } from "./semantic-graph/name-evidence"; import { frameBackendKey, type ResolvedSemanticGraph } from "./semantic-graph/types"; @@ -9,11 +9,18 @@ const MAX_TOOLTIP_CANDIDATES = 8; const MAX_TOOLTIP_PROBE_MS = 2_400; const TOOLTIP_SETTLE_MS = 250; const CACHE_TTL_MS = 5 * 60_000; +/** + * Misses expire sooner than hits: a control that gains a tooltip once the app + * finishes booting should still be discoverable, but not at the cost of + * re-probing every unlabelled control on every observation. + */ +const MISS_CACHE_TTL_MS = 60_000; const MAX_CACHE_ENTRIES = 500; const FORM_CONTROL_TAGS = new Set(["input", "select", "textarea"]); interface TooltipCacheEntry { - name: string; + /** `null` records "probed, revealed no tooltip" so the miss is not re-paid. */ + name: string | null; expiresAt: number; } @@ -27,7 +34,6 @@ interface TooltipCandidate { export interface TooltipNameProbeOptions { signal?: AbortSignal; - hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise; } const tooltipCache = new Map(); @@ -39,10 +45,11 @@ function cacheKey( return `${document.target.tabId}\u0000${document.frameId}\u0000${document.url ?? ""}\u0000${backendNodeId}`; } -function cachedName( +/** Returns the cache entry, or `undefined` when this control was never probed. */ +function cachedEntry( document: FrameDocument, backendNodeId: number, -): string | undefined { +): TooltipCacheEntry | undefined { const key = cacheKey(document, backendNodeId); const entry = tooltipCache.get(key); if (!entry) return undefined; @@ -50,13 +57,13 @@ function cachedName( tooltipCache.delete(key); return undefined; } - return entry.name; + return entry; } function storeName( document: FrameDocument, backendNodeId: number, - name: string, + name: string | null, ): void { if (tooltipCache.size >= MAX_CACHE_ENTRIES) { const oldest = tooltipCache.keys().next().value; @@ -64,7 +71,7 @@ function storeName( } tooltipCache.set(cacheKey(document, backendNodeId), { name, - expiresAt: Date.now() + CACHE_TTL_MS, + expiresAt: Date.now() + (name === null ? MISS_CACHE_TTL_MS : CACHE_TTL_MS), }); } @@ -237,22 +244,27 @@ export async function probeTooltipNames( const names = new Map(); const pending: TooltipCandidate[] = []; for (const candidate of candidates(documents, graph)) { - const cached = cachedName(candidate.document, candidate.backendNodeId); + const cached = cachedEntry(candidate.document, candidate.backendNodeId); if (cached) { - names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), cached); + // A cached miss is still a decision: skip the control instead of re-probing. + if (cached.name !== null) { + names.set( + frameBackendKey(candidate.document.frameId, candidate.backendNodeId), + cached.name, + ); + } } else if (pending.length < MAX_TOOLTIP_CANDIDATES) { pending.push(candidate); } } if (pending.length === 0) return names; - const started = Date.now(); + const budget = new ProbeBudget(MAX_TOOLTIP_PROBE_MS); const contexts = new Map(); - await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined); try { for (const candidate of pending) { if (options.signal?.aborted) throw new DOMException("observation aborted", "AbortError"); - if (Date.now() - started >= MAX_TOOLTIP_PROBE_MS) break; + if (!budget.canAfford(TOOLTIP_SETTLE_MS)) break; let objectId: string | undefined; try { let contextId = contexts.get(candidate.document.frameId); @@ -278,9 +290,9 @@ export async function probeTooltipNames( await waitForHover(TOOLTIP_SETTLE_MS, options.signal); const after = await tooltipTexts(cdp, candidate.document, objectId); const name = cleanTooltipDiff(before, after); + storeName(candidate.document, candidate.backendNodeId, name ?? null); if (!name) continue; names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), name); - storeName(candidate.document, candidate.backendNodeId, name); } catch (error) { if (error instanceof Error && error.name === "AbortError") throw error; } finally { @@ -290,7 +302,6 @@ export async function probeTooltipNames( } } finally { await clearHover(cdp, tabId); - await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined); } return names; } diff --git a/apps/extension/src/tools/vom/record-safe-observation.ts b/apps/extension/src/tools/vom/record-safe-observation.ts index d58e757..13c4f5d 100644 --- a/apps/extension/src/tools/vom/record-safe-observation.ts +++ b/apps/extension/src/tools/vom/record-safe-observation.ts @@ -37,6 +37,17 @@ export interface CaptureVomObservationResult { frames: CaptureVomFrame[]; matchNodes: CaptureVomMatchNode[]; surfaceProbes?: CapturedSurfaceProbe[]; + /** + * Whether this observation actively hovered the page, and whether that + * revealed content the static tree does not contain. Lets callers judge how + * far the page may have drifted from the returned snapshot. + */ + hoverProbe?: HoverProbeReport; +} + +export interface HoverProbeReport { + performed: boolean; + revealedContent: boolean; } function projectFrames(documents: CapturedFrameDocument[]): CaptureVomFrame[] { @@ -75,6 +86,7 @@ export function projectRecordSafeObservation(input: { frameDocuments: CapturedFrameDocument[]; rendered: VomResult; surfaceProbes?: CapturedSurfaceProbe[]; + hoverProbe?: HoverProbeReport; }): CaptureVomObservationResult { return { text: input.rendered.text, @@ -84,5 +96,6 @@ export function projectRecordSafeObservation(input: { frames: projectFrames(input.frameDocuments), matchNodes: projectMatchNodes(input.frameDocuments), surfaceProbes: input.surfaceProbes, + ...(input.hoverProbe ? { hoverProbe: input.hoverProbe } : {}), }; } diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index dc60fe2..fc9968a 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -33,10 +33,25 @@ export type RpcErrorReason = | "restricted_tab_url" | "borrow_conflict" | "screenshot_capture_failed" + | "file_input_probe_failed" + | "file_input_not_activated" + | "set_file_input_failed" + | "upload_mechanism_unsupported" + | "file_drop_target_unavailable" + | "file_drop_failed" + | "download_capture_failed" + | "transfer_outcome_unknown" + | "transfer_timeout" | "cleanup_failed"; +export type TransferEffectState = "none" | "committed" | "unknown"; +export type TransferCleanupState = "complete" | "failed"; + export interface RpcErrorData { reason?: RpcErrorReason; + effect_state?: TransferEffectState; + phase?: string; + cleanup_state?: TransferCleanupState; [key: string]: unknown; } @@ -318,9 +333,14 @@ export interface SnapshotResult { export interface ObserveParams extends SnapshotParams { debug_surfaces?: boolean; + probe_hover?: boolean; } export interface ObserveResult extends SnapshotResult { + hover_probe?: { + performed: boolean; + revealed_content: boolean; + }; debug?: { surface_probes?: Array<{ trigger_backend_node_id: number; @@ -500,6 +520,53 @@ export interface SelectResult { dialogs?: JavaScriptDialogInfo[]; } +export interface UploadFile { + transfer_id: string; + name: string; + staged_path?: string; +} + +export type UploadMode = "input" | "drop"; + +export interface UploadParams { + session_id: string; + ref?: string; + selector?: string; + tab_id?: number; + files: UploadFile[]; + mode?: UploadMode; + timeout_ms?: number; +} + +export interface UploadResult { + tab_id: number; + used_ref?: string; + used_selector?: string; + file_names: string[]; +} + +export interface DownloadParams { + session_id: string; + ref?: string; + selector?: string; + tab_id?: number; + timeout_ms?: number; + browser_relative_dir?: string; + max_byte_size?: number; +} + +export interface DownloadResult { + tab_id: number; + used_ref?: string; + used_selector?: string; + suggested_filename: string; + byte_size: number; + mime?: string; + danger?: string; + browser_path?: string; + transfer_id?: string; +} + // -------------------------------------------------------------------------- // M9 tool payloads — evaluate / wait_for_navigation / wait_ms // -------------------------------------------------------------------------- diff --git a/apps/extension/wxt.config.ts b/apps/extension/wxt.config.ts index 6e272d9..ff4fc53 100644 --- a/apps/extension/wxt.config.ts +++ b/apps/extension/wxt.config.ts @@ -30,6 +30,7 @@ export default defineConfig({ permissions: [ "alarms", "debugger", + "downloads", "idle", "notifications", "tabs", diff --git a/crates/bsk-cli/Cargo.toml b/crates/bsk-cli/Cargo.toml index afc980f..e24ef88 100644 --- a/crates/bsk-cli/Cargo.toml +++ b/crates/bsk-cli/Cargo.toml @@ -53,6 +53,7 @@ flate2 = { workspace = true } tar = { workspace = true } zip = { workspace = true } sha2 = { workspace = true } +base64 = { workspace = true } [target.'cfg(unix)'.dependencies] nix = { workspace = true } @@ -63,6 +64,7 @@ windows-sys = { version = "0.59", features = [ "Win32_Foundation", "Win32_System_Threading", "Win32_Security", + "Win32_Storage_FileSystem", ] } [dev-dependencies] diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index 42bb452..357625c 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -1,327 +1,180 @@ --- name: browser-skill description: | - Use when the user asks to perform browser automation tasks against their - logged-in browser: visit and read pages, fill forms, scrape data, click - through a flow, regression-test a PR's UI, validate a deployed page. - Requires the bsk CLI installed and the browser-skill extension loaded. + Use when the user asks to automate their logged-in Chromium browser: visit + and read pages, fill forms, scrape data, click through flows, regression-test + a PR's UI, validate a deployed page, or operate a tab they identify. Requires + the bsk CLI and browser extension. --- # browser-skill -Drive the user's **real Chromium browser** (with their logins and cookies) through the `bsk` CLI. The extension opens an isolated **Agent Window** for automation; the user's normal windows stay protected unless you explicitly borrow a tab. +Drive the user's real Chromium browser through `bsk`. Automation runs in an isolated **Agent +Window** with the user's existing logins and cookies. User-window tabs remain protected unless they +are explicitly borrowed. -## When to use +Do not use this skill for tasks with no browser, for extension installation, or when the user only +wants instructions. Never extract credentials, cookies, tokens, or other secrets from pages. -- Open pages, read titles/text, scrape structured data from sites the user can already access -- Fill forms, click through multi-step flows, smoke-test a UI change -- Understand pages with `bsk snapshot` first; use `bsk get-html` or `bsk screenshot` only when the snapshot is insufficient -- Operate on a specific user tab they point you at (after `bsk tab borrow`) +## Required lifecycle -## When NOT to use +Every browser task owns a bounded session: -- Tasks with **no browser** involved (files, APIs, databases only) -- Installing or configuring the extension (point the user to setup docs instead) -- **Credential harvesting** — never run `bsk evaluate` on banking, SSO, or password-manager pages to extract tokens, cookies, or secrets -- Long-lived control of a user's personal login window — borrow only for the immediate step, then `bsk tab return` or end the session -- Replacing the user's manual browsing when they only wanted an explanation - -## Prerequisites - -1. `bsk` on `PATH` (Rust CLI from browser-skill) -2. browser-skill **extension** loaded in Chromium and connected (popup shows green, or an upgrade reminder — still connected; continue) -3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails - -## Mandatory workflow - -Every automation task **must** follow this lifecycle. Do **not** rely on idle timeouts (default session idle is 5 minutes). - -``` -1. bsk session start → capture the 4-letter session id printed on stdout -2. … every tool command … → always pass --session -3. bsk session stop → REQUIRED when done (even on error paths) +```text +1. bsk session start # retain the printed 4-letter session id +2. bsk ... --session # pass it to every session-scoped command +3. bsk session stop # always run on success and error paths ``` -Optional: `bsk session start --browser ` when multiple browsers are connected (`bsk browsers` / error output lists them). Add `--no-focus` to open the Agent Window in the background without stealing focus from the user's current window. +Do not rely on the idle timeout for cleanup. Stop the session as soon as the goal is met unless the +user explicitly asks to keep it open. Stopping also returns borrowed tabs. -Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**. +Any `bsk` command auto-starts the background services it needs; never manage the daemon by hand. +When multiple browsers are connected, use `bsk browsers` and start with +`bsk session start --browser `. Add `--no-focus` to that same start command when the +Agent Window does not need to interrupt the user's current work; it is not a flag on other commands. +Run `bsk doctor` when startup or transport problems persist after one retry. -## Stop when the goal is met +## Work toward one observable goal -Every task is a **bounded goal**, not open-ended browsing. The goal may come from the user's request, a recorded `trace.json`, or both. +- Derive a concrete success condition from the user's request or a supplied trace. +- Take the shortest purposeful path: observe, act, then make at most one observation to confirm an + ambiguous result. +- Once success is visible, do not click, refresh, navigate, switch tabs, or perform extra checks. +- If a human-only step appears or two attempts make no progress, request help instead of + brute-forcing. -1. **Define success first** — one concrete, observable condition derived from the user's words, `purpose`, or the last meaningful step in a trace (e.g. "form submitted", "item added to cart", "playback started"). -2. **Take the shortest path** — snapshot → act → at most one check. Do not wander, re-try unrelated actions, or stack exploratory steps. -3. **Stop as soon as success is reached** — run `bsk session stop ` immediately unless the user explicitly asked to keep the session open (e.g. "don't close yet", "keep browsing"). -4. **No post-success work** — once the goal is met, do not click, refresh, navigate, re-search, switch tabs, or "double-check" that it worked. Further verification is a new task. -5. **When blocked, pause — do not brute-force** — if the page requires human input (login, captcha, OTP, payment confirmation) or an action fails twice with no progress, call `bsk request-help` instead of retrying blindly. See **Ask the human for help** below. -6. **When unsure** — at most one extra `bsk snapshot`. If success looks met, stop. If not, ask the user; do not keep clicking. +With a trace, follow its semantic target information and values in order, but treat its refs as +record-local hints. Stop when its purpose or last meaningful effect is satisfied. A trace guides the +task; it does not expand the user's goal or authorize additional actions. -**With a trace:** replay steps in order using `target` role/name/tag and raw `value`/`selection` fields. After the last step (or when its `effect.navigated_to` / success hint is satisfied), apply rules 3–4 immediately. The trace guides execution; it does not extend control beyond the goal. +## Observe, act, observe -**Without a trace:** the user's request *is* the success condition. Satisfying it ends the task — same stop rules apply. +Use this default loop: -## Core interaction loop - -Write operations only affect tabs in the **Agent Window** (or tabs you **borrowed** into it). - -``` +```text bsk navigate --session -bsk observe --session → primary semantic VOM view; reveals hover/focus surfaces -bsk snapshot --session → static aria tree fallback when VOM is insufficient -bsk hover @e3 --session → reveal hover-triggered menus before re-observing/clicking -bsk click @e4 --session → or bsk fill, bsk select, bsk press -bsk observe --session → again after navigation / DOM change +bsk observe --session +bsk click|hover|fill|select|press ... --session +bsk observe --session # after navigation or a meaningful DOM change ``` -**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page. +Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also +make them stale. Observe again before the next interaction. -Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`). +An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. +The listed items are labels, not usable refs: hover the trigger, observe again, then act on the +revealed item's own ref. Do not click the trigger itself unless the user wants the trigger's action. +`[has-submenu]` and `[expanded]` mark the same kind of trigger without listing what it hides. -When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover --session `, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action. +`bsk observe` does not hover the page on its own. Reach for `--probe-hover` when a control you have +good reason to expect is absent **and** no marker points at a trigger — that combination is what a +CSS-only hover menu looks like from here. It hovers a bounded set of likely triggers, so it costs a +few seconds and touches the live page; once you know which element hides the menu, `bsk hover ` +is cheaper and more precise. -## Observation priority +Escalate page reading only as needed: -Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question: +1. `bsk observe` for normal semantic understanding, text, controls, and refs. +2. `bsk observe --probe-hover` once when an expected control is missing and no marker points at a + trigger. +3. `bsk snapshot` when a stricter static accessibility tree is more useful. +4. `bsk get-html` for exact markup or hidden metadata that semantic views cannot provide. +5. `bsk screenshot` for layout, styling, canvas, images, or requested visual evidence. -1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery -2. `bsk snapshot` — strict static accessibility tree fallback -3. `bsk get-html` — when hidden DOM, metadata, or markup details are required -4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab. +Do not start with raw HTML or screenshots merely to discover ordinary controls. When interaction is +needed, obtain a fresh observation before acting on screenshot or HTML findings. -Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page. +## Respect the Agent Window boundary -## Sandbox rules +Normal page writes affect only Agent Window tabs. To operate a user tab, first list it with +`bsk tab list --scope user --session `, then `bsk tab borrow `. Return it immediately +after the relevant step with `bsk tab return `; never invent a tab id or keep a personal tab +borrowed across unrelated work. -| Rule | Detail | -|------|--------| -| Agent Window | `bsk tab create`, `bsk navigate`, `bsk click`, etc. work on agent tabs by default | -| User tabs | Read-only until borrowed: `bsk tab list --session --scope user` then `bsk tab borrow --session ` | -| Return borrowed tabs | Call `bsk tab return --session ` when finished; unreturned tabs are **auto-returned** on `bsk session stop` | -| Writes off-agent | Commands that mutate the page fail if the tab is not in the Agent Window — borrow or create a tab first | +## Ask the human when needed -## Global flags +Use `bsk request-help` for login, captcha, OTP, payment confirmation, consent, or another step the +user must complete. Give a precise prompt and pass fresh `--target` refs/selectors when concrete +controls can be highlighted. Use completion criteria only when the page has a clear stable success +signal. -| Flag | Purpose | -|------|---------| -| `--json` | Machine-readable JSON on stdout (errors too) | -| `--quiet` | Suppress informational stderr | -| `-v` / `-vv` | More verbose logging | +The result `outcome` is one of `continued`, `completed`, `cancelled`, `timed_out`, or `disabled` +(`navigated` is deprecated — never treat navigation as a completion signal). Resume only after +`continued` or `completed`. Treat `cancelled` as rejection, and `timed_out` or `disabled` as a +blocker rather than a reason to retry. After control returns, run a fresh `bsk observe` before +reasoning about the page or using refs. -Auto-update is **on by default** — the background daemon upgrades `bsk` itself when a new release is available (postponed while a session is active). Set `BSK_AUTO_UPDATE=off` to disable it and upgrade manually with `bsk update`. +## Command inventory -Command-specific flags (timeouts, `--tab-id`, `--wait-until`, …): **`bsk --help`** +This list of names is complete. Never invent a command outside it; read +`bsk --help` for flags instead of guessing them. -## CLI command reference (one line each) - -Details and flags: **`bsk --help`** - -### Diagnostics - -| Command | Summary | -|---------|---------| -| `bsk status` | Connection health, connected browsers, active sessions | -| `bsk doctor` | Deep diagnostics and repair hints | -| `bsk browsers` | List connected browser instances (ids, labels, versions) | - -### Session - -| Command | Summary | -|---------|---------| -| `bsk session start` | Open Agent Window (`--width`/`--height` for initial size); prints **4-letter session id** | -| `bsk session start --no-focus` | Open Agent Window in the background without stealing focus | -| `bsk session stop ` | End session, close Agent Window, auto-return borrowed tabs | -| `bsk session stop --all` | Stop every active session | -| `bsk session list` | List active sessions | - -### Window (require `--session `) - -| Command | Summary | -|---------|---------| -| `bsk window resize` | Resize the Agent Window (`--width`, `--height`; 100..=7680 CSS px) | - -### Device emulation — `bsk emulate` (requires `--session `) - -Emulate a mobile device environment on the agent tab via CDP — viewport, User-Agent, and touch — to debug a page's mobile layout and behaviour: - -```bash -bsk emulate --session --device iphone-14 -bsk emulate --session --width 390 --height 844 --dpr 3 --mobile --ua "Mozilla/5.0 (iPhone…" --touch -bsk emulate --session --off +```text +session start|stop|list browsers status doctor update logs +navigate navigate-back navigate-forward reload wait-for-navigation wait-ms +observe snapshot get-html screenshot console network +click hover fill select press evaluate +tab list|create|close|select|borrow|return window resize emulate +upload download request-help record start|stop ``` -- Presets (`--device`): `iphone-14`, `iphone-14-pro-max`, `iphone-se`, `pixel-7`, `galaxy-s23`, `ipad-mini`, `galaxy-tab-s8`. Manual flags (`--width`/`--height`/`--dpr`/`--mobile`/`--ua`/`--accept-language`/`--touch`/`--max-touch-points`) also work without a preset, or override individual preset fields; `--no-mobile`/`--no-touch` turn a preset's mobile viewport / touch emulation back off. -- Repeated runs merge field by field onto the tab's current emulation state — only the flags you pass change. E.g. after `--device iphone-14`, `bsk emulate --session --width 390 --height 844` keeps the preset's dpr (`3`) and mobile viewport. (The extension remembers the state per tab; after an extension reload the next run applies only the fields it carries.) -- `--off` clears every override (viewport, UA, touch) and the remembered state, restoring the tab's real environment. -- Scope: overrides apply to **one tab only** (CDP per-target) and are **not inherited by new tabs** — re-run `bsk emulate` after opening or switching to another tab (default target is the session's active tab; `--tab-id` overrides). -- Emulation covers viewport/UA/touch only; it does not throttle the network, fake geolocation, or synthesise real touch-event streams. +Required flags that are easy to get wrong: -### Tabs (require `--session `) - -| Command | Summary | -|---------|---------| -| `bsk tab list` | List tabs (`--scope user\|agent\|all`, default `all`) | -| `bsk tab create` | New tab in Agent Window (`--url`, `--no-active`, `--index`) | -| `bsk tab close ` | Close an agent tab | -| `bsk tab select ` | Focus an agent tab | -| `bsk tab borrow ` | Move a user tab into the Agent Window | -| `bsk tab return ` | Return a borrowed tab to its original window | - -### Observation (require `--session` unless noted) - -| Command | Summary | -|---------|---------| -| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs | -| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces | -| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) | -| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) | - -### Console & network debugging (read-only; require `--session`) - -| Command | Summary | -|---------|---------| -| `bsk console` | Buffered page console messages, JS exceptions, and browser log entries (`--include-stack` for stack traces) | -| `bsk network` | Buffered network responses (status, method, URL, MIME/resource type) and failures (`net::ERR_*` reason) | - -Both capture from the moment the tab is attached and read a bounded per-tab buffer: `--since ` pages from a cursor (`next_since` in the result), `--limit` (default 50, max 200), `--max-text-chars` (default 1000, max 4096), `--tab-id` to target a non-active tab. Both are strictly read-only — they never intercept or modify traffic, and request/response headers, bodies, and timings are not captured. - -### Navigation - -| Command | Summary | -|---------|---------| -| `bsk navigate ` | Go to URL in agent tab (`--wait-until`, `--timeout`) | -| `bsk navigate-back` | History back one step | -| `bsk navigate-forward` | History forward one step | -| `bsk reload` | Reload current tab (`--hard` bypass cache) | - -(`bsk navigate back` / `bsk navigate forward` are equivalent subcommands.) - -### Interaction - -| Command | Summary | -|---------|---------| -| `bsk click ` | Click element (`--button`, `--click-count`, `--modifiers`) | -| `bsk hover ` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) | -| `bsk fill --value ` | Clear and type into input | -| `bsk select --value ` | Set `` option values by `value` attribute. Select(SelectArgs), + /// Upload files through a page file input or explicit drop target. + Upload(UploadArgs), + + /// Capture one browser download and write it to a local path. + Download(DownloadArgs), + /// Evaluate a JavaScript expression inside the Agent Window. Evaluate(EvaluateArgs), diff --git a/crates/bsk-cli/src/cli/observe.rs b/crates/bsk-cli/src/cli/observe.rs index bae34fe..e8ef4ee 100644 --- a/crates/bsk-cli/src/cli/observe.rs +++ b/crates/bsk-cli/src/cli/observe.rs @@ -33,6 +33,12 @@ pub struct ObserveArgs { /// Include conditional surface probe diagnostics in JSON output. #[arg(long = "debug-surfaces")] pub debug_surfaces: bool, + + /// Actively hover page controls to reveal hover-only menus and tooltips. + /// Costs seconds of wall clock and touches the live page, so it is off + /// unless a static observation looks like it is missing hover content. + #[arg(long = "probe-hover")] + pub probe_hover: bool, } pub fn dispatch(args: ObserveArgs, format: Format) -> Result<(), CliError> { @@ -47,6 +53,7 @@ fn run(sock: PathBuf, args: ObserveArgs, format: Format) -> Result<(), CliError> max_depth: args.max_depth, max_tokens: args.max_tokens, debug_surfaces: args.debug_surfaces, + probe_hover: args.probe_hover, }; let reply: ObserveResult = call(sock, params)?; match format { diff --git a/crates/bsk-cli/src/cli/render_error.rs b/crates/bsk-cli/src/cli/render_error.rs index db52f47..01f80dd 100644 --- a/crates/bsk-cli/src/cli/render_error.rs +++ b/crates/bsk-cli/src/cli/render_error.rs @@ -49,6 +49,15 @@ pub mod reason { pub const TAB_NOT_ACTIVE: &str = "tab_not_active"; pub const BORROW_CONFLICT: &str = "borrow_conflict"; pub const SCREENSHOT_CAPTURE_FAILED: &str = "screenshot_capture_failed"; + pub const FILE_INPUT_PROBE_FAILED: &str = "file_input_probe_failed"; + pub const FILE_INPUT_NOT_ACTIVATED: &str = "file_input_not_activated"; + pub const SET_FILE_INPUT_FAILED: &str = "set_file_input_failed"; + pub const UPLOAD_MECHANISM_UNSUPPORTED: &str = "upload_mechanism_unsupported"; + pub const FILE_DROP_TARGET_UNAVAILABLE: &str = "file_drop_target_unavailable"; + pub const FILE_DROP_FAILED: &str = "file_drop_failed"; + pub const DOWNLOAD_CAPTURE_FAILED: &str = "download_capture_failed"; + pub const TRANSFER_OUTCOME_UNKNOWN: &str = "transfer_outcome_unknown"; + pub const TRANSFER_TIMEOUT: &str = "transfer_timeout"; pub const SESSION_BUSY: &str = crate::rpc_reason::SESSION_BUSY; pub const RECORD_START_PAGE_UNREACHABLE: &str = "record_start_page_unreachable"; } @@ -197,6 +206,20 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend return base; }; match (code, reason) { + (_, reason::TRANSFER_OUTCOME_UNKNOWN) => RenderInfo { + summary: "the file transfer outcome could not be confirmed", + hint: Some( + "do not retry the transfer: the browser may already have applied it; inspect the page or stop safely", + ), + exit_code: base.exit_code, + }, + (ErrorCode::Timeout, reason::TRANSFER_TIMEOUT) => RenderInfo { + summary: "the file transfer timed out after browser dispatch", + hint: Some( + "do not retry when effect_state is unknown or committed; inspect the page before taking another action", + ), + exit_code: base.exit_code, + }, (ErrorCode::PermissionDenied, reason::ELEMENT_NOT_VISIBLE) => RenderInfo { summary: "target element has no visible geometry", hint: Some( @@ -280,6 +303,60 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend ), exit_code: base.exit_code, }, + (ErrorCode::Unsupported, reason::FILE_INPUT_NOT_ACTIVATED) => RenderInfo { + summary: "the upload trigger did not activate a file input", + hint: Some( + "the page may use a non-input picker such as showOpenFilePicker(); when effect_state is none, re-observe and use `--mode drop` once only for a reliably identified attachment-receiving area — otherwise use `bsk request-help` with the exact original local path", + ), + exit_code: base.exit_code, + }, + (ErrorCode::Timeout | ErrorCode::CdpFailed, reason::FILE_INPUT_PROBE_FAILED) => { + RenderInfo { + summary: "the browser upload transaction could not be established", + hint: Some( + "do not retry the same upload blindly; use `bsk request-help` when human help is available", + ), + exit_code: base.exit_code, + } + } + (ErrorCode::Timeout | ErrorCode::CdpFailed, reason::SET_FILE_INPUT_FAILED) => RenderInfo { + summary: "the browser could not attach the staged file to the input", + hint: Some( + "check that BrowserSkill has Chrome's 'Allow access to file URLs' permission; otherwise use `bsk request-help`", + ), + exit_code: base.exit_code, + }, + (ErrorCode::Unsupported, reason::UPLOAD_MECHANISM_UNSUPPORTED) => RenderInfo { + summary: "the browser does not support native file-drop upload", + hint: Some( + "do not retry the same drop; use a standard file-input target or `bsk request-help`", + ), + exit_code: base.exit_code, + }, + ( + ErrorCode::PermissionDenied | ErrorCode::Timeout | ErrorCode::CdpFailed, + reason::FILE_DROP_TARGET_UNAVAILABLE, + ) => RenderInfo { + summary: "the file-drop target could not be safely resolved", + hint: Some( + "rerun observe and select the attachment-receiving editor or drop zone itself; do not guess another target — no file was delivered when effect_state is none", + ), + exit_code: base.exit_code, + }, + (ErrorCode::Timeout | ErrorCode::CdpFailed, reason::FILE_DROP_FAILED) => RenderInfo { + summary: "the browser could not complete the native file drop", + hint: Some( + "do not retry when effect_state is unknown; observe the page first, otherwise use `bsk request-help`", + ), + exit_code: base.exit_code, + }, + (ErrorCode::CdpFailed, reason::DOWNLOAD_CAPTURE_FAILED) => RenderInfo { + summary: "the browser download could not be attributed or completed", + hint: Some( + "do not retry blindly; the target browser may not expose a reliable download intent signal", + ), + exit_code: base.exit_code, + }, _ => base, } } @@ -419,6 +496,64 @@ mod tests { assert_eq!(info.exit_code, 3); } + #[test] + fn file_transfer_reasons_render_actionable_fallbacks() { + let unsupported = serde_json::json!({ "reason": reason::FILE_INPUT_NOT_ACTIVATED }); + let info = info_for_error(ErrorCode::Unsupported, Some(&unsupported)); + assert_eq!( + info.summary, + "the upload trigger did not activate a file input" + ); + let hint = info.hint.unwrap(); + assert!(hint.contains("--mode drop")); + assert!(hint.contains("reliably identified attachment-receiving area")); + assert!(hint.contains("exact original local path")); + + let control = serde_json::json!({ "reason": reason::FILE_INPUT_PROBE_FAILED }); + let info = info_for_error(ErrorCode::Timeout, Some(&control)); + assert!( + info.summary + .contains("transaction could not be established") + ); + assert!(info.hint.unwrap().contains("do not retry")); + + let unsupported_drop = + serde_json::json!({ "reason": reason::UPLOAD_MECHANISM_UNSUPPORTED }); + let info = info_for_error(ErrorCode::Unsupported, Some(&unsupported_drop)); + assert_eq!( + info.summary, + "the browser does not support native file-drop upload" + ); + assert!(info.hint.unwrap().contains("standard file-input")); + + let unavailable_target = + serde_json::json!({ "reason": reason::FILE_DROP_TARGET_UNAVAILABLE }); + let info = info_for_error(ErrorCode::PermissionDenied, Some(&unavailable_target)); + assert_eq!( + info.summary, + "the file-drop target could not be safely resolved" + ); + assert!(info.hint.unwrap().contains("effect_state is none")); + + let failed_drop = serde_json::json!({ "reason": reason::FILE_DROP_FAILED }); + let info = info_for_error(ErrorCode::Timeout, Some(&failed_drop)); + assert!(info.summary.contains("native file drop")); + assert!(info.hint.unwrap().contains("effect_state is unknown")); + + let download = serde_json::json!({ "reason": reason::DOWNLOAD_CAPTURE_FAILED }); + let info = info_for_error(ErrorCode::CdpFailed, Some(&download)); + assert!(info.summary.contains("download could not be attributed")); + + let unknown = serde_json::json!({ "reason": reason::TRANSFER_OUTCOME_UNKNOWN }); + let info = info_for_error(ErrorCode::ProtocolError, Some(&unknown)); + assert!(info.summary.contains("outcome could not be confirmed")); + assert!(info.hint.unwrap().contains("do not retry")); + + let timeout = serde_json::json!({ "reason": reason::TRANSFER_TIMEOUT }); + let info = info_for_error(ErrorCode::Timeout, Some(&timeout)); + assert!(info.summary.contains("timed out after browser dispatch")); + } + #[test] fn ref_not_found_overrides_not_found_copy() { let data = serde_json::json!({ "reason": reason::REF_NOT_FOUND }); diff --git a/crates/bsk-cli/src/cli/upload.rs b/crates/bsk-cli/src/cli/upload.rs new file mode 100644 index 0000000..a602732 --- /dev/null +++ b/crates/bsk-cli/src/cli/upload.rs @@ -0,0 +1,205 @@ +//! `bsk upload` — stage caller-readable files and attach them to a page. + +use std::fs::File; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::Context; +use base64::Engine; +use bsk_protocol::Method; +use bsk_protocol::tools::{ + TransferBeginParams, TransferBeginResult, TransferChunkParams, TransferChunkResult, + TransferIdParams, TransferReadyResult, TransferReleaseResult, UploadFile, UploadMode, + UploadParams, UploadResult, +}; +use clap::{Args, ValueEnum}; + +use crate::cli::ensure_daemon::ensure_daemon; +use crate::cli::error::{CliError, Format}; +use crate::cli::interaction::split_target; +use crate::cli::navigate::parse_timeout_ms; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, ValueEnum)] +pub enum UploadModeArg { + #[default] + Input, + Drop, +} + +impl From for UploadMode { + fn from(value: UploadModeArg) -> Self { + match value { + UploadModeArg::Input => Self::Input, + UploadModeArg::Drop => Self::Drop, + } + } +} + +#[derive(Debug, Clone, Args)] +pub struct UploadArgs { + /// Snapshot ref (`@e3`) or CSS selector for the chooser trigger / drop target. + pub target: Option, + #[arg(long = "ref")] + pub ref_: Option, + #[arg(long = "selector")] + pub selector: Option, + /// Local file to upload. Repeat for a multiple-file input. + #[arg(long = "file", required = true)] + pub files: Vec, + /// Browser mechanism used to deliver the staged files. + #[arg(long, value_enum, default_value_t)] + pub mode: UploadModeArg, + #[arg(long)] + pub session: String, + #[arg(long = "tab-id")] + pub tab_id: Option, + #[arg(long, default_value = "2m", value_parser = parse_timeout_ms)] + pub timeout: u32, +} + +pub fn dispatch(args: UploadArgs, format: Format) -> Result<(), CliError> { + if args.files.len() > crate::daemon::file_transfer::MAX_UPLOAD_FILES { + return Err(CliError::Local(anyhow::anyhow!( + "upload accepts at most {} files", + crate::daemon::file_transfer::MAX_UPLOAD_FILES + ))); + } + let info = ensure_daemon().context("ensure daemon is running")?; + let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?; + let mut staged = Vec::new(); + for path in &args.files { + match stage_file(&info.sock_path, &args.session, path) { + Ok(file) => staged.push(file), + Err(err) => { + for (id, _) in &staged { + let _ = release(&info.sock_path, id); + } + return Err(err); + } + } + } + let params = UploadParams { + session_id: args.session, + ref_, + selector, + tab_id: args.tab_id, + files: staged + .iter() + .map(|(id, name)| UploadFile { + transfer_id: id.clone(), + name: name.clone(), + staged_path: None, + }) + .collect(), + mode: args.mode.into(), + timeout_ms: Some(args.timeout), + }; + let result = crate::cli::business_rpc::call::<_, UploadResult>( + info.sock_path.clone(), + "upload", + Method::ToolUpload, + Some(params), + ipc_timeout(args.timeout), + ); + // Once tool.upload is dispatched, staging ownership belongs to the + // session. A transport timeout cannot prove that Chrome did not attach + // the file, so releasing here could invalidate a late successful attach. + // Session teardown remains the single cleanup boundary after dispatch. + let reply = result?; + match format { + Format::Json => println!("{}", serde_json::to_string_pretty(&reply).unwrap()), + Format::Human => println!( + "upload ok tab={} files={}", + reply.tab_id, + reply.file_names.join(", ") + ), + } + Ok(()) +} + +fn stage_file(sock: &Path, session: &str, path: &PathBuf) -> Result<(String, String), CliError> { + let mut file = File::open(path) + .with_context(|| format!("open upload file {}", path.display())) + .map_err(CliError::Local)?; + let meta = file + .metadata() + .with_context(|| format!("read upload file metadata {}", path.display())) + .map_err(CliError::Local)?; + if !meta.is_file() { + return Err(CliError::Local(anyhow::anyhow!( + "upload source is not a regular file: {}", + path.display() + ))); + } + let name = path + .file_name() + .and_then(|v| v.to_str()) + .filter(|v| !v.is_empty()) + .ok_or_else(|| CliError::Local(anyhow::anyhow!("upload file has no valid basename")))? + .to_string(); + let begin: TransferBeginResult = crate::cli::business_rpc::call( + sock.to_path_buf(), + "transfer-begin", + Method::TransferBegin, + Some(TransferBeginParams { + session_id: session.to_string(), + name: name.clone(), + byte_size: meta.len(), + }), + Duration::from_secs(10), + )?; + let staged = (|| { + let mut offset = 0u64; + let mut buf = vec![0u8; begin.chunk_size as usize]; + loop { + let n = file.read(&mut buf).map_err(|e| CliError::Local(e.into()))?; + if n == 0 { + break; + } + let reply: TransferChunkResult = crate::cli::business_rpc::call( + sock.to_path_buf(), + "transfer-chunk", + Method::TransferChunk, + Some(TransferChunkParams { + transfer_id: begin.transfer_id.clone(), + offset, + data_base64: base64::engine::general_purpose::STANDARD.encode(&buf[..n]), + }), + Duration::from_secs(30), + )?; + offset = reply.next_offset; + } + let _: TransferReadyResult = crate::cli::business_rpc::call( + sock.to_path_buf(), + "transfer-finish", + Method::TransferFinish, + Some(TransferIdParams { + transfer_id: begin.transfer_id.clone(), + }), + Duration::from_secs(10), + )?; + Ok::<_, CliError>(()) + })(); + if let Err(err) = staged { + let _ = release(sock, &begin.transfer_id); + return Err(err); + } + Ok((begin.transfer_id, name)) +} + +fn release(sock: &Path, id: &str) -> Result { + crate::cli::business_rpc::call( + sock.to_path_buf(), + "transfer-release", + Method::TransferRelease, + Some(TransferIdParams { + transfer_id: id.to_string(), + }), + Duration::from_secs(5), + ) +} + +fn ipc_timeout(timeout_ms: u32) -> Duration { + Duration::from_millis(u64::from(timeout_ms) + 5_000) +} diff --git a/crates/bsk-cli/src/daemon/file_transfer.rs b/crates/bsk-cli/src/daemon/file_transfer.rs new file mode 100644 index 0000000..397e30b --- /dev/null +++ b/crates/bsk-cli/src/daemon/file_transfer.rs @@ -0,0 +1,728 @@ +//! Session-scoped, content-transparent staging for upload/download tools. +//! +//! The registry deliberately exposes no arbitrary-path operations. Upload +//! bytes arrive from the invoking CLI in bounded chunks; browser downloads +//! land in a daemon-minted directory and are read back by the invoking CLI. + +use std::collections::HashMap; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Component, Path, PathBuf}; +use std::sync::Mutex; + +use base64::Engine; +use bsk_protocol::tools::{ + TransferBeginParams, TransferBeginResult, TransferChunkParams, TransferChunkResult, + TransferIdParams, TransferReadyResult, TransferReleaseResult, +}; +use bsk_protocol::{ErrorCode, RpcError}; +use uuid::Uuid; + +use super::paths; + +pub const TRANSFER_CHUNK_SIZE: u32 = 512 * 1024; +pub const MAX_TRANSFER_BYTES: u64 = 512 * 1024 * 1024; +pub const MAX_UPLOAD_FILES: usize = 20; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Direction { + Upload, + Download, +} + +#[derive(Debug)] +struct Entry { + session_id: String, + direction: Direction, + path: PathBuf, + expected_size: Option, + written: u64, + ready: bool, +} + +/// Browser download path after the daemon has verified that it belongs to +/// the capability minted for this transfer. From this point onward the daemon +/// owns cleanup on every error path; unvalidated arbitrary paths are never +/// removed. +#[derive(Debug)] +struct ValidatedBrowserFile { + path: PathBuf, + parent: PathBuf, + cleanup_armed: bool, +} + +impl ValidatedBrowserFile { + fn validate(reported_path: &Path, transfer_id: &str) -> Result { + if !reported_path.is_absolute() { + return Err(permission( + "download path is outside its browser capability", + )); + } + let expected_parent = Path::new("BrowserSkill").join(transfer_id); + let parent = reported_path + .parent() + .ok_or_else(|| permission("download path has no parent directory"))?; + if !parent.ends_with(&expected_parent) { + return Err(permission( + "download escaped its browser capability directory", + )); + } + for path in [reported_path, parent] { + if fs::symlink_metadata(path) + .map_err(io_error)? + .file_type() + .is_symlink() + { + return Err(permission("download capability path contains a symlink")); + } + } + Ok(Self { + path: reported_path.to_path_buf(), + parent: parent.to_path_buf(), + cleanup_armed: true, + }) + } + + fn remove_source(mut self) -> Result<(), RpcError> { + remove_if_present(&self.path).map_err(io_error)?; + remove_dir_if_present(&self.parent).map_err(io_error)?; + self.cleanup_armed = false; + Ok(()) + } +} + +impl Drop for ValidatedBrowserFile { + fn drop(&mut self) { + if !self.cleanup_armed { + return; + } + let _ = remove_if_present(&self.path); + let _ = remove_dir_if_present(&self.parent); + } +} + +#[derive(Debug)] +pub struct DownloadStaging { + pub transfer_id: String, + pub browser_relative_dir: String, +} + +#[derive(Debug)] +pub struct TransferRegistry { + root: PathBuf, + initialized: Mutex, + entries: Mutex>, +} + +impl TransferRegistry { + pub fn new() -> anyhow::Result { + Ok(Self { + root: paths::bsk_home()?.join("run").join("transfers"), + initialized: Mutex::new(false), + entries: Mutex::new(HashMap::new()), + }) + } + + #[cfg(test)] + fn at_root(root: PathBuf) -> anyhow::Result { + let registry = Self { + root, + initialized: Mutex::new(false), + entries: Mutex::new(HashMap::new()), + }; + registry.ensure_root()?; + Ok(registry) + } + + pub fn initialize(&self) -> anyhow::Result<()> { + self.ensure_root() + } + + fn ensure_root(&self) -> anyhow::Result<()> { + let mut initialized = self.initialized.lock().unwrap(); + if *initialized { + return Ok(()); + } + if self.root.exists() { + // Transfers never survive a daemon; removing stale directories on + // startup gives crash cleanup without a second persistence model. + let _ = fs::remove_dir_all(&self.root); + } + fs::create_dir_all(&self.root)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&self.root, fs::Permissions::from_mode(0o700))?; + } + *initialized = true; + Ok(()) + } + + pub fn begin_upload(&self, p: TransferBeginParams) -> Result { + self.ensure_root() + .map_err(|err| protocol_error(err.to_string()))?; + let basename = safe_upload_basename(&p.name)?; + if p.byte_size > MAX_TRANSFER_BYTES { + return Err(invalid(format!( + "file size {} exceeds transfer limit {}", + p.byte_size, MAX_TRANSFER_BYTES + ))); + } + let mut entries = self.entries.lock().unwrap(); + let session_bytes: u64 = entries + .values() + .filter(|entry| entry.session_id == p.session_id) + .map(|entry| entry.expected_size.unwrap_or(entry.written)) + .sum(); + if session_bytes.saturating_add(p.byte_size) > MAX_TRANSFER_BYTES { + return Err(invalid(format!( + "session transfer staging exceeds limit {}", + MAX_TRANSFER_BYTES + ))); + } + let id = new_id(); + let dir = self.root.join(&id); + fs::create_dir(&dir).map_err(io_error)?; + set_private_dir(&dir).map_err(io_error)?; + let path = dir.join(basename); + let file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + .map_err(io_error)?; + set_private_file(&file).map_err(io_error)?; + entries.insert( + id.clone(), + Entry { + session_id: p.session_id, + direction: Direction::Upload, + path, + expected_size: Some(p.byte_size), + written: 0, + ready: false, + }, + ); + Ok(TransferBeginResult { + transfer_id: id, + chunk_size: TRANSFER_CHUNK_SIZE, + }) + } + + pub fn write_chunk(&self, p: TransferChunkParams) -> Result { + let bytes = base64::engine::general_purpose::STANDARD + .decode(p.data_base64) + .map_err(|e| invalid(format!("invalid transfer chunk: {e}")))?; + if bytes.len() > TRANSFER_CHUNK_SIZE as usize { + return Err(invalid("transfer chunk exceeds negotiated size")); + } + let mut entries = self.entries.lock().unwrap(); + let entry = entries + .get_mut(&p.transfer_id) + .ok_or_else(|| not_found("transfer not found"))?; + if entry.direction != Direction::Upload || entry.ready { + return Err(invalid("transfer is not writable")); + } + if p.offset != entry.written { + return Err(invalid(format!( + "non-sequential transfer offset: expected {}, got {}", + entry.written, p.offset + ))); + } + let next = entry.written.saturating_add(bytes.len() as u64); + if next > entry.expected_size.unwrap_or(MAX_TRANSFER_BYTES) { + return Err(invalid("transfer exceeds declared byte size")); + } + let mut file = OpenOptions::new() + .append(true) + .open(&entry.path) + .map_err(io_error)?; + file.write_all(&bytes).map_err(io_error)?; + entry.written = next; + Ok(TransferChunkResult { + next_offset: next, + eof: false, + data_base64: None, + }) + } + + pub fn finish_upload(&self, p: TransferIdParams) -> Result { + let mut entries = self.entries.lock().unwrap(); + let entry = entries + .get_mut(&p.transfer_id) + .ok_or_else(|| not_found("transfer not found"))?; + if entry.direction != Direction::Upload { + return Err(invalid("transfer is not an upload")); + } + if entry.written != entry.expected_size.unwrap_or(entry.written) { + return Err(invalid(format!( + "incomplete transfer: expected {} bytes, received {}", + entry.expected_size.unwrap_or(0), + entry.written + ))); + } + entry.ready = true; + Ok(TransferReadyResult { + transfer_id: p.transfer_id, + byte_size: entry.written, + }) + } + + pub fn resolve_uploads( + &self, + session_id: &str, + ids: &[String], + ) -> Result, RpcError> { + let entries = self.entries.lock().unwrap(); + ids.iter() + .map(|id| { + let entry = entries + .get(id) + .ok_or_else(|| not_found("upload transfer not found"))?; + if entry.session_id != session_id + || entry.direction != Direction::Upload + || !entry.ready + { + return Err(permission( + "upload transfer is outside this session or not ready", + )); + } + Ok(entry.path.clone()) + }) + .collect() + } + + pub fn begin_download(&self, session_id: &str) -> Result { + self.ensure_root() + .map_err(|err| protocol_error(err.to_string()))?; + let id = new_id(); + let dir = self.root.join(&id); + fs::create_dir(&dir).map_err(io_error)?; + set_private_dir(&dir).map_err(io_error)?; + self.entries.lock().unwrap().insert( + id.clone(), + Entry { + session_id: session_id.to_string(), + direction: Direction::Download, + path: dir.clone(), + expected_size: None, + written: 0, + ready: false, + }, + ); + Ok(DownloadStaging { + browser_relative_dir: format!("BrowserSkill/{id}"), + transfer_id: id, + }) + } + + pub fn import_download(&self, id: &str, reported_path: &Path) -> Result { + let mut entries = self.entries.lock().unwrap(); + let entry = entries + .get_mut(id) + .ok_or_else(|| not_found("download transfer not found"))?; + if entry.direction != Direction::Download { + return Err(permission("transfer is not a download capability")); + } + let browser_file = ValidatedBrowserFile::validate(reported_path, id)?; + let canonical = browser_file.path.canonicalize().map_err(io_error)?; + let meta = fs::metadata(&canonical).map_err(io_error)?; + if !meta.is_file() || meta.len() > MAX_TRANSFER_BYTES { + return Err(invalid( + "download is not a regular file or exceeds the transfer limit", + )); + } + + let destination = entry.path.join("payload"); + let source = File::open(&canonical).map_err(io_error)?; + let mut destination_file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&destination) + .map_err(io_error)?; + let copied = std::io::copy( + &mut source.take(MAX_TRANSFER_BYTES.saturating_add(1)), + &mut destination_file, + ) + .map_err(io_error)?; + if copied > MAX_TRANSFER_BYTES { + let _ = fs::remove_file(&destination); + return Err(invalid("download exceeds the transfer limit")); + } + entry.path = destination; + entry.written = copied; + entry.expected_size = Some(copied); + entry.ready = true; + browser_file.remove_source()?; + Ok(copied) + } + + pub fn read_chunk(&self, p: TransferChunkParams) -> Result { + let entries = self.entries.lock().unwrap(); + let entry = entries + .get(&p.transfer_id) + .ok_or_else(|| not_found("transfer not found"))?; + if entry.direction != Direction::Download || !entry.ready { + return Err(invalid("transfer is not readable")); + } + if p.offset > entry.written { + return Err(invalid("read offset is beyond transfer length")); + } + let mut file = File::open(&entry.path).map_err(io_error)?; + file.seek(SeekFrom::Start(p.offset)).map_err(io_error)?; + let mut buf = vec![0u8; TRANSFER_CHUNK_SIZE as usize]; + let count = file.read(&mut buf).map_err(io_error)?; + buf.truncate(count); + let next = p.offset + count as u64; + Ok(TransferChunkResult { + next_offset: next, + eof: next >= entry.written, + data_base64: Some(base64::engine::general_purpose::STANDARD.encode(buf)), + }) + } + + pub fn release(&self, p: TransferIdParams) -> TransferReleaseResult { + let entry = self.entries.lock().unwrap().remove(&p.transfer_id); + if let Some(entry) = entry { + let dir = if entry.path.is_dir() { + entry.path + } else { + entry.path.parent().unwrap_or(&self.root).to_path_buf() + }; + let _ = fs::remove_dir_all(dir); + TransferReleaseResult { released: true } + } else { + TransferReleaseResult { released: false } + } + } + + pub fn release_session(&self, session_id: &str) { + let ids: Vec = self + .entries + .lock() + .unwrap() + .iter() + .filter(|(_, entry)| entry.session_id == session_id) + .map(|(id, _)| id.clone()) + .collect(); + for id in ids { + self.release(TransferIdParams { transfer_id: id }); + } + } +} + +fn new_id() -> String { + format!("tr_{}", Uuid::new_v4().simple()) +} + +fn set_private_dir(path: &Path) -> std::io::Result<()> { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(path, fs::Permissions::from_mode(0o700))?; + } + Ok(()) +} + +fn set_private_file(file: &File) -> std::io::Result<()> { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + file.set_permissions(fs::Permissions::from_mode(0o600))?; + } + Ok(()) +} + +fn remove_if_present(path: &Path) -> std::io::Result<()> { + match fs::remove_file(path) { + Ok(()) => Ok(()), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(err) => Err(err), + } +} + +fn remove_dir_if_present(path: &Path) -> std::io::Result<()> { + match fs::remove_dir(path) { + Ok(()) => Ok(()), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(err) => Err(err), + } +} + +fn safe_upload_basename(name: &str) -> Result<&str, RpcError> { + if name.is_empty() || name.contains(['/', '\\', '\0']) { + return Err(invalid("upload name must be a safe basename")); + } + let mut components = Path::new(name).components(); + match (components.next(), components.next()) { + (Some(Component::Normal(_)), None) => Ok(name), + _ => Err(invalid("upload name must be a safe basename")), + } +} + +fn invalid(message: impl Into) -> RpcError { + RpcError { + code: ErrorCode::InvalidParams, + message: message.into(), + data: None, + } +} +fn not_found(message: impl Into) -> RpcError { + RpcError { + code: ErrorCode::NotFound, + message: message.into(), + data: None, + } +} +fn permission(message: impl Into) -> RpcError { + RpcError { + code: ErrorCode::PermissionDenied, + message: message.into(), + data: None, + } +} +fn io_error(err: std::io::Error) -> RpcError { + RpcError { + code: ErrorCode::ProtocolError, + message: err.to_string(), + data: None, + } +} + +fn protocol_error(message: impl Into) -> RpcError { + RpcError { + code: ErrorCode::ProtocolError, + message: message.into(), + data: None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn registry() -> (tempfile::TempDir, TransferRegistry) { + let temp = tempfile::tempdir().unwrap(); + let registry = TransferRegistry::at_root(temp.path().join("transfers")).unwrap(); + (temp, registry) + } + + #[test] + fn upload_is_not_resolvable_until_complete_or_from_another_session() { + let (_temp, registry) = registry(); + let begin = registry + .begin_upload(TransferBeginParams { + session_id: "s1".into(), + name: "image.png".into(), + byte_size: 3, + }) + .unwrap(); + assert!( + registry + .resolve_uploads("s1", std::slice::from_ref(&begin.transfer_id)) + .is_err() + ); + registry + .write_chunk(TransferChunkParams { + transfer_id: begin.transfer_id.clone(), + offset: 0, + data_base64: base64::engine::general_purpose::STANDARD.encode(b"abc"), + }) + .unwrap(); + registry + .finish_upload(TransferIdParams { + transfer_id: begin.transfer_id.clone(), + }) + .unwrap(); + assert!( + registry + .resolve_uploads("s2", std::slice::from_ref(&begin.transfer_id)) + .is_err() + ); + let [path] = registry + .resolve_uploads("s1", &[begin.transfer_id]) + .unwrap() + .try_into() + .unwrap(); + assert_eq!(path.file_name().unwrap(), "image.png"); + assert_eq!(fs::read(path).unwrap(), b"abc"); + } + + #[test] + fn upload_name_must_be_one_safe_path_component() { + let (_temp, registry) = registry(); + for name in ["", ".", "..", "../secret", "folder/file", "folder\\file"] { + let result = registry.begin_upload(TransferBeginParams { + session_id: "s1".into(), + name: name.into(), + byte_size: 0, + }); + assert!(result.is_err(), "accepted unsafe upload name"); + } + assert!(registry.entries.lock().unwrap().is_empty()); + assert!(fs::read_dir(®istry.root).unwrap().next().is_none()); + } + + #[test] + fn same_upload_names_are_isolated_by_transfer_directory() { + let (_temp, registry) = registry(); + let first = registry + .begin_upload(TransferBeginParams { + session_id: "s1".into(), + name: "image.png".into(), + byte_size: 0, + }) + .unwrap(); + let second = registry + .begin_upload(TransferBeginParams { + session_id: "s1".into(), + name: "image.png".into(), + byte_size: 0, + }) + .unwrap(); + let entries = registry.entries.lock().unwrap(); + let first_path = &entries[&first.transfer_id].path; + let second_path = &entries[&second.transfer_id].path; + assert_ne!(first_path.parent(), second_path.parent()); + assert_eq!(first_path.file_name().unwrap(), "image.png"); + assert_eq!(second_path.file_name().unwrap(), "image.png"); + } + + #[cfg(unix)] + #[test] + fn upload_staging_is_private() { + use std::os::unix::fs::PermissionsExt; + + let (_temp, registry) = registry(); + let begin = registry + .begin_upload(TransferBeginParams { + session_id: "s1".into(), + name: "private.png".into(), + byte_size: 0, + }) + .unwrap(); + let entries = registry.entries.lock().unwrap(); + let path = &entries[&begin.transfer_id].path; + assert_eq!( + fs::metadata(path.parent().unwrap()) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o700 + ); + assert_eq!( + fs::metadata(path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + + #[test] + fn download_must_import_from_its_browser_capability_directory() { + let (temp, registry) = registry(); + let staging = registry.begin_download("s1").unwrap(); + let outside = temp.path().join("outside.bin"); + fs::write(&outside, b"secret").unwrap(); + assert!( + registry + .import_download(&staging.transfer_id, &outside) + .is_err() + ); + assert_eq!(fs::read(&outside).unwrap(), b"secret"); + + let browser_dir = temp + .path() + .join("Downloads") + .join("BrowserSkill") + .join(&staging.transfer_id); + fs::create_dir_all(&browser_dir).unwrap(); + let inside = browser_dir.join("result.bin"); + fs::write(&inside, b"result").unwrap(); + assert_eq!( + registry + .import_download(&staging.transfer_id, &inside) + .unwrap(), + 6 + ); + assert!(!inside.exists()); + let chunk = registry + .read_chunk(TransferChunkParams { + transfer_id: staging.transfer_id, + offset: 0, + data_base64: String::new(), + }) + .unwrap(); + assert_eq!( + base64::engine::general_purpose::STANDARD + .decode(chunk.data_base64.unwrap()) + .unwrap(), + b"result" + ); + } + + #[test] + fn validated_oversized_browser_download_is_removed_on_rejection() { + let (temp, registry) = registry(); + let staging = registry.begin_download("s1").unwrap(); + let browser_dir = temp + .path() + .join("Downloads") + .join("BrowserSkill") + .join(&staging.transfer_id); + fs::create_dir_all(&browser_dir).unwrap(); + let inside = browser_dir.join("oversized.bin"); + File::create(&inside) + .unwrap() + .set_len(MAX_TRANSFER_BYTES + 1) + .unwrap(); + + assert!( + registry + .import_download(&staging.transfer_id, &inside) + .is_err() + ); + assert!(!inside.exists()); + assert!(!browser_dir.exists()); + } + + #[test] + fn releasing_a_session_removes_all_staging() { + let (_temp, registry) = registry(); + let first = registry.begin_download("s1").unwrap(); + let second = registry.begin_download("s2").unwrap(); + let first_directory = registry.root.join(&first.transfer_id); + let second_directory = registry.root.join(&second.transfer_id); + registry.release_session("s1"); + assert!(!first_directory.exists()); + assert!(second_directory.exists()); + } + + #[test] + fn upload_staging_is_bounded_across_a_session() { + let (_temp, registry) = registry(); + registry + .begin_upload(TransferBeginParams { + session_id: "s1".into(), + name: "large.bin".into(), + byte_size: MAX_TRANSFER_BYTES, + }) + .unwrap(); + assert!( + registry + .begin_upload(TransferBeginParams { + session_id: "s1".into(), + name: "one-more-byte.bin".into(), + byte_size: 1, + }) + .is_err() + ); + assert!( + registry + .begin_upload(TransferBeginParams { + session_id: "s2".into(), + name: "other-session.bin".into(), + byte_size: 1, + }) + .is_ok() + ); + } +} diff --git a/crates/bsk-cli/src/daemon/ipc.rs b/crates/bsk-cli/src/daemon/ipc.rs index 38d0046..186cade 100644 --- a/crates/bsk-cli/src/daemon/ipc.rs +++ b/crates/bsk-cli/src/daemon/ipc.rs @@ -27,7 +27,8 @@ use bsk_protocol::system::{ VersionSkewEntry, }; use bsk_protocol::tools::{ - HelpOutcome, RequestHelpResult, ReturnFailure, WaitMsParams, WaitMsResult, + DownloadParams, DownloadResult, HelpOutcome, RequestHelpResult, ReturnFailure, + TransferBeginParams, TransferIdParams, UploadParams, WaitMsParams, WaitMsResult, }; use bsk_protocol::{ CancelParams, CancelResult, ErrorCode, Method, PingResult, ResponseBody, RpcError, RpcId, @@ -61,6 +62,10 @@ pub type RpcHandler = Arc< >; const DEFAULT_RPC_TIMEOUT: Duration = Duration::from_secs(15); +// The upload transaction owns its operation deadline and may need a bounded +// cleanup before it can return a useful structured error. Keep only that +// transport alive slightly longer so it does not replace the result. +const EXTENSION_RESPONSE_GRACE: Duration = Duration::from_secs(2); /// Upper bound on `wait_for_browser_ms` accepted over IPC. const MAX_BROWSER_WAIT: Duration = Duration::from_secs(60); // `session.stop` fast-fails while another tool is active; this budget @@ -223,6 +228,11 @@ pub fn full_handler(status: DaemonStatus, state: Arc) -> RpcHandler Ok(v) => ResponseBody::Ok(v), Err(e) => ResponseBody::Err(e), }, + Method::TransferBegin => handle_transfer_begin(&state, params), + Method::TransferChunk => handle_transfer_chunk(&state, params), + Method::TransferFinish => handle_transfer_finish(&state, params), + Method::TransferRead => handle_transfer_read(&state, params), + Method::TransferRelease => handle_transfer_release(&state, params), Method::ToolTabList | Method::ToolTabCreate | Method::ToolTabClose @@ -246,6 +256,8 @@ pub fn full_handler(status: DaemonStatus, state: Arc) -> RpcHandler | Method::ToolFill | Method::ToolPress | Method::ToolSelect + | Method::ToolUpload + | Method::ToolDownload | Method::ToolEvaluate | Method::ToolWaitForNavigation | Method::ToolRequestHelp @@ -324,7 +336,7 @@ async fn handle_tool_dispatch( data: None, }); } - let timeout = match tool_dispatch_timeout(¶ms) { + let timeout = match tool_dispatch_transport_timeout(&method, ¶ms) { Ok(timeout) => timeout, Err(err) => return ResponseBody::Err(err), }; @@ -338,24 +350,180 @@ async fn handle_tool_dispatch( }); } }; + let mut params = params; + let mut download_transfer_id: Option = None; + if method == Method::ToolUpload { + let mut upload: UploadParams = match serde_json::from_value(params) { + Ok(v) => v, + Err(err) => return ResponseBody::Err(invalid_params(err.to_string())), + }; + if upload.files.is_empty() || upload.files.len() > super::file_transfer::MAX_UPLOAD_FILES { + return ResponseBody::Err(invalid_params(format!( + "upload requires 1..={} files", + super::file_transfer::MAX_UPLOAD_FILES + ))); + } + let ids: Vec = upload.files.iter().map(|f| f.transfer_id.clone()).collect(); + let paths = match state.transfers.resolve_uploads(&session_id.0, &ids) { + Ok(v) => v, + Err(err) => return ResponseBody::Err(err), + }; + for (file, path) in upload.files.iter_mut().zip(paths) { + file.staged_path = Some(path.to_string_lossy().into_owned()); + } + params = serde_json::to_value(upload).unwrap_or(Value::Null); + } else if method == Method::ToolDownload { + let mut download: DownloadParams = match serde_json::from_value(params) { + Ok(v) => v, + Err(err) => return ResponseBody::Err(invalid_params(err.to_string())), + }; + let staging = match state.transfers.begin_download(&session_id.0) { + Ok(v) => v, + Err(err) => return ResponseBody::Err(err), + }; + download.browser_relative_dir = Some(staging.browser_relative_dir); + download.max_byte_size = Some(super::file_transfer::MAX_TRANSFER_BYTES); + download_transfer_id = Some(staging.transfer_id); + params = serde_json::to_value(download).unwrap_or(Value::Null); + } let entry = inflight_guard.entry(); // `record_stop` must reach the extension while `record_await` holds the // serial busy lock — finishing the recording unblocks await. let outcome = if method == Method::ToolRecordStop { state .tool_queues - .dispatch_unlocked(&session_id, method, params, timeout, Some(entry)) + .dispatch_unlocked(&session_id, method.clone(), params, timeout, Some(entry)) .await } else { state .tool_queues - .dispatch(&session_id, method, params, timeout, Some(entry)) + .dispatch(&session_id, method.clone(), params, timeout, Some(entry)) .await }; drop(inflight_guard); match outcome { + Ok(v) if method == Method::ToolDownload => { + let id = download_transfer_id.expect("download transfer allocated"); + let mut result: DownloadResult = match serde_json::from_value(v) { + Ok(v) => v, + Err(err) => { + state + .transfers + .release(TransferIdParams { transfer_id: id }); + return ResponseBody::Err(RpcError { + code: ErrorCode::ProtocolError, + message: format!("invalid tool.download result: {err}"), + data: None, + }); + } + }; + let Some(path) = result.browser_path.take() else { + state + .transfers + .release(TransferIdParams { transfer_id: id }); + return ResponseBody::Err(RpcError { + code: ErrorCode::ProtocolError, + message: "tool.download returned no browser_path".into(), + data: None, + }); + }; + match state + .transfers + .import_download(&id, std::path::Path::new(&path)) + { + Ok(size) => { + result.byte_size = size; + result.transfer_id = Some(id); + ResponseBody::Ok(serde_json::to_value(result).unwrap_or(Value::Null)) + } + Err(err) => { + state + .transfers + .release(TransferIdParams { transfer_id: id }); + ResponseBody::Err(err) + } + } + } Ok(v) => ResponseBody::Ok(v), - Err(err) => ResponseBody::Err(err.into_rpc()), + Err(err) => { + if let Some(id) = download_transfer_id { + state + .transfers + .release(TransferIdParams { transfer_id: id }); + } + ResponseBody::Err(err.into_rpc()) + } + } +} + +fn handle_transfer_begin(state: &Arc, params: Value) -> ResponseBody { + let p: TransferBeginParams = match serde_json::from_value(params) { + Ok(v) => v, + Err(err) => return ResponseBody::Err(invalid_params(err.to_string())), + }; + if state + .sessions + .get(&SessionId(p.session_id.clone())) + .is_none() + { + return ResponseBody::Err(RpcError { + code: ErrorCode::NotFound, + message: format!("session {} unknown", p.session_id), + data: None, + }); + } + match state.transfers.begin_upload(p) { + Ok(v) => ResponseBody::Ok(serde_json::to_value(v).unwrap_or(Value::Null)), + Err(err) => ResponseBody::Err(err), + } +} + +fn handle_transfer_chunk(state: &Arc, params: Value) -> ResponseBody { + let p = match serde_json::from_value(params) { + Ok(v) => v, + Err(err) => return ResponseBody::Err(invalid_params(err.to_string())), + }; + match state.transfers.write_chunk(p) { + Ok(v) => ResponseBody::Ok(serde_json::to_value(v).unwrap_or(Value::Null)), + Err(err) => ResponseBody::Err(err), + } +} + +fn handle_transfer_finish(state: &Arc, params: Value) -> ResponseBody { + let p = match serde_json::from_value(params) { + Ok(v) => v, + Err(err) => return ResponseBody::Err(invalid_params(err.to_string())), + }; + match state.transfers.finish_upload(p) { + Ok(v) => ResponseBody::Ok(serde_json::to_value(v).unwrap_or(Value::Null)), + Err(err) => ResponseBody::Err(err), + } +} + +fn handle_transfer_read(state: &Arc, params: Value) -> ResponseBody { + let p = match serde_json::from_value(params) { + Ok(v) => v, + Err(err) => return ResponseBody::Err(invalid_params(err.to_string())), + }; + match state.transfers.read_chunk(p) { + Ok(v) => ResponseBody::Ok(serde_json::to_value(v).unwrap_or(Value::Null)), + Err(err) => ResponseBody::Err(err), + } +} + +fn handle_transfer_release(state: &Arc, params: Value) -> ResponseBody { + let p: TransferIdParams = match serde_json::from_value(params) { + Ok(v) => v, + Err(err) => return ResponseBody::Err(invalid_params(err.to_string())), + }; + ResponseBody::Ok(serde_json::to_value(state.transfers.release(p)).unwrap_or(Value::Null)) +} + +fn invalid_params(message: impl Into) -> RpcError { + RpcError { + code: ErrorCode::InvalidParams, + message: message.into(), + data: None, } } @@ -533,6 +701,16 @@ fn tool_dispatch_timeout(params: &Value) -> Result { Ok(Duration::from_millis(u64::from(ms))) } +fn tool_dispatch_transport_timeout(method: &Method, params: &Value) -> Result { + tool_dispatch_timeout(params).map(|timeout| { + if matches!(method, Method::ToolUpload | Method::ToolDownload) { + timeout.saturating_add(EXTENSION_RESPONSE_GRACE) + } else { + timeout + } + }) +} + // Local CLI-facing shapes. Intentionally distinct from // `bsk_protocol::tools::SessionStart*` (which describes the WS-facing // `tool.session_*` round-trip with the extension) so the two sides can @@ -798,6 +976,7 @@ async fn handle_session_stop( .await { Ok(stop) => { + state.transfers.release_session(&session_id.0); let result = CliSessionStopResult { stopped: vec![session_id.0], failed: Vec::new(), @@ -897,6 +1076,7 @@ async fn handle_session_stop_all( .await { Ok(stop) => { + state.transfers.release_session(&id.0); stopped.push(id.0); returned_tab_ids.extend(stop.returned_tab_ids); return_failures.extend(stop.return_failures); @@ -1329,6 +1509,30 @@ mod tests { assert_eq!(got, std::time::Duration::from_millis(300_000)); } + #[test] + fn extension_transport_outlives_the_operation_deadline() { + let params = serde_json::json!({ + "session_id": "abcd", + "timeout_ms": 60_000, + }); + let dispatch_timeout = + tool_dispatch_transport_timeout(&Method::ToolUpload, ¶ms).unwrap(); + + assert_eq!(dispatch_timeout, Duration::from_secs(62)); + } + + #[test] + fn extension_response_grace_does_not_change_other_tools() { + let params = serde_json::json!({ + "session_id": "abcd", + "timeout_ms": 60_000, + }); + let dispatch_timeout = + tool_dispatch_transport_timeout(&Method::ToolClick, ¶ms).unwrap(); + + assert_eq!(dispatch_timeout, Duration::from_secs(60)); + } + #[test] fn tool_dispatch_timeout_rejects_zero_timeout_ms() { let params = serde_json::json!({ diff --git a/crates/bsk-cli/src/daemon/mod.rs b/crates/bsk-cli/src/daemon/mod.rs index c639f73..3196062 100644 --- a/crates/bsk-cli/src/daemon/mod.rs +++ b/crates/bsk-cli/src/daemon/mod.rs @@ -3,6 +3,7 @@ pub mod abort; pub mod browsers; mod cancel_forward; +pub mod file_transfer; pub mod inflight; pub mod info; pub mod ipc; diff --git a/crates/bsk-cli/src/daemon/queue.rs b/crates/bsk-cli/src/daemon/queue.rs index 05f09e8..6b7366a 100644 --- a/crates/bsk-cli/src/daemon/queue.rs +++ b/crates/bsk-cli/src/daemon/queue.rs @@ -438,6 +438,7 @@ async fn dispatch_with_sender( lifecycle_cancel: Option, cancel_cleanup_timeout: Duration, ) -> Result { + let effect_aware_transfer = is_effect_aware_transfer(&method); let (respond_tx, respond_rx) = oneshot::channel(); let job = ToolJob { method, @@ -474,15 +475,21 @@ async fn dispatch_with_sender( mpsc::error::TrySendError::Closed(_) => DispatchError::QueueClosed, }); } - let mut response_timeout = timeout.saturating_add(Duration::from_secs(1)); - if lifecycle_cancellable { - // A lifecycle cancel can arrive at the original RPC deadline and - // intentionally keeps the worker alive while extension cleanup - // settles. Do not let this outer waiter abandon that reconciliation - // early, which could reopen a daemon session whose window is gone. - response_timeout = response_timeout.saturating_add(cancel_cleanup_timeout); - } - let waited = tokio::time::timeout(response_timeout, respond_rx).await; + // Lifecycle teardown and effect-aware transfers both keep the worker busy + // for bounded compensation after their original deadline. Keep the outer + // waiter alive for the same grace period so it cannot abandon reconciliation. + let response_grace = if lifecycle_cancellable || effect_aware_transfer { + cancel_cleanup_timeout + } else { + Duration::ZERO + }; + let waited = tokio::time::timeout( + timeout + .saturating_add(response_grace) + .saturating_add(Duration::from_secs(1)), + respond_rx, + ) + .await; match waited { Ok(Ok(Ok(v))) => Ok(v), Ok(Ok(Err(rpc))) => Err(DispatchError::Rpc(rpc)), @@ -636,44 +643,85 @@ async fn forward_one( .lifecycle_cancel .as_ref() .map(|_| &forward_lifecycle_cancel as &(dyn Fn() + Sync)); + let send_deadline_cancel = || { + client + .sink + .send(Frame::Request(RequestFrame { + id: format!("deadline-cancel-{rpc_id}"), + method: Method::Cancel, + params: Some(serde_json::json!({ "rpc_id": rpc_id })), + })) + .is_ok() + }; + let deadline_cancel: Option<&(dyn Fn() -> bool + Sync)> = + is_effect_aware_transfer(&job.method).then_some(&send_deadline_cancel); let waited = await_with_optional_cancel( job.timeout, job.cancel_cleanup_timeout, waiter, cancel_token.as_ref(), on_abort, + deadline_cancel, ) .await; let response = match waited { WaitOutcome::Response(resp) => resp, WaitOutcome::CancelledAfterResponse(resp) => { if job.method == Method::ToolSessionStop - && let ResponseBody::Ok(value) = resp.body + && let ResponseBody::Ok(value) = &resp.body { // Teardown crossed its final irreversible boundary before // the cancel landed. Commit the real extension result so the // daemon does not retain a session whose window is gone. - return Ok(value); + return Ok(value.clone()); } - // Cancellation wins the external verdict, but only after the - // extension's original RPC has settled. Preserve any non-cancel - // error so compensation failures remain explicit instead of - // being hidden behind a generic cancelled result. - if let ResponseBody::Err(err) = resp.body - && !matches!(err.code, ErrorCode::Cancelled | ErrorCode::UserAborted) - { - return Err(err); + // File transfer commits are irreversible. A late cancel cannot + // overwrite a confirmed success, and an unknown transfer effect + // must remain explicit so callers do not retry or release upload + // staging as though nothing happened. + if is_effect_aware_transfer(&job.method) { + match &resp.body { + ResponseBody::Ok(_) => resp, + ResponseBody::Err(err) + if transfer_effect(err) == Some("unknown") + || transfer_effect(err) == Some("committed") => + { + return Err(err.clone()); + } + ResponseBody::Err(err) + if !matches!(err.code, ErrorCode::Cancelled | ErrorCode::UserAborted) => + { + return Err(err.clone()); + } + ResponseBody::Err(_) => { + return Err(cancelled_error( + job.inflight.as_deref(), + "tool dispatch cancelled after extension cleanup", + )); + } + } + } else { + // For ordinary tools cancellation keeps the existing verdict, + // while non-cancel errors still expose compensation failures. + if let ResponseBody::Err(err) = resp.body + && !matches!(err.code, ErrorCode::Cancelled | ErrorCode::UserAborted) + { + return Err(err); + } + return Err(cancelled_error( + job.inflight.as_deref(), + "tool dispatch cancelled after extension cleanup", + )); } - return Err(cancelled_error( - job.inflight.as_deref(), - "tool dispatch cancelled after extension cleanup", - )); } WaitOutcome::TimedOutAfterResponse(resp) => match resp.body { - ResponseBody::Ok(value) if job.method == Method::ToolSessionStop => { + ResponseBody::Ok(value) + if job.method == Method::ToolSessionStop + || is_effect_aware_transfer(&job.method) => + { // The close crossed its irreversible boundary during the - // timeout cleanup grace. Reconcile daemon state instead of - // reopening a session whose window is already gone. + // timeout cleanup grace, or the transfer committed before its + // deadline cancel settled. Preserve the irreversible result. return Ok(value); } ResponseBody::Err(err) @@ -681,6 +729,9 @@ async fn forward_one( { return Err(err); } + ResponseBody::Err(err) if is_effect_aware_transfer(&job.method) => { + return Err(timed_out_transfer_error(&err)); + } _ => { return Err(RpcError { code: ErrorCode::Timeout, @@ -691,6 +742,14 @@ async fn forward_one( }, WaitOutcome::CleanupTimeout => { client.pending.lock().unwrap().cancel(&rpc_id); + if is_effect_aware_transfer(&job.method) { + return Err(unknown_transfer_error( + ErrorCode::Timeout, + "cancelled file transfer did not confirm its outcome before cleanup timed out", + "cleanup", + true, + )); + } return Err(RpcError { code: ErrorCode::Timeout, message: format!( @@ -700,8 +759,25 @@ async fn forward_one( data: Some(serde_json::json!({ "reason": "cancel_cleanup_timeout" })), }); } + WaitOutcome::TimeoutCleanupFailed => { + client.pending.lock().unwrap().cancel(&rpc_id); + return Err(unknown_transfer_error( + ErrorCode::Timeout, + "file transfer timed out and cleanup could not be confirmed", + "cleanup", + true, + )); + } WaitOutcome::WaiterClosed => { client.pending.lock().unwrap().cancel(&rpc_id); + if is_effect_aware_transfer(&job.method) { + return Err(unknown_transfer_error( + ErrorCode::ProtocolError, + "file transfer transport closed after dispatch; outcome is unknown", + "transport", + false, + )); + } return Err(RpcError { code: ErrorCode::ProtocolError, message: "transport closed mid-call".into(), @@ -710,6 +786,14 @@ async fn forward_one( } WaitOutcome::Timeout => { client.pending.lock().unwrap().cancel(&rpc_id); + if is_effect_aware_transfer(&job.method) { + return Err(unknown_transfer_error( + ErrorCode::Timeout, + "file transfer timed out after dispatch; outcome is unknown", + "transport", + false, + )); + } return Err(RpcError { code: ErrorCode::Timeout, message: format!("tool RPC timed out after {:?}", job.timeout), @@ -723,12 +807,65 @@ async fn forward_one( } } +fn is_effect_aware_transfer(method: &Method) -> bool { + matches!(method, Method::ToolUpload | Method::ToolDownload) +} + +fn transfer_effect(err: &RpcError) -> Option<&str> { + err.data.as_ref()?.get("effect_state")?.as_str() +} + +fn unknown_transfer_error( + code: ErrorCode, + message: impl Into, + phase: &str, + cleanup_failed: bool, +) -> RpcError { + let mut data = serde_json::json!({ + "reason": "transfer_outcome_unknown", + "effect_state": "unknown", + "phase": phase, + }); + if cleanup_failed { + data["cleanup_state"] = serde_json::json!("failed"); + } + RpcError { + code, + message: message.into(), + data: Some(data), + } +} + +fn timed_out_transfer_error(err: &RpcError) -> RpcError { + let data = err.data.as_ref(); + RpcError { + code: ErrorCode::Timeout, + message: "file transfer timed out after dispatch".into(), + data: Some(serde_json::json!({ + "reason": "transfer_timeout", + "effect_state": data + .and_then(|value| value.get("effect_state")) + .and_then(Value::as_str) + .unwrap_or("unknown"), + "phase": data + .and_then(|value| value.get("phase")) + .and_then(Value::as_str) + .unwrap_or("cleanup"), + "cleanup_state": data + .and_then(|value| value.get("cleanup_state")) + .and_then(Value::as_str) + .unwrap_or("complete"), + })), + } +} + #[derive(Debug)] enum WaitOutcome { Response(bsk_protocol::ResponseFrame), CancelledAfterResponse(bsk_protocol::ResponseFrame), TimedOutAfterResponse(bsk_protocol::ResponseFrame), CleanupTimeout, + TimeoutCleanupFailed, WaiterClosed, Timeout, } @@ -739,6 +876,7 @@ async fn await_with_optional_cancel( mut waiter: oneshot::Receiver, cancel: Option<&super::abort::AbortToken>, on_abort: Option<&(dyn Fn() + Sync)>, + on_deadline: Option<&(dyn Fn() -> bool + Sync)>, ) -> WaitOutcome { match cancel { Some(token) => { @@ -764,6 +902,16 @@ async fn await_with_optional_cancel( Err(_) => WaitOutcome::WaiterClosed, }, _ = &mut deadline => { + if let Some(send_cancel) = on_deadline { + return if send_cancel() { + match tokio::time::timeout(cleanup_timeout, &mut waiter).await { + Ok(Ok(resp)) => WaitOutcome::TimedOutAfterResponse(resp), + Ok(Err(_)) | Err(_) => WaitOutcome::TimeoutCleanupFailed, + } + } else { + WaitOutcome::TimeoutCleanupFailed + }; + } let Some(on_abort) = on_abort else { return WaitOutcome::Timeout; }; @@ -779,7 +927,16 @@ async fn await_with_optional_cancel( None => match tokio::time::timeout(timeout, &mut waiter).await { Ok(Ok(resp)) => WaitOutcome::Response(resp), Ok(Err(_)) => WaitOutcome::WaiterClosed, - Err(_) => WaitOutcome::Timeout, + Err(_) => match on_deadline { + Some(send_cancel) if send_cancel() => { + match tokio::time::timeout(cleanup_timeout, &mut waiter).await { + Ok(Ok(resp)) => WaitOutcome::TimedOutAfterResponse(resp), + Ok(Err(_)) | Err(_) => WaitOutcome::TimeoutCleanupFailed, + } + } + Some(_) => WaitOutcome::TimeoutCleanupFailed, + None => WaitOutcome::Timeout, + }, }, } } @@ -864,6 +1021,7 @@ mod await_with_optional_cancel_tests { rx, Some(&token), None, + None, ) .await; assert!( @@ -887,6 +1045,7 @@ mod await_with_optional_cancel_tests { rx, Some(&token), None, + None, ) .await; match outcome { @@ -907,6 +1066,7 @@ mod await_with_optional_cancel_tests { rx, None, None, + None, ) .await; assert!(matches!(outcome, WaitOutcome::Response(_))); @@ -924,6 +1084,7 @@ mod await_with_optional_cancel_tests { rx, Some(&token), None, + None, ) .await }); @@ -948,6 +1109,7 @@ mod await_with_optional_cancel_tests { rx, Some(&token), None, + None, ) .await; assert!(matches!(outcome, WaitOutcome::CleanupTimeout)); @@ -972,12 +1134,49 @@ mod await_with_optional_cancel_tests { rx, Some(&token), Some(&forward_abort), + None, ) .await; assert!(abort_forwarded.load(Ordering::SeqCst)); assert!(matches!(outcome, WaitOutcome::TimedOutAfterResponse(_))); } + + #[tokio::test] + async fn transfer_deadline_sends_cancel_and_waits_for_cleanup_response() { + let (tx, rx) = oneshot::channel(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(5)).await; + tx.send(dummy_response()).unwrap(); + }); + let send_cancel = || true; + let outcome = await_with_optional_cancel( + Duration::from_millis(1), + Duration::from_secs(1), + rx, + None, + None, + Some(&send_cancel), + ) + .await; + assert!(matches!(outcome, WaitOutcome::TimedOutAfterResponse(_))); + } + + #[tokio::test] + async fn transfer_deadline_is_unknown_when_cancel_cannot_be_sent() { + let (_tx, rx) = oneshot::channel(); + let send_cancel = || false; + let outcome = await_with_optional_cancel( + Duration::from_millis(1), + Duration::from_secs(1), + rx, + None, + None, + Some(&send_cancel), + ) + .await; + assert!(matches!(outcome, WaitOutcome::TimeoutCleanupFailed)); + } } #[cfg(test)] diff --git a/crates/bsk-cli/src/daemon/start.rs b/crates/bsk-cli/src/daemon/start.rs index 241eb85..cb6460c 100644 --- a/crates/bsk-cli/src/daemon/start.rs +++ b/crates/bsk-cli/src/daemon/start.rs @@ -225,6 +225,10 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> { .with_context(|| format!("bind IPC endpoint {}", sock_path.display()))?; let state = Arc::new(DaemonState::new(cfg.clone())); + state + .transfers + .initialize() + .context("initialize transfer staging")?; let session_idle_task = spawn_session_idle_reaper(Arc::clone(&state)); let browser_liveness_task = spawn_browser_liveness_reaper(Arc::clone(&state)); // Fired by the update check task after a successful auto-update: @@ -448,6 +452,7 @@ pub(crate) fn spawn_browser_liveness_reaper( for s in state.sessions.purge_browser(&client.id) { state.tool_queues.remove(&s.id); state.session_interrupts.drop_session(&s.id); + state.transfers.release_session(&s.id.0); debug!(session = %s.id, "purged session on browser liveness timeout"); } } @@ -485,7 +490,10 @@ pub(crate) fn spawn_session_idle_reaper(state: Arc) -> tokio::task: ) .await { - Ok(_) => info!(session = %session_id, "idle session stopped"), + Ok(_) => { + state.transfers.release_session(&session_id.0); + info!(session = %session_id, "idle session stopped"); + } Err(StopSessionError::SessionBusy | StopSessionError::Stopping) => { debug!(session = %session_id, "idle session still active; retrying later"); } @@ -496,6 +504,7 @@ pub(crate) fn spawn_session_idle_reaper(state: Arc) -> tokio::task: &state.session_interrupts, &session_id, ); + state.transfers.release_session(&session_id.0); } Err(err) => { warn!(session = %session_id, error = %err, "failed to stop idle session"); diff --git a/crates/bsk-cli/src/daemon/state.rs b/crates/bsk-cli/src/daemon/state.rs index ad3b48b..1001f3e 100644 --- a/crates/bsk-cli/src/daemon/state.rs +++ b/crates/bsk-cli/src/daemon/state.rs @@ -7,6 +7,7 @@ use tokio::task::JoinHandle; use super::abort::AbortRegistry; use super::browsers::BrowserRegistry; +use super::file_transfer::TransferRegistry; use super::inflight::ToolInflightRegistry; use super::ipc::IpcHandle; use super::queue::ToolQueueRegistry; @@ -51,6 +52,9 @@ pub struct DaemonState { /// `SessionRegistry` because the signal is a transient runtime /// control state. pub session_interrupts: Arc, + /// Operation-scoped local file staging. The extension only sees paths + /// minted here; agent-facing RPCs use opaque transfer ids. + pub transfers: Arc, } impl DaemonState { @@ -64,6 +68,7 @@ impl DaemonState { )); let abort_registry = Arc::new(AbortRegistry::new()); let session_interrupts = Arc::new(SessionInterruptRegistry::new()); + let transfers = Arc::new(TransferRegistry::new().expect("initialise transfer staging")); Self { config, browsers, @@ -72,6 +77,7 @@ impl DaemonState { abort_registry, tool_inflight, session_interrupts, + transfers, } } } diff --git a/crates/bsk-cli/src/daemon/ws.rs b/crates/bsk-cli/src/daemon/ws.rs index 435b0a9..e571f3d 100644 --- a/crates/bsk-cli/src/daemon/ws.rs +++ b/crates/bsk-cli/src/daemon/ws.rs @@ -295,6 +295,7 @@ async fn drive_connection( for session in state.sessions.purge_browser(&browser_id) { state.tool_queues.remove(&session.id); state.session_interrupts.drop_session(&session.id); + state.transfers.release_session(&session.id.0); debug!(session = %session.id, "purged stale session before browser reconnect"); } let (tx, mut rx) = mpsc::unbounded_channel::(); @@ -407,6 +408,7 @@ async fn drive_connection( for s in state.sessions.purge_browser(&browser_id) { state.tool_queues.remove(&s.id); state.session_interrupts.drop_session(&s.id); + state.transfers.release_session(&s.id.0); debug!(session = %s.id, "purged session on browser disconnect"); } } else { @@ -503,6 +505,7 @@ fn handle_session_window_closed( &state.session_interrupts, &session_id, ) { + state.transfers.release_session(&session_id.0); info!(session = %session_id, "session removed: user closed Agent Window"); } else { debug!(session = %session_id, "session.window_closed for unknown session id"); diff --git a/crates/bsk-cli/src/main.rs b/crates/bsk-cli/src/main.rs index 86c1451..6ac5468 100644 --- a/crates/bsk-cli/src/main.rs +++ b/crates/bsk-cli/src/main.rs @@ -97,6 +97,8 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> { Command::Fill(args) => cli::interaction::dispatch_fill(args, format), Command::Press(args) => cli::interaction::dispatch_press(args, format), Command::Select(args) => cli::interaction::dispatch_select(args, format), + Command::Upload(args) => cli::upload::dispatch(args, format), + Command::Download(args) => cli::download::dispatch(args, format), Command::Evaluate(args) => cli::evaluate::dispatch(args, format), Command::WaitForNavigation(args) => cli::waits::dispatch_wait_for_navigation(args, format), Command::WaitMs(args) => cli::waits::dispatch_wait_ms(args, format), diff --git a/crates/bsk-cli/tests/cli_parse.rs b/crates/bsk-cli/tests/cli_parse.rs index f8cb0f2..7a52f65 100644 --- a/crates/bsk-cli/tests/cli_parse.rs +++ b/crates/bsk-cli/tests/cli_parse.rs @@ -6,6 +6,7 @@ use bsk::cli::daemon::{DaemonCmd, parse_duration}; use bsk::cli::navigate::NavigateCmd; use bsk::cli::record::{RecordCmd, RecordSub}; use bsk::cli::session::{SessionCmd, SessionSub}; +use bsk::cli::upload::UploadModeArg; use bsk::{Cli, Command}; use clap::Parser; @@ -13,6 +14,54 @@ fn parse(args: &[&str]) -> Cli { Cli::try_parse_from(args).expect("clap parse should succeed") } +#[test] +fn parses_upload_modes() { + let cli = parse(&[ + "bsk", + "upload", + "@e1", + "--file", + "image.png", + "--session", + "s1", + ]); + let Command::Upload(default_args) = cli.command else { + panic!("expected upload subcommand"); + }; + assert_eq!(default_args.mode, UploadModeArg::Input); + + let cli = parse(&[ + "bsk", + "upload", + "@e1", + "--file", + "image.png", + "--session", + "s1", + "--mode", + "drop", + ]); + let Command::Upload(drop_args) = cli.command else { + panic!("expected upload subcommand"); + }; + assert_eq!(drop_args.mode, UploadModeArg::Drop); + + assert!( + Cli::try_parse_from([ + "bsk", + "upload", + "@e1", + "--file", + "image.png", + "--session", + "s1", + "--mode", + "auto", + ]) + .is_err() + ); +} + #[test] fn parses_daemon_start_with_defaults() { let cli = parse(&["bsk", "daemon", "start"]); @@ -213,6 +262,45 @@ fn rejects_zero_click_count() { ); } +#[test] +fn parses_upload_with_repeated_files() { + let cli = parse(&[ + "bsk", + "upload", + "@e3", + "--file", + "one.png", + "--file", + "two.png", + "--session", + "s1", + ]); + let Command::Upload(args) = cli.command else { + panic!("expected upload command"); + }; + assert_eq!(args.target.as_deref(), Some("@e3")); + assert_eq!(args.files.len(), 2); +} + +#[test] +fn parses_download_with_exact_output_policy() { + let cli = parse(&[ + "bsk", + "download", + "#export", + "--out", + "result.zip", + "--session", + "s1", + "--overwrite", + ]); + let Command::Download(args) = cli.command else { + panic!("expected download command"); + }; + assert_eq!(args.target.as_deref(), Some("#export")); + assert!(args.overwrite); +} + #[test] fn parses_record_start_with_browser_and_url() { let cli = parse(&[ diff --git a/crates/bsk-cli/tests/tools_ipc.rs b/crates/bsk-cli/tests/tools_ipc.rs index d40bb9c..4d8d2bc 100644 --- a/crates/bsk-cli/tests/tools_ipc.rs +++ b/crates/bsk-cli/tests/tools_ipc.rs @@ -478,6 +478,7 @@ async fn observe_returns_semantic_text_and_ref_count() { tab_id: 13, truncated: false, dialogs: vec![], + hover_probe: None, debug: None, }) .unwrap(), @@ -494,6 +495,7 @@ async fn observe_returns_semantic_text_and_ref_count() { max_depth: None, max_tokens: None, debug_surfaces: false, + probe_hover: false, }, ) .await diff --git a/crates/bsk-protocol/schema/tool_download_params.json b/crates/bsk-protocol/schema/tool_download_params.json new file mode 100644 index 0000000..3bfbba7 --- /dev/null +++ b/crates/bsk-protocol/schema/tool_download_params.json @@ -0,0 +1,56 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "DownloadParams", + "type": "object", + "required": [ + "session_id" + ], + "properties": { + "browser_relative_dir": { + "description": "Daemon-injected relative directory beneath Chrome's Downloads root.", + "type": [ + "string", + "null" + ] + }, + "max_byte_size": { + "description": "Daemon-injected authoritative transfer size limit. The extension uses it only for early cancellation; daemon import remains the final check.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, + "ref": { + "type": [ + "string", + "null" + ] + }, + "selector": { + "type": [ + "string", + "null" + ] + }, + "session_id": { + "type": "string" + }, + "tab_id": { + "type": [ + "integer", + "null" + ], + "format": "int64" + }, + "timeout_ms": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0.0 + } + } +} diff --git a/crates/bsk-protocol/schema/tool_download_result.json b/crates/bsk-protocol/schema/tool_download_result.json new file mode 100644 index 0000000..726ca50 --- /dev/null +++ b/crates/bsk-protocol/schema/tool_download_result.json @@ -0,0 +1,62 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "DownloadResult", + "type": "object", + "required": [ + "byte_size", + "suggested_filename", + "tab_id" + ], + "properties": { + "browser_path": { + "description": "Extension-internal completed Chrome download path, stripped by the daemon.", + "type": [ + "string", + "null" + ] + }, + "byte_size": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "danger": { + "type": [ + "string", + "null" + ] + }, + "mime": { + "type": [ + "string", + "null" + ] + }, + "suggested_filename": { + "type": "string" + }, + "tab_id": { + "type": "integer", + "format": "int64" + }, + "transfer_id": { + "description": "Opaque id returned by the daemon to the CLI.", + "type": [ + "string", + "null" + ] + }, + "used_ref": { + "type": [ + "string", + "null" + ] + }, + "used_selector": { + "type": [ + "string", + "null" + ] + } + } +} diff --git a/crates/bsk-protocol/schema/tool_observe_params.json b/crates/bsk-protocol/schema/tool_observe_params.json index 10e57a0..a89e9ec 100644 --- a/crates/bsk-protocol/schema/tool_observe_params.json +++ b/crates/bsk-protocol/schema/tool_observe_params.json @@ -1,7 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "title": "ObserveParams", - "description": "Parameters for `tool.observe` — produce a semantic VOM observation for one tab plus a fresh `@e` ref-store on the extension side.\n\nUnlike `tool.snapshot`, this path may run bounded perception probes such as hover-surface discovery. It must not submit input, click, navigate, or otherwise commit page state.", + "description": "Parameters for `tool.observe` — produce a semantic VOM observation for one tab plus a fresh `@e` ref-store on the extension side.\n\nUnlike `tool.snapshot`, this path may run bounded perception probes such as hover-surface discovery, but only when `probe_hover` asks for them. It must not submit input, click, navigate, or otherwise commit page state.", "type": "object", "required": [ "session_id" @@ -30,6 +30,11 @@ "format": "uint32", "minimum": 0.0 }, + "probe_hover": { + "description": "Opt in to active hover probing: the extension moves the real cursor over a bounded set of controls to discover hover-only menus and tooltips.\n\nOff by default. Probing costs seconds of wall clock, mutates the live page, and pays off only on pages whose content is genuinely hidden behind hover. Request it when a static observation looks like it is missing hover-revealed structure.", + "default": false, + "type": "boolean" + }, "session_id": { "type": "string" }, diff --git a/crates/bsk-protocol/schema/tool_observe_result.json b/crates/bsk-protocol/schema/tool_observe_result.json index 22ee410..3e89a2f 100644 --- a/crates/bsk-protocol/schema/tool_observe_result.json +++ b/crates/bsk-protocol/schema/tool_observe_result.json @@ -24,6 +24,17 @@ "$ref": "#/definitions/JavaScriptDialogInfo" } }, + "hover_probe": { + "description": "Present only when `probe_hover` was requested and probing actually ran.", + "anyOf": [ + { + "$ref": "#/definitions/HoverProbeReport" + }, + { + "type": "null" + } + ] + }, "ref_count": { "description": "Number of `@e` refs registered for this session by this observation. Equivalent to the size of the new ref-store map.", "type": "integer", @@ -46,6 +57,24 @@ } }, "definitions": { + "HoverProbeReport": { + "description": "Reports what active hover probing did during an observation, so the agent can judge how far the live page may have drifted from the returned text.", + "type": "object", + "required": [ + "performed", + "revealed_content" + ], + "properties": { + "performed": { + "description": "The real cursor was moved over page content during this observation.", + "type": "boolean" + }, + "revealed_content": { + "description": "Hovering surfaced content the static tree does not contain, so the page reacts to the cursor and may not have fully reverted.", + "type": "boolean" + } + } + }, "JavaScriptDialogHandledAction": { "description": "How the extension resolved the dialog so CDP could continue.", "type": "string", diff --git a/crates/bsk-protocol/schema/tool_upload_params.json b/crates/bsk-protocol/schema/tool_upload_params.json new file mode 100644 index 0000000..dd784b9 --- /dev/null +++ b/crates/bsk-protocol/schema/tool_upload_params.json @@ -0,0 +1,80 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "UploadParams", + "type": "object", + "required": [ + "files", + "session_id" + ], + "properties": { + "files": { + "type": "array", + "items": { + "$ref": "#/definitions/UploadFile" + } + }, + "mode": { + "$ref": "#/definitions/UploadMode" + }, + "ref": { + "type": [ + "string", + "null" + ] + }, + "selector": { + "type": [ + "string", + "null" + ] + }, + "session_id": { + "type": "string" + }, + "tab_id": { + "type": [ + "integer", + "null" + ], + "format": "int64" + }, + "timeout_ms": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0.0 + } + }, + "definitions": { + "UploadFile": { + "type": "object", + "required": [ + "name", + "transfer_id" + ], + "properties": { + "name": { + "type": "string" + }, + "staged_path": { + "type": [ + "string", + "null" + ] + }, + "transfer_id": { + "type": "string" + } + } + }, + "UploadMode": { + "type": "string", + "enum": [ + "input", + "drop" + ] + } + } +} diff --git a/crates/bsk-protocol/schema/tool_upload_result.json b/crates/bsk-protocol/schema/tool_upload_result.json new file mode 100644 index 0000000..192c10d --- /dev/null +++ b/crates/bsk-protocol/schema/tool_upload_result.json @@ -0,0 +1,33 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "UploadResult", + "type": "object", + "required": [ + "file_names", + "tab_id" + ], + "properties": { + "file_names": { + "type": "array", + "items": { + "type": "string" + } + }, + "tab_id": { + "type": "integer", + "format": "int64" + }, + "used_ref": { + "type": [ + "string", + "null" + ] + }, + "used_selector": { + "type": [ + "string", + "null" + ] + } + } +} diff --git a/crates/bsk-protocol/src/bin/dump-schema.rs b/crates/bsk-protocol/src/bin/dump-schema.rs index e1f4e6d..6599dce 100644 --- a/crates/bsk-protocol/src/bin/dump-schema.rs +++ b/crates/bsk-protocol/src/bin/dump-schema.rs @@ -85,6 +85,10 @@ fn main() { dump!(PressResult, "tool_press_result"); dump!(SelectParams, "tool_select_params"); dump!(SelectResult, "tool_select_result"); + dump!(UploadParams, "tool_upload_params"); + dump!(UploadResult, "tool_upload_result"); + dump!(DownloadParams, "tool_download_params"); + dump!(DownloadResult, "tool_download_result"); dump!(SnapshotParams, "tool_snapshot_params"); dump!(SnapshotResult, "tool_snapshot_result"); diff --git a/crates/bsk-protocol/src/method.rs b/crates/bsk-protocol/src/method.rs index 75f8b4d..740498c 100644 --- a/crates/bsk-protocol/src/method.rs +++ b/crates/bsk-protocol/src/method.rs @@ -78,6 +78,10 @@ pub enum Method { ToolPress, #[serde(rename = "tool.select")] ToolSelect, + #[serde(rename = "tool.upload")] + ToolUpload, + #[serde(rename = "tool.download")] + ToolDownload, #[serde(rename = "tool.snapshot")] ToolSnapshot, #[serde(rename = "tool.observe")] @@ -105,6 +109,17 @@ pub enum Method { #[serde(rename = "tool.record_await")] ToolRecordAwait, + #[serde(rename = "transfer.begin")] + TransferBegin, + #[serde(rename = "transfer.chunk")] + TransferChunk, + #[serde(rename = "transfer.finish")] + TransferFinish, + #[serde(rename = "transfer.read")] + TransferRead, + #[serde(rename = "transfer.release")] + TransferRelease, + #[serde(rename = "cancel")] Cancel, } @@ -156,6 +171,8 @@ impl Method { | Method::ToolFill | Method::ToolPress | Method::ToolSelect + | Method::ToolUpload + | Method::ToolDownload | Method::ToolEvaluate // May navigate via optional `url` and changes Agent Window // chrome; gate behind pending-interrupt like other writes. @@ -194,6 +211,11 @@ impl Method { | Method::SystemPing | Method::SystemStatus | Method::BrowserList + | Method::TransferBegin + | Method::TransferChunk + | Method::TransferFinish + | Method::TransferRead + | Method::TransferRelease | Method::Cancel => MethodEffect::ControlPlane, } } diff --git a/crates/bsk-protocol/src/tools/file_transfer.rs b/crates/bsk-protocol/src/tools/file_transfer.rs new file mode 100644 index 0000000..e2cd97c --- /dev/null +++ b/crates/bsk-protocol/src/tools/file_transfer.rs @@ -0,0 +1,213 @@ +//! Content-transparent file transfer contracts. +//! +//! Public tool calls carry opaque transfer ids. Filesystem paths are injected +//! by the daemon only on the daemon -> extension hop and are never accepted as +//! an agent-facing source or destination. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct UploadFile { + pub transfer_id: String, + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub staged_path: Option, +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum UploadMode { + #[default] + Input, + Drop, +} + +impl UploadMode { + fn is_input(&self) -> bool { + matches!(self, Self::Input) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct UploadParams { + pub session_id: String, + #[serde( + rename = "ref", + alias = "ref_", + default, + skip_serializing_if = "Option::is_none" + )] + pub ref_: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub selector: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tab_id: Option, + pub files: Vec, + #[serde(default, skip_serializing_if = "UploadMode::is_input")] + pub mode: UploadMode, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub timeout_ms: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct UploadResult { + pub tab_id: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub used_ref: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub used_selector: Option, + pub file_names: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct DownloadParams { + pub session_id: String, + #[serde( + rename = "ref", + alias = "ref_", + default, + skip_serializing_if = "Option::is_none" + )] + pub ref_: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub selector: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tab_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub timeout_ms: Option, + /// Daemon-injected relative directory beneath Chrome's Downloads root. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub browser_relative_dir: Option, + /// Daemon-injected authoritative transfer size limit. The extension uses + /// it only for early cancellation; daemon import remains the final check. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_byte_size: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct DownloadResult { + pub tab_id: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub used_ref: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub used_selector: Option, + pub suggested_filename: String, + pub byte_size: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mime: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub danger: Option, + /// Extension-internal completed Chrome download path, stripped by the daemon. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub browser_path: Option, + /// Opaque id returned by the daemon to the CLI. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub transfer_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct TransferBeginParams { + pub session_id: String, + pub name: String, + pub byte_size: u64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct TransferBeginResult { + pub transfer_id: String, + pub chunk_size: u32, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct TransferChunkParams { + pub transfer_id: String, + pub offset: u64, + pub data_base64: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct TransferChunkResult { + pub next_offset: u64, + #[serde(default)] + pub eof: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub data_base64: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct TransferIdParams { + pub transfer_id: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct TransferReadyResult { + pub transfer_id: String, + pub byte_size: u64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct TransferReleaseResult { + pub released: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn upload_agent_shape_omits_daemon_path_and_uses_ref_wire_name() { + let value = serde_json::to_value(UploadParams { + session_id: "s1".into(), + ref_: Some("@e3".into()), + selector: None, + tab_id: None, + files: vec![UploadFile { + transfer_id: "tr_1".into(), + name: "image.png".into(), + staged_path: None, + }], + mode: UploadMode::Input, + timeout_ms: None, + }) + .unwrap(); + assert_eq!(value["ref"], "@e3"); + assert!(value["files"][0].get("staged_path").is_none()); + assert!(value.get("mode").is_none()); + } + + #[test] + fn upload_drop_mode_is_explicit_on_the_wire() { + let value = serde_json::to_value(UploadParams { + session_id: "s1".into(), + ref_: Some("@e3".into()), + selector: None, + tab_id: None, + files: vec![UploadFile { + transfer_id: "tr_1".into(), + name: "image.png".into(), + staged_path: None, + }], + mode: UploadMode::Drop, + timeout_ms: None, + }) + .unwrap(); + assert_eq!(value["mode"], "drop"); + } + + #[test] + fn download_agent_shape_omits_daemon_path() { + let value = serde_json::to_value(DownloadParams { + session_id: "s1".into(), + ref_: None, + selector: Some("#export".into()), + tab_id: None, + timeout_ms: None, + browser_relative_dir: None, + max_byte_size: None, + }) + .unwrap(); + assert!(value.get("browser_relative_dir").is_none()); + assert!(value.get("max_byte_size").is_none()); + } +} diff --git a/crates/bsk-protocol/src/tools/mod.rs b/crates/bsk-protocol/src/tools/mod.rs index 88347b3..9e841bb 100644 --- a/crates/bsk-protocol/src/tools/mod.rs +++ b/crates/bsk-protocol/src/tools/mod.rs @@ -3,6 +3,7 @@ pub mod console; pub mod dialog; pub mod emulate; +pub mod file_transfer; pub mod human_loop; pub mod interaction; pub mod navigation; @@ -21,6 +22,7 @@ pub mod window; pub use console::*; pub use dialog::*; pub use emulate::*; +pub use file_transfer::*; pub use human_loop::*; pub use interaction::*; pub use navigation::*; diff --git a/crates/bsk-protocol/src/tools/observation.rs b/crates/bsk-protocol/src/tools/observation.rs index 0103ea5..9cbfc8d 100644 --- a/crates/bsk-protocol/src/tools/observation.rs +++ b/crates/bsk-protocol/src/tools/observation.rs @@ -58,8 +58,9 @@ pub struct SnapshotResult { /// for one tab plus a fresh `@e` ref-store on the extension side. /// /// Unlike `tool.snapshot`, this path may run bounded perception probes -/// such as hover-surface discovery. It must not submit input, click, -/// navigate, or otherwise commit page state. +/// such as hover-surface discovery, but only when `probe_hover` asks for +/// them. It must not submit input, click, navigate, or otherwise commit +/// page state. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct ObserveParams { pub session_id: String, @@ -77,6 +78,16 @@ pub struct ObserveParams { /// Include implementation diagnostics for conditional surface probes. #[serde(default)] pub debug_surfaces: bool, + /// Opt in to active hover probing: the extension moves the real cursor + /// over a bounded set of controls to discover hover-only menus and + /// tooltips. + /// + /// Off by default. Probing costs seconds of wall clock, mutates the live + /// page, and pays off only on pages whose content is genuinely hidden + /// behind hover. Request it when a static observation looks like it is + /// missing hover-revealed structure. + #[serde(default)] + pub probe_hover: bool, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] @@ -102,6 +113,17 @@ pub struct ObserveDebug { pub surface_probes: Vec, } +/// Reports what active hover probing did during an observation, so the agent +/// can judge how far the live page may have drifted from the returned text. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct HoverProbeReport { + /// The real cursor was moved over page content during this observation. + pub performed: bool, + /// Hovering surfaced content the static tree does not contain, so the page + /// reacts to the cursor and may not have fully reverted. + pub revealed_content: bool, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct ObserveResult { /// Semantic VOM observation text. Refs are rendered as `@e` so @@ -118,6 +140,9 @@ pub struct ObserveResult { pub truncated: bool, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub dialogs: Vec, + /// Present only when `probe_hover` was requested and probing actually ran. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hover_probe: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub debug: Option, } @@ -308,6 +333,7 @@ mod tests { tab_id: 42, truncated: false, dialogs: Vec::new(), + hover_probe: None, debug: None, }; let v = serde_json::to_value(&r).unwrap(); @@ -315,4 +341,30 @@ mod tests { let round: ObserveResult = serde_json::from_value(v).unwrap(); assert_eq!(round, r); } + + #[test] + fn observe_params_default_to_no_hover_probing() { + let params: ObserveParams = + serde_json::from_value(serde_json::json!({ "session_id": "s1" })).unwrap(); + assert!(!params.probe_hover); + } + + #[test] + fn observe_result_round_trips_with_hover_probe_report() { + let r = ObserveResult { + text: "@vom 1\n".into(), + ref_count: 0, + tab_id: 42, + truncated: false, + dialogs: Vec::new(), + hover_probe: Some(HoverProbeReport { + performed: true, + revealed_content: true, + }), + debug: None, + }; + let v = serde_json::to_value(&r).unwrap(); + let round: ObserveResult = serde_json::from_value(v).unwrap(); + assert_eq!(round, r); + } } diff --git a/docs/architecture.md b/docs/architecture.md index 2e02c57..e55f592 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -143,6 +143,15 @@ flowchart LR - No credential storage in bsk — cookies stay in the user's browser profile. - `evaluate` restricted to Agent Window tabs in sandbox mode. +### File-transfer boundary + +- The invoking agent/harness decides whether a transfer is authorized and supplies the task-local source or destination path. +- The CLI is the only component that reads an upload source or writes the final download destination. Before browser dispatch it owns rollback of partially staged uploads; after dispatch, ownership moves to the session because a transport timeout cannot prove that Chrome did not attach the file. Download output becomes visible through one atomic commit, and replacement is opt-in without a pre-delete window. The extension never receives either agent-facing path. +- The daemon is the authority for storage capabilities and limits. It issues opaque session-scoped transfer IDs, stages bounded chunks in a private runtime directory, and injects only private staged upload paths. For download it mints one relative Chrome directory capability. Only after validating the reported path, file type, symlink boundary, and authoritative byte limit does it take ownership of browser-file cleanup and import the bytes. +- The extension owns only the browser transaction. Every transfer resolves one `ResolvedActionTarget`. The default upload mechanism arms Chrome's chooser interception before clicking, then accepts either an exact `Page.fileChooserOpened` input node or an independent probe anchored in the trigger node's document; one verified input is committed with `DOM.setFileInputFiles`, while a non-input picker is rejected immediately. Explicit drop mode performs no click and never falls back to the chooser mechanism: after geometry resolution it temporarily excludes BrowserSkill's own overlay, verifies that the resolved drop zone still owns its local action point, and sends one native `dragEnter` / `dragOver` / `drop` transaction to that node's CDP target before restoring the overlay. OOPIF drops use target-local coordinates rather than top-level click coordinates. Download correlates exact-target CDP intent and `chrome.downloads` filename candidates in either arrival order, claims only one unique match, and never cancels an unclaimed candidate. +- Browser-side operations report `effect_state` (`none`, `committed`, or `unknown`), `phase`, and `cleanup_state`. Confirmed success wins over a late cancel; an unknown effect is preserved across timeout or transport loss and must not be retried blindly. A transfer deadline sends cancellation to the extension and keeps the session queue occupied for bounded compensation rather than abandoning an in-flight browser effect. +- Download staging is released after CLI commit. Upload staging remains until session teardown because the page may read an attached file only on a later form submission. Remaining staging is released on session stop/browser disconnect and on daemon startup after a crash. BrowserSkill does not inspect content or decide whether a transfer is appropriate. + ## Repository layout ``` diff --git a/packages/dsh-plugin-browserskill/package.json b/packages/dsh-plugin-browserskill/package.json index b1ebf92..427b435 100644 --- a/packages/dsh-plugin-browserskill/package.json +++ b/packages/dsh-plugin-browserskill/package.json @@ -1,7 +1,7 @@ { "name": "@wxg-prc-cpg/browser-skill-dsh-plugin", "description": "DeepSeek Harness tool plugin that exposes BrowserSkill browser automation (browser_* tools) to the model", - "version": "0.1.2", + "version": "0.2.0", "license": "MIT", "publishConfig": { "access": "public", diff --git a/packages/dsh-plugin-browserskill/skill/SKILL.md b/packages/dsh-plugin-browserskill/skill/SKILL.md index 0ec88ce..a2d9c4e 100644 --- a/packages/dsh-plugin-browserskill/skill/SKILL.md +++ b/packages/dsh-plugin-browserskill/skill/SKILL.md @@ -5,142 +5,99 @@ description: Browser automation through six injected domain tools. # browser-skill for DeepSeek Harness -Drive the user's real Chromium browser through the six structured domain tools provided by this -plugin. The browser keeps the user's existing logins and cookies while isolating automation in an -Agent Window. Every call must include the tool's `action` field; other fields depend on that action. +Drive the user's logged-in Chromium through this plugin's structured browser tools. Automation is +isolated in an Agent Window; user-window tabs remain protected unless explicitly borrowed. -## Tool availability +Loading this skill reveals six tools for the rest of the conversation: -Loading this skill reveals the browser tools for the rest of the conversation. All browser work -must use the injected tools directly. Do not invoke another process to control the browser. Direct -tool calls preserve session ownership, the live observation window, cancellation, attachments, and -cleanup. +- `browser_session` owns session lifecycle. +- `browser_page` handles navigation and lifecycle waits. +- `browser_inspect` reads semantic, visual, console, and network state. +- `browser_interact` performs normal page interactions. +- `browser_tabs` manages Agent Window tabs and temporary user-tab borrowing. +- `browser_assist` handles human help, window size, and device emulation. -The six tools are: - -- `browser_session`: start, stop, or list plugin-owned sessions. -- `browser_page`: navigate, back, forward, reload, or wait for navigation. -- `browser_inspect`: observe, snapshot, html, screenshot, console, or network. -- `browser_interact`: click, hover, fill, select, or press. -- `browser_tabs`: list, create, select, close, borrow, or return tabs. -- `browser_assist`: resize the Agent Window, emulate a device, or request help from the user. +Every call includes an `action`. Treat each loaded tool schema as authoritative for its actions and +parameters; do not guess fields. All browser work must use the injected tools directly so session +ownership, cancellation, attachments, observation UI, and cleanup remain intact. Do not invoke +another process to control the browser. ## Mandatory workflow -Every browser task follows this lifecycle: - -1. Call `browser_session` with `action: "start"` and retain its returned `sessionId`. -2. Use that session for every later call. Pass `session` explicitly when more than one exists. -3. Call `browser_session` with `action: "stop"` when the goal is reached or an error ends the task. - Treat cleanup like a finally path unless the user explicitly asks to keep the session open. - -Conceptual sequence: +Every task owns a bounded plugin session: ```text -browser_session({ action: "start", url: "https://example.com" }) -browser_inspect({ action: "observe", session: sessionId }) -browser_interact({ action: "click", session: sessionId, target: "@e4" }) -browser_inspect({ action: "observe", session: sessionId }) +browser_session({ action: "start", ... }) +... use the returned sessionId for browser work ... browser_session({ action: "stop", session: sessionId }) ``` -The plugin owns only sessions it starts. Never guess or reuse a session id from another program. -When `session` is omitted, most actions use the current owned session, but explicit ids are safer. +Pass the session explicitly when more than one exists. Never guess or reuse an id owned by another +program. Stop in a finally-style path on success and failure unless the user explicitly asks to keep +the session open. Stopping also returns borrowed tabs. -## Stop when the goal is met +## Work toward one observable goal -Browser work is bounded. Define an observable success condition, take the shortest path, and stop -as soon as success is visible. Do not refresh or click after success. If an action fails twice -without progress, pause instead of brute-forcing. If state is ambiguous, make at most one fresh -observation and then ask the user. +- Derive a concrete success condition from the user's request. +- Take the shortest purposeful path: observe, act, then make at most one observation to confirm an + ambiguous result. +- Once success is visible, do not click, refresh, navigate, switch tabs, or perform extra checks. +- If a human-only step appears or two attempts make no progress, request help instead of + brute-forcing. -## Observe and interact +## Observe, act, observe -Use `browser_inspect` with `action: "observe"` as the primary semantic page view. It returns roles, -states, text, and `@eN` refs. Use `action: "snapshot"` when a static accessibility tree is more useful. -Prefer a fresh ref over a raw CSS selector for interactions. +Use `browser_inspect` action `observe` as the primary semantic page view. It returns roles, states, +text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may +also become stale after large DOM changes, so observe again before the next interaction. -Refs invalidate after navigation and may become stale after large DOM changes. Observe again after -navigation or a meaningful DOM update before using another ref. If an observation identifies a -hover-triggered surface, hover the trigger, observe again, then interact with the revealed control. +Use `browser_interact` for click, hover, fill, select, and key actions. An observation marks a +hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are +labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref. +Do not click the trigger itself unless the user wants the trigger's action. -Use `browser_interact` as follows: +Escalate reading only as needed: -- `action: "click"`: requires `target`; optionally accepts `button` and `clickCount`. -- `action: "hover"`: requires `target`; optionally accepts `modifiers`, `settleMs`, and `timeoutMs`. -- `action: "fill"`: requires `target` and `value`; `noClear` appends instead of replacing. -- `action: "select"`: requires `target` and a non-empty `values` array. Values are option value - attributes, not necessarily the visible labels. -- `action: "press"`: requires `key`; optionally focus `target` first and set `holdMs`. +1. `observe` for normal understanding and interaction refs. +2. `snapshot` when a stricter static accessibility tree is more useful. +3. `html` for exact markup or hidden metadata that semantic views cannot provide. +4. `screenshot` for layout, styling, canvas, images, or requested visual evidence. -## Navigate and wait +Do not start with raw HTML or screenshots merely to discover ordinary controls. When interaction is +needed, obtain a fresh observation before acting on screenshot or HTML findings. -Use `browser_page` with these actions: +Use `browser_page` for purposeful navigation, history, reload, or a lifecycle wait. Avoid speculative +waits when no navigation is expected. After any page change, discard old refs and observe again. -- `navigate` requires `url` and optionally accepts `waitUntil` and `timeoutMs`. -- `back` and `forward` move one history entry and optionally accept lifecycle wait fields. -- `reload` refreshes the current tab; set `hard: true` only when bypassing cache is necessary. -- `wait` waits for an expected navigation after an interaction that did not already wait. +## Respect the Agent Window boundary -Avoid speculative waits when no navigation is expected. After any page change, discard old refs and -observe again. +Use `browser_tabs` to list returned tab ids before selecting, closing, borrowing, or returning tabs. +Borrow a user tab only for the immediate task, and return it as soon as that step is complete. Never +invent a tab id or keep a personal tab borrowed across unrelated work. -## Reading priority +## Ask the human when needed -Escalate page reading only as needed: +Use `browser_assist` action `request-help` for login, captcha, OTP, payment confirmation, consent, or +another step the user must complete. Give a precise prompt and highlight fresh targets when concrete +controls are involved. Use completion criteria only for a clear stable success signal. -1. `browser_inspect` with `action: "observe"` for normal understanding and refs. -2. The same tool with `action: "snapshot"` for a stricter static tree. -3. `action: "html"` for exact markup that semantic views cannot provide; keep `maxBytes` bounded and - use only a fresh ref when scoping a subtree. -4. `action: "screenshot"` for layout, styling, canvas, or genuinely visual evidence. A fresh `ref` - crops to an element; omit it for the visible tab. +Resume only after the user continues or the criteria complete. Treat cancellation as rejection and +timeout as a blocker rather than retrying. Observe again after control returns before reasoning about +the new state or using refs. -Do not start with raw HTML or a screenshot merely to discover ordinary controls. +The same tool can resize the Agent Window or emulate a device when the task requires visual or +responsive testing. Emulation is scoped to one tab. -## Tabs and the Agent Window boundary +## Debug and recover without wandering -Use `browser_tabs` with these actions: +Use `browser_inspect` console or network actions only for relevant, bounded, read-only diagnostics. +Continue from returned sequence cursors instead of rereading the same buffer. -- `list` returns visible tabs; `scope: "user"` finds a user tab before borrowing. -- `create` makes an Agent Window tab and optionally accepts `url`, `active`, and `index`. -- `select` focuses an Agent Window tab and requires a returned `tabId`. -- `close` closes an Agent Window tab and requires a returned `tabId`. -- `borrow` moves a user tab into the Agent Window and requires a listed user `tabId`. -- `return` restores a borrowed tab and requires that borrowed `tabId`. +- Stale ref: observe again and retry the intended action once. +- Unknown tab: list tabs instead of guessing. +- Unknown session: list owned sessions or start one; never try foreign ids. +- Timeout: inspect current state before deciding whether one longer purposeful wait is useful. +- Unrecoverable failure: report the blocker and stop the owned session. -Return borrowed tabs as soon as the immediate task finishes. Stopping the session is only fallback -cleanup. Never invent a tab id. - -## Human help and display controls - -Use `browser_assist` with `action: "request-help"` for login, captcha, OTP, payment confirmation, -consent, or another human-only step. `prompt` is required. Optional `targets` highlight fresh refs or -selectors. `completionCriteria` can detect explicit stable success through URL, selector, or text -conditions. After the user continues or criteria complete, observe again before reasoning about the -new state. - -Use `action: "resize"` with `width` and `height` to resize the outer Agent Window. Use -`action: "emulate"` with a device preset or explicit width and height; `mobile: true` also requires -dimensions. Use `off: true` alone to clear emulation. Emulation is per tab. - -## Read-only diagnostics - -Use `browser_inspect` with `action: "console"` for buffered logs and exceptions, and -`action: "network"` for response and failure metadata. Both accept `tabId`, `since`, `limit`, and -`maxTextChars`; continue with the returned sequence cursor rather than rereading the same entries. -Console additionally accepts `includeStack`. - -Arbitrary page script evaluation and interaction recording are intentionally unsupported. Do not -invent tool names or route around that limitation. - -## Error handling and final checklist - -On a stale ref, observe again and retry once. On an unknown tab, list tabs instead of guessing. On -an unknown session, list owned sessions or start one; never try foreign ids. On timeout, decide -whether one longer purposeful wait is justified, and never immediately repeat a failed history -action without first checking page state. - -Before finishing, verify that the observable goal is met or the blocker is clear, borrowed tabs are -returned when practical, the owned session is stopped, and no extra page action occurs after -success. +Arbitrary page-script evaluation and interaction recording are intentionally unsupported. Do not +invent tools or route around those limits. diff --git a/packages/dsh-plugin-browserskill/src/client/ObservationOverlay.tsx b/packages/dsh-plugin-browserskill/src/client/ObservationOverlay.tsx index edca403..a07fe98 100644 --- a/packages/dsh-plugin-browserskill/src/client/ObservationOverlay.tsx +++ b/packages/dsh-plugin-browserskill/src/client/ObservationOverlay.tsx @@ -15,6 +15,7 @@ import { RiArrowDownSLine, RiCheckLine, RiCloseCircleLine, + RiCloseLine, RiErrorWarningLine, RiPictureInPicture2Line, RiPushpinFill, @@ -28,6 +29,7 @@ const asIcon = (component: unknown): IconComponent => component as IconComponent const IconStop = asIcon(RiStopCircleLine); const IconPip = asIcon(RiPictureInPicture2Line); const IconDown = asIcon(RiArrowDownSLine); +const IconClose = asIcon(RiCloseLine); const IconWarn = asIcon(RiErrorWarningLine); const IconPin = asIcon(RiPushpinFill); const IconCloseSession = asIcon(RiCloseCircleLine); @@ -320,6 +322,7 @@ export function OverlayBody(props: { now: number; onPopOut?: (() => void) | undefined; onCollapse?: (() => void) | undefined; + onClosePip?: (() => void) | undefined; inPip: boolean; onHeaderPointerDown?: (event: ReactPointerEvent) => void; }) { @@ -333,6 +336,7 @@ export function OverlayBody(props: { now, onPopOut, onCollapse, + onClosePip, inPip, onHeaderPointerDown, } = props; @@ -389,6 +393,16 @@ export function OverlayBody(props: { ) : null} + {onClosePip !== undefined ? ( + + ) : null}
{displayUrl !== undefined ? ( @@ -572,6 +586,7 @@ export function ObservationOverlay({ store }: { store: ObservationClientStore }) : undefined } onCollapse={pipWindow === null ? () => setCollapsed(true) : undefined} + onClosePip={pipWindow !== null ? () => pipWindow.close() : undefined} onHeaderPointerDown={pipWindow === null ? beginMove : undefined} /> ); diff --git a/packages/dsh-plugin-browserskill/src/client/observation-sidebar.tsx b/packages/dsh-plugin-browserskill/src/client/observation-sidebar.tsx index 96095d8..a4fea16 100644 --- a/packages/dsh-plugin-browserskill/src/client/observation-sidebar.tsx +++ b/packages/dsh-plugin-browserskill/src/client/observation-sidebar.tsx @@ -149,6 +149,7 @@ export function ObservationSidebarTab({ now={now} inPip={pipWindow !== null} onPopOut={pipWindow === null && pipSupported ? () => popOut() : undefined} + onClosePip={pipWindow !== null ? () => pipWindow.close() : undefined} /> ); diff --git a/packages/dsh-plugin-browserskill/src/image.ts b/packages/dsh-plugin-browserskill/src/image.ts index a0b1801..8f5ed5e 100644 --- a/packages/dsh-plugin-browserskill/src/image.ts +++ b/packages/dsh-plugin-browserskill/src/image.ts @@ -42,7 +42,7 @@ interface LlmLike { * @param data - first bytes of the capture. * @returns the detected media type, or undefined when unrecognized. */ -function sniffImageMediaType(data: Uint8Array): string | undefined { +export function sniffImageMediaType(data: Uint8Array): string | undefined { if (data.length >= 4) { // PNG signature: 89 50 4E 47 (0D 0A 1A 0A) if (data[0] === 0x89 && data[1] === 0x50 && data[2] === 0x4e && data[3] === 0x47) { diff --git a/packages/dsh-plugin-browserskill/src/observation.ts b/packages/dsh-plugin-browserskill/src/observation.ts index 8705b24..5c280f2 100644 --- a/packages/dsh-plugin-browserskill/src/observation.ts +++ b/packages/dsh-plugin-browserskill/src/observation.ts @@ -17,6 +17,7 @@ import { readFile, unlink } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { Context } from "@deepseek-ai/cordis"; +import { sniffImageMediaType } from "./image"; import type { KeyedExecutor } from "./queue"; import { BskError, type BskRunner, parseBskJson, runWithSessionBusyRetry } from "./runner"; import type { SessionRegistry } from "./sessions"; @@ -467,7 +468,7 @@ export class ObservationService { const seq = (this.seqs.get(sessionId) ?? 0) + 1; this.seqs.set(sessionId, seq); const id = `obs-${sessionId}-${seq}`; - this.frames.set(id, { data, mediaType: "image/png" }); + this.frames.set(id, { data, mediaType: sniffImageMediaType(data) ?? "image/png" }); this.hashes.set(sessionId, hash); const ring = this.rings.get(sessionId) ?? []; ring.push(id); diff --git a/packages/dsh-plugin-browserskill/tests/client/observation-overlay.test.tsx b/packages/dsh-plugin-browserskill/tests/client/observation-overlay.test.tsx index 7290296..21cbdbd 100644 --- a/packages/dsh-plugin-browserskill/tests/client/observation-overlay.test.tsx +++ b/packages/dsh-plugin-browserskill/tests/client/observation-overlay.test.tsx @@ -298,6 +298,33 @@ describe("ObservationOverlay", () => { expect(close).toHaveBeenCalledTimes(1); }); + it("shows a close button inside the PiP window that closes it", async () => { + const h = makeHarness([BUSY]); + const pipDoc = document.implementation.createHTMLDocument("pip"); + const close = vi.fn(); + const listeners = new Map void)[]>(); + const requestWindow = vi.fn(async () => { + return { + document: pipDoc, + addEventListener: (name: string, fn: () => void) => { + listeners.set(name, [...(listeners.get(name) ?? []), fn]); + }, + close, + } as unknown as Window; + }); + (window as unknown as Record).documentPictureInPicture = { requestWindow }; + render(); + const popout = await screen.findByRole("button", { name: /Pop out/ }); + fireEvent.click(popout); + await waitFor(() => expect(pipDoc.body.textContent).toContain("s1")); + const closeBtn = pipDoc.body.querySelector( + 'button[aria-label="Close mini window"]', + ); + expect(closeBtn).not.toBeNull(); + closeBtn!.click(); + expect(close).toHaveBeenCalledTimes(1); + }); + it("renders the strip for two sessions and pins focus on click", async () => { const older: SessionObservation = { sessionId: "s1", diff --git a/packages/dsh-plugin-browserskill/tests/observation.test.ts b/packages/dsh-plugin-browserskill/tests/observation.test.ts index 98182e2..05b2eb2 100644 --- a/packages/dsh-plugin-browserskill/tests/observation.test.ts +++ b/packages/dsh-plugin-browserskill/tests/observation.test.ts @@ -335,6 +335,18 @@ describe("thumbnail cadence", () => { await waitFor(() => service.getState()[0]?.thumbnailAttachmentId === "obs-s1-1"); expect((await service.readThumbnail("obs-s1-1"))?.mediaType).toBe("image/png"); }); + + it("detects JPEG thumbnail bytes instead of hardcoding image/png", async () => { + const scheduler = fakeScheduler(); + const runner = fakeRunner({ + screenshotBytes: () => new Uint8Array([0xff, 0xd8, 0xff, 0xdb, 0x00, 0x01]), + }); + const { service } = setup({ runner, scheduler }); + service.addSession("s1"); + scheduler.runNext(); + await waitFor(() => service.getState()[0]?.thumbnailAttachmentId === "obs-s1-1"); + expect((await service.readThumbnail("obs-s1-1"))?.mediaType).toBe("image/jpeg"); + }); }); describe("observation traffic isolation", () => { diff --git a/packages/dsh-plugin-browserskill/tests/skill.test.ts b/packages/dsh-plugin-browserskill/tests/skill.test.ts index 6a6de21..c7a1ea3 100644 --- a/packages/dsh-plugin-browserskill/tests/skill.test.ts +++ b/packages/dsh-plugin-browserskill/tests/skill.test.ts @@ -47,7 +47,10 @@ describe("registerBskSkill", () => { expect(content).toContain("Mandatory workflow"); expect(content).toContain("Refs invalidate after navigation"); expect(content).toContain("evaluation and interaction recording are intentionally unsupported"); - expect(content.length).toBeGreaterThan(6_000); + // Keep the lazily injected instructions inside a bounded prompt budget, + // while the lower bound catches accidental truncation of the guidance. + expect(content.length).toBeGreaterThan(3_000); + expect(content.length).toBeLessThan(6_000); expect(skill.source).toBe("bundled"); // Source frontmatter is registration metadata and must not leak into the body. expect(content.startsWith("---")).toBe(false); diff --git a/packages/vom/src/__tests__/render.test.ts b/packages/vom/src/__tests__/render.test.ts index fa02ae5..a064df6 100644 --- a/packages/vom/src/__tests__/render.test.ts +++ b/packages/vom/src/__tests__/render.test.ts @@ -298,6 +298,34 @@ describe("renderVom single-layer page", () => { expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]); }); + it("renders through deeply nested transparent wrappers without exhausting the call stack", () => { + const wrapperCount = 10_000; + const nodes = [node({ id: 1, role: "RootWebArea", name: "Doc" })]; + for (let index = 0; index < wrapperCount; index += 1) { + nodes.push( + node({ + id: index + 2, + parentId: index + 1, + role: "generic", + }), + ); + } + nodes.push( + node({ + id: wrapperCount + 2, + parentId: wrapperCount + 1, + role: "button", + name: "Deep action", + tag: "button", + }), + ); + + const out = renderVom(scene(nodes)); + + expect(out.text).toContain('@e1 button "Deep action"'); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: wrapperCount + 2 }]); + }); + it("truncates when maxTokens is exceeded while keeping refs in sync with rendered text", () => { const out = renderVom( scene([ @@ -785,6 +813,55 @@ describe("renderVom single-layer page", () => { expect(out.text).not.toContain("北京市小客车指标调控管理信息系统]"); }); + it("derives duplicate-label context from indexed DOM ancestry without semantic proximity", () => { + const out = renderVom( + scene([ + node({ id: 1, role: "RootWebArea", name: "Doc", contextScopeId: "root" }), + node({ + id: 3, + parentId: 1, + role: "button", + name: "View", + tag: "button", + domParentId: 2, + domAncestorIds: [2, 1], + contextScopeId: "root", + }), + node({ + id: 5, + parentId: 1, + role: "button", + name: "View", + tag: "button", + domParentId: 4, + domAncestorIds: [4, 1], + contextScopeId: "root", + }), + node({ + id: 2, + parentId: 1, + role: "group", + name: "Project Alpha", + domParentId: 1, + domAncestorIds: [1], + contextScopeId: "root", + }), + node({ + id: 4, + parentId: 1, + role: "group", + name: "Project Beta", + domParentId: 1, + domAncestorIds: [1], + contextScopeId: "root", + }), + ]), + ); + + expect(out.text).toContain('@e1 button "View" [ctx: Project Alpha]'); + expect(out.text).toContain('@e2 button "View" [ctx: Project Beta]'); + }); + it("does not add low-value root context to duplicated brand links", () => { const out = renderVom( scene([ @@ -833,6 +910,34 @@ describe("renderVom single-layer page", () => { }); describe("renderVom double-layer page", () => { + it("collects blocking-layer descendants even when children precede parents in source order", () => { + const out = renderVom( + scene([ + node({ id: 1, role: "RootWebArea", name: "Doc" }), + node({ + id: 3, + parentId: 2, + role: "button", + name: "Close", + tag: "button", + paintOrder: 0, + }), + node({ + id: 2, + parentId: 1, + role: "dialog", + name: "Modal", + rect: { x: 0, y: 0, w: 1280, h: 720 }, + paintOrder: 10, + position: "fixed", + }), + ]), + ); + + expect(out.text).toContain('@e1 button "Close"'); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]); + }); + it("renders a blocking modal first and folds the base page into an occluded summary", () => { const out = renderVom( scene([ diff --git a/packages/vom/src/render.ts b/packages/vom/src/render.ts index b943dbd..c7c4164 100644 --- a/packages/vom/src/render.ts +++ b/packages/vom/src/render.ts @@ -421,6 +421,41 @@ function buildChildren(nodes: VomNode[]): Map { return children; } +const CONTEXT_LABEL_ROLES = new Set([ + "heading", + "article", + "section", + "group", + "region", + "row", + "form", +]); + +interface DomContextIndex { + nonBoundaryByScopeParent: Map>; + sourceOrderById: Map; +} + +function buildDomContextIndex(nodes: VomNode[]): DomContextIndex { + const nonBoundaryByScopeParent = new Map>(); + const sourceOrderById = new Map(); + for (const [sourceOrder, node] of nodes.entries()) { + sourceOrderById.set(node.id, sourceOrder); + if (!CONTEXT_LABEL_ROLES.has(normalizedRole(node)) || isContextBoundary(node)) continue; + const parentId = node.domParentId; + if (parentId === undefined || parentId === null) continue; + let byParent = nonBoundaryByScopeParent.get(node.contextScopeId); + if (!byParent) { + byParent = new Map(); + nonBoundaryByScopeParent.set(node.contextScopeId, byParent); + } + const siblings = byParent.get(parentId) ?? []; + siblings.push(node); + byParent.set(parentId, siblings); + } + return { nonBoundaryByScopeParent, sourceOrderById }; +} + function duplicateReferenceNames(nodes: VomNode[]): Set { const counts = new Map(); for (const node of nodes) { @@ -440,13 +475,7 @@ function needsHandleContext(nodeName: string, duplicateRefNames: Set): b } function contextLabel(node: VomNode, targetName: string): string | undefined { - if ( - !["heading", "article", "section", "group", "region", "row", "form"].includes( - normalizedRole(node), - ) - ) { - return undefined; - } + if (!CONTEXT_LABEL_ROLES.has(normalizedRole(node))) return undefined; return weakContextText( cleaned(node.name) ?? cleaned(node.text) ?? cleaned(node.nearbyText), targetName, @@ -607,19 +636,45 @@ function collectDomContext( ): ContextCandidate[] { const ancestorIds = node.domAncestorIds ?? []; if (ancestorIds.length === 0) return []; - const ancestors = new Set(ancestorIds); const contexts: ContextCandidate[] = []; + const ancestorRank = new Map(); + for (const [rank, ancestorId] of ancestorIds.entries()) { + if (!ancestorRank.has(ancestorId)) ancestorRank.set(ancestorId, rank); + } - for (const candidate of state.nodesById.values()) { + const candidates = new Map(); + const nonBoundaryByParent = state.domContextIndex.nonBoundaryByScopeParent.get( + node.contextScopeId, + ); + for (const ancestorId of ancestorRank.keys()) { + const boundary = state.nodesById.get(ancestorId); + const boundaryParentId = boundary?.domParentId; + if ( + boundary && + sharesContextScope(node, boundary) && + isContextBoundary(boundary) && + boundaryParentId !== undefined && + boundaryParentId !== null && + ancestorRank.has(boundaryParentId) + ) { + candidates.set(boundary.id, boundary); + } + for (const candidate of nonBoundaryByParent?.get(ancestorId) ?? []) { + candidates.set(candidate.id, candidate); + } + } + + const sourceOrderById = state.domContextIndex.sourceOrderById; + const orderedCandidates = [...candidates.values()].sort( + (a, b) => (sourceOrderById.get(a.id) ?? 0) - (sourceOrderById.get(b.id) ?? 0), + ); + for (const candidate of orderedCandidates) { if (candidate.id === node.id) continue; - if (!sharesContextScope(node, candidate)) continue; - if (isContextBoundary(candidate) && !ancestorIds.includes(candidate.id)) continue; const parentId = candidate.domParentId; - if (parentId === undefined || parentId === null || !ancestors.has(parentId)) continue; + if (parentId === undefined || parentId === null) continue; const label = contextLabel(candidate, nodeName); if (!label) continue; - const rank = ancestorIds.indexOf(parentId); - contexts.push({ text: label, source: "dom", order: rank < 0 ? contexts.length : rank }); + contexts.push({ text: label, source: "dom", order: ancestorRank.get(parentId) ?? 0 }); } return contexts; @@ -734,6 +789,7 @@ interface RenderState { children: Map; parentMap: Map; nodesById: Map; + domContextIndex: DomContextIndex; duplicateRefNames: Set; surfaceMap: Map; scopeMap: Map; @@ -768,20 +824,25 @@ function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: Ren } } -function renderTree( - children: Map, - parentId: number | null, +function pushRenderChildren( + stack: Array<{ node: VomNode; depth: number }>, + children: readonly VomNode[], depth: number, - state: RenderState, ): void { - if (state.stopped) return; + for (let index = children.length - 1; index >= 0; index -= 1) { + stack.push({ node: children[index], depth }); + } +} - const nodes = children.get(parentId) ?? []; - for (const node of nodes) { - if (state.stopped) return; +function renderTree(children: Map, state: RenderState): void { + const stack: Array<{ node: VomNode; depth: number }> = []; + pushRenderChildren(stack, children.get(null) ?? [], 1); + + while (stack.length > 0 && !state.stopped) { + const { node, depth } = stack.pop() as { node: VomNode; depth: number }; if (!shouldRender(node)) { - renderTree(children, node.id, depth, state); + pushRenderChildren(stack, children.get(node.id) ?? [], depth); continue; } @@ -798,7 +859,7 @@ function renderTree( if (nextTokens > state.maxTokens) { state.truncated = true; state.stopped = true; - return; + break; } state.lines.push(line); @@ -823,7 +884,7 @@ function renderTree( if ((ref && shouldSkipRedundantRefChildren(node)) || shouldSkipRedundantChildren(node, state)) { continue; } - renderTree(children, node.id, depth + 1, state); + pushRenderChildren(stack, children.get(node.id) ?? [], depth + 1); } } @@ -848,12 +909,13 @@ function renderNodes( children, parentMap: buildParentMap(nodes), nodesById: new Map(nodes.map((node) => [node.id, node])), + domContextIndex: buildDomContextIndex(nodes), duplicateRefNames: duplicateReferenceNames(nodes), surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])), scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])), }; - renderTree(children, null, 1, state); + renderTree(children, state); return state; } @@ -922,23 +984,33 @@ function paintLineageByFrame( return lineage; } +function cachedPaintLineageByFrame( + node: VomNode, + parentMap: Map, + nodesById: Map, + cache: Map>, +): Map { + const cached = cache.get(node.id); + if (cached) return cached; + const lineage = paintLineageByFrame(node, parentMap, nodesById); + cache.set(node.id, lineage); + return lineage; +} + /** Resolve both nodes to the nearest document where their paint order is comparable. */ function comparablePaintNodes( target: VomNode, blocker: VomNode, parentMap: Map, nodesById: Map, + lineageCache: Map>, ): { target: VomNode; blocker: VomNode } | null { if (target.frameId === blocker.frameId) return { target, blocker }; - const targetLineage = paintLineageByFrame(target, parentMap, nodesById); - let blockerInFrame: VomNode | undefined = blocker; - let guard = 0; - while (blockerInFrame && guard <= parentMap.size) { - const targetInFrame = targetLineage.get(blockerInFrame.frameId); + const targetLineage = cachedPaintLineageByFrame(target, parentMap, nodesById, lineageCache); + const blockerLineage = cachedPaintLineageByFrame(blocker, parentMap, nodesById, lineageCache); + for (const [frameId, blockerInFrame] of blockerLineage) { + const targetInFrame = targetLineage.get(frameId); if (targetInFrame) return { target: targetInFrame, blocker: blockerInFrame }; - const parentId: number | null = parentMap.get(blockerInFrame.id) ?? null; - blockerInFrame = parentId === null ? undefined : nodesById.get(parentId); - guard += 1; } return null; } @@ -949,12 +1021,13 @@ function isBlockedByRegion( parentMap: Map, nodesById: Map, viewportCoverage: number, + lineageCache: Map>, ): boolean { if (target.id === blocker.id) return false; if (isAncestorOf(parentMap, blocker.id, target.id)) return false; if (isAncestorOf(parentMap, target.id, blocker.id)) return false; if (!target.rect || !blocker.rect) return false; - const paintNodes = comparablePaintNodes(target, blocker, parentMap, nodesById); + const paintNodes = comparablePaintNodes(target, blocker, parentMap, nodesById, lineageCache); if (!paintNodes) return false; const points = interactionPoints(target.rect); @@ -979,17 +1052,22 @@ function isBlockedByRegion( return points.some(([x, y]) => rectContains(blocker.rect as Rect, x, y)); } -function collectDescendantsOfIds(nodes: VomNode[], roots: Set): Set { +function collectDescendants(nodes: VomNode[], roots: Set): Set { + const children = new Map(); + for (const node of nodes) { + if (node.parentId === null) continue; + const siblings = children.get(node.parentId) ?? []; + siblings.push(node.id); + children.set(node.parentId, siblings); + } + const included = new Set(roots); - let changed = true; - while (changed) { - changed = false; - for (const node of nodes) { - if (included.has(node.id)) continue; - if (node.parentId !== null && included.has(node.parentId)) { - included.add(node.id); - changed = true; - } + const queue = [...roots]; + for (let index = 0; index < queue.length; index += 1) { + for (const childId of children.get(queue[index]) ?? []) { + if (included.has(childId)) continue; + included.add(childId); + queue.push(childId); } } return included; @@ -998,6 +1076,7 @@ function collectDescendantsOfIds(nodes: VomNode[], roots: Set): Set [node.id, node])); + const lineageCache = new Map>(); const candidates = nodes .filter(isPositionedRegionCandidate) .map((node) => { @@ -1014,40 +1093,29 @@ function applyActiveRegionPolicy(nodes: VomNode[], scene: VomScene): VomNode[] { for (const target of nodes) { if (!isVomReferenceNode(target)) continue; const blocked = candidates.some((candidate) => - isBlockedByRegion(target, candidate.node, parentMap, nodesById, candidate.viewportCoverage), + isBlockedByRegion( + target, + candidate.node, + parentMap, + nodesById, + candidate.viewportCoverage, + lineageCache, + ), ); if (blocked) blockedRoots.add(target.id); } if (blockedRoots.size === 0) return nodes; - const blocked = collectDescendantsOfIds(nodes, blockedRoots); + const blocked = collectDescendants(nodes, blockedRoots); return nodes.filter((node) => !blocked.has(node.id)); } -function collectDescendantsOfMembers(nodes: VomNode[], members: Set): Set { - const included = new Set(members); - let changed = true; - - while (changed) { - changed = false; - for (const node of nodes) { - if (included.has(node.id)) continue; - if (node.parentId !== null && included.has(node.parentId)) { - included.add(node.id); - changed = true; - } - } - } - - return included; -} - function countRenderable(nodes: VomNode[]): number { return nodes.filter(shouldRender).length; } function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOptions): VomResult { - const included = collectDescendantsOfMembers(scene.nodes, layer.members); + const included = collectDescendants(scene.nodes, layer.members); const visibleNodes = scene.nodes.filter((node) => included.has(node.id)); const hiddenCount = countRenderable(scene.nodes.filter((node) => !included.has(node.id))); const header = [ diff --git a/scripts/release.mjs b/scripts/release.mjs new file mode 100755 index 0000000..f7a6e95 --- /dev/null +++ b/scripts/release.mjs @@ -0,0 +1,299 @@ +#!/usr/bin/env node +/** + * Unified release script for BrowserSkill. + * + * Bumps CLI, Extension, and DSH Plugin to the same version, updates + * CHANGELOG.md, commits, tags, and optionally pushes. + * + * Usage: + * node scripts/release.mjs # bump + commit + tag + * node scripts/release.mjs --dry-run # preview changes only + * node scripts/release.mjs --push # also push to origin + * node scripts/release.mjs --push --only ext # push only ext tag + * node scripts/release.mjs --push --only cli,dsh # push only cli + dsh tags + * node scripts/release.mjs --push-tags --only ext # push tag(s) for an already-committed release + * + * Options: + * --dry-run Show what would change without writing anything + * --push After commit+tag, push the branch and all release tags + * --push-tags Push release tags only (skip version bump; useful for staged rollout) + * --only Comma-separated subset of: cli, ext, dsh (default: all three) + * --no-commit Write files but skip git commit and tagging + */ + +import { execSync } from "node:child_process"; +import { readFileSync, writeFileSync } from "node:fs"; +import { parseArgs } from "node:util"; + +const TAG_PREFIX = { cli: "cli-v", ext: "ext-v", dsh: "dsh-plugin-v" }; +const ALL_COMPONENTS = ["cli", "ext", "dsh"]; + +const VERSION_FILES = { + cargo: "Cargo.toml", + extension: "apps/extension/package.json", + dshPlugin: "packages/dsh-plugin-browserskill/package.json", +}; + +// --------------------------------------------------------------------------- +// CLI argument parsing +// --------------------------------------------------------------------------- + +const { values, positionals } = parseArgs({ + allowPositionals: true, + options: { + "dry-run": { type: "boolean", default: false }, + push: { type: "boolean", default: false }, + "push-tags": { type: "boolean", default: false }, + "no-commit": { type: "boolean", default: false }, + only: { type: "string", default: "" }, + help: { type: "boolean", short: "h", default: false }, + }, +}); + +if (values.help) { + console.log( + readFileSync(new URL(import.meta.url), "utf8").match(/\/\*\*([\s\S]*?)\*\//)?.[1] ?? "", + ); + process.exit(0); +} + +const dryRun = values["dry-run"]; +const pushAfter = values.push; +const pushTagsOnly = values["push-tags"]; +const noCommit = values["no-commit"]; +const onlyRaw = values.only; +const components = onlyRaw ? onlyRaw.split(",").map((s) => s.trim().toLowerCase()) : ALL_COMPONENTS; + +for (const c of components) { + if (!ALL_COMPONENTS.includes(c)) { + console.error(`Unknown component "${c}". Valid: ${ALL_COMPONENTS.join(", ")}`); + process.exit(1); + } +} + +// --------------------------------------------------------------------------- +// Push-tags-only mode (no version bump) +// --------------------------------------------------------------------------- + +if (pushTagsOnly) { + const currentVersion = readCurrentVersions(); + const ver = positionals[0] ?? currentVersion.cargo; + for (const c of components) { + const tag = `${TAG_PREFIX[c]}${ver}`; + console.log(` pushing tag ${tag}`); + if (!dryRun) { + run(`git push origin ${tag}`); + } + } + console.log("\nDone."); + process.exit(0); +} + +// --------------------------------------------------------------------------- +// Normal mode — require version argument +// --------------------------------------------------------------------------- + +const newVersion = positionals[0]; +if (!newVersion) { + console.error( + "Usage: node scripts/release.mjs [--dry-run] [--push] [--only cli,ext,dsh]", + ); + process.exit(1); +} + +if (!/^\d+\.\d+\.\d+(-[\w.]+)?$/.test(newVersion)) { + console.error(`Invalid semver: "${newVersion}"`); + process.exit(1); +} + +// --------------------------------------------------------------------------- +// Read current versions +// --------------------------------------------------------------------------- + +function readCurrentVersions() { + const cargo = readFileSync(VERSION_FILES.cargo, "utf8"); + const cargoVer = + cargo.match(/^\[workspace\.package\]\s*\nversion\s*=\s*"([^"]+)"/m)?.[1] ?? + cargo.match(/^version\s*=\s*"([^"]+)"/m)?.[1]; + + const extPkg = JSON.parse(readFileSync(VERSION_FILES.extension, "utf8")); + const dshPkg = JSON.parse(readFileSync(VERSION_FILES.dshPlugin, "utf8")); + + return { cargo: cargoVer, extension: extPkg.version, dshPlugin: dshPkg.version }; +} + +const current = readCurrentVersions(); + +console.log("Current versions:"); +console.log(` CLI/Daemon (Cargo.toml): ${current.cargo}`); +console.log(` Extension (package.json): ${current.extension}`); +console.log(` DSH Plugin (package.json): ${current.dshPlugin}`); +console.log(`\nBumping all → ${newVersion}`); +if (dryRun) console.log(" (dry-run mode — no files will be written)\n"); + +// --------------------------------------------------------------------------- +// Bump Cargo.toml +// --------------------------------------------------------------------------- + +function bumpCargo(version) { + let content = readFileSync(VERSION_FILES.cargo, "utf8"); + + // workspace.package version + content = content.replace( + /^(\[workspace\.package\]\s*\nversion\s*=\s*")([^"]+)(")/m, + `$1${version}$3`, + ); + + // workspace.dependencies.bsk-protocol version (for cargo publish) + content = content.replace( + /(bsk-protocol\s*=\s*\{\s*version\s*=\s*")([^"]+)(")/, + `$1${version}$3`, + ); + + return content; +} + +function bumpPackageJson(filePath, version) { + const pkg = JSON.parse(readFileSync(filePath, "utf8")); + pkg.version = version; + return JSON.stringify(pkg, null, 2) + "\n"; +} + +// --------------------------------------------------------------------------- +// Update CHANGELOG.md +// --------------------------------------------------------------------------- + +function updateChangelog(version) { + const changelogPath = "CHANGELOG.md"; + let content; + try { + content = readFileSync(changelogPath, "utf8"); + } catch { + return null; + } + + const today = new Date().toISOString().slice(0, 10); + const escapedVer = version.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + + // First: fill in any date-placeholder line for this version (e.g. "## [0.2.0] - 2026-09-XX") + const versionPlaceholder = new RegExp(`^## \\[${escapedVer}\\]\\s*-\\s*\\d{4}-\\d{2}-XX`, "m"); + if (versionPlaceholder.test(content)) { + content = content.replace(versionPlaceholder, `## [${version}] - ${today}`); + } else { + // Otherwise: insert a new section after [Unreleased] + const unreleased = /^## \[Unreleased\]/m; + if (unreleased.test(content)) { + content = content.replace(unreleased, `## [Unreleased]\n\n## [${version}] - ${today}`); + } + } + + return { path: changelogPath, content }; +} + +// --------------------------------------------------------------------------- +// Write files +// --------------------------------------------------------------------------- + +const writes = [ + { path: VERSION_FILES.cargo, content: bumpCargo(newVersion), label: "Cargo.toml" }, + { + path: VERSION_FILES.extension, + content: bumpPackageJson(VERSION_FILES.extension, newVersion), + label: "Extension package.json", + }, + { + path: VERSION_FILES.dshPlugin, + content: bumpPackageJson(VERSION_FILES.dshPlugin, newVersion), + label: "DSH Plugin package.json", + }, +]; + +const changelogUpdate = updateChangelog(newVersion); +if (changelogUpdate) { + writes.push({ + path: changelogUpdate.path, + content: changelogUpdate.content, + label: "CHANGELOG.md", + }); +} + +for (const w of writes) { + console.log(` ✓ ${w.label}`); + if (!dryRun) { + writeFileSync(w.path, w.content); + } +} + +// --------------------------------------------------------------------------- +// Update Cargo.lock (cargo check triggers it) +// --------------------------------------------------------------------------- + +if (!dryRun) { + console.log("\n Updating Cargo.lock..."); + run("cargo check --workspace 2>&1 || true"); +} + +// --------------------------------------------------------------------------- +// Git commit + tag +// --------------------------------------------------------------------------- + +if (noCommit || dryRun) { + if (dryRun) { + console.log(`\nWould commit: "chore(release): ${newVersion}"`); + console.log("Would create tags:"); + for (const c of components) { + console.log(` ${TAG_PREFIX[c]}${newVersion}`); + } + } + console.log("\nDone."); + process.exit(0); +} + +console.log("\nCommitting..."); +run( + "git add Cargo.toml Cargo.lock apps/extension/package.json packages/dsh-plugin-browserskill/package.json CHANGELOG.md 2>/dev/null || true", +); +run(`git commit -m "chore(release): ${newVersion}"`); + +console.log("Tagging..."); +for (const c of components) { + const tag = `${TAG_PREFIX[c]}${newVersion}`; + run(`git tag ${tag}`); + console.log(` ✓ ${tag}`); +} + +// --------------------------------------------------------------------------- +// Push (optional) +// --------------------------------------------------------------------------- + +if (pushAfter) { + console.log("\nPushing..."); + run("git push origin HEAD"); + for (const c of components) { + const tag = `${TAG_PREFIX[c]}${newVersion}`; + console.log(` pushing tag ${tag}`); + run(`git push origin ${tag}`); + } +} + +console.log(`\nRelease ${newVersion} prepared successfully.`); +if (!pushAfter) { + console.log("\nNext steps:"); + console.log(" # Push branch"); + console.log(" git push origin main"); + console.log(""); + console.log(" # Push extension tag first (store review takes time)"); + console.log(` git push origin ext-v${newVersion}`); + console.log(""); + console.log(" # Then push CLI and DSH plugin tags"); + console.log(` git push origin cli-v${newVersion}`); + console.log(` git push origin dsh-plugin-v${newVersion}`); +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function run(cmd) { + execSync(cmd, { stdio: "inherit" }); +} diff --git a/skill/SKILL.md b/skill/SKILL.md index 42bb452..357625c 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -1,327 +1,180 @@ --- name: browser-skill description: | - Use when the user asks to perform browser automation tasks against their - logged-in browser: visit and read pages, fill forms, scrape data, click - through a flow, regression-test a PR's UI, validate a deployed page. - Requires the bsk CLI installed and the browser-skill extension loaded. + Use when the user asks to automate their logged-in Chromium browser: visit + and read pages, fill forms, scrape data, click through flows, regression-test + a PR's UI, validate a deployed page, or operate a tab they identify. Requires + the bsk CLI and browser extension. --- # browser-skill -Drive the user's **real Chromium browser** (with their logins and cookies) through the `bsk` CLI. The extension opens an isolated **Agent Window** for automation; the user's normal windows stay protected unless you explicitly borrow a tab. +Drive the user's real Chromium browser through `bsk`. Automation runs in an isolated **Agent +Window** with the user's existing logins and cookies. User-window tabs remain protected unless they +are explicitly borrowed. -## When to use +Do not use this skill for tasks with no browser, for extension installation, or when the user only +wants instructions. Never extract credentials, cookies, tokens, or other secrets from pages. -- Open pages, read titles/text, scrape structured data from sites the user can already access -- Fill forms, click through multi-step flows, smoke-test a UI change -- Understand pages with `bsk snapshot` first; use `bsk get-html` or `bsk screenshot` only when the snapshot is insufficient -- Operate on a specific user tab they point you at (after `bsk tab borrow`) +## Required lifecycle -## When NOT to use +Every browser task owns a bounded session: -- Tasks with **no browser** involved (files, APIs, databases only) -- Installing or configuring the extension (point the user to setup docs instead) -- **Credential harvesting** — never run `bsk evaluate` on banking, SSO, or password-manager pages to extract tokens, cookies, or secrets -- Long-lived control of a user's personal login window — borrow only for the immediate step, then `bsk tab return` or end the session -- Replacing the user's manual browsing when they only wanted an explanation - -## Prerequisites - -1. `bsk` on `PATH` (Rust CLI from browser-skill) -2. browser-skill **extension** loaded in Chromium and connected (popup shows green, or an upgrade reminder — still connected; continue) -3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails - -## Mandatory workflow - -Every automation task **must** follow this lifecycle. Do **not** rely on idle timeouts (default session idle is 5 minutes). - -``` -1. bsk session start → capture the 4-letter session id printed on stdout -2. … every tool command … → always pass --session -3. bsk session stop → REQUIRED when done (even on error paths) +```text +1. bsk session start # retain the printed 4-letter session id +2. bsk ... --session # pass it to every session-scoped command +3. bsk session stop # always run on success and error paths ``` -Optional: `bsk session start --browser ` when multiple browsers are connected (`bsk browsers` / error output lists them). Add `--no-focus` to open the Agent Window in the background without stealing focus from the user's current window. +Do not rely on the idle timeout for cleanup. Stop the session as soon as the goal is met unless the +user explicitly asks to keep it open. Stopping also returns borrowed tabs. -Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**. +Any `bsk` command auto-starts the background services it needs; never manage the daemon by hand. +When multiple browsers are connected, use `bsk browsers` and start with +`bsk session start --browser `. Add `--no-focus` to that same start command when the +Agent Window does not need to interrupt the user's current work; it is not a flag on other commands. +Run `bsk doctor` when startup or transport problems persist after one retry. -## Stop when the goal is met +## Work toward one observable goal -Every task is a **bounded goal**, not open-ended browsing. The goal may come from the user's request, a recorded `trace.json`, or both. +- Derive a concrete success condition from the user's request or a supplied trace. +- Take the shortest purposeful path: observe, act, then make at most one observation to confirm an + ambiguous result. +- Once success is visible, do not click, refresh, navigate, switch tabs, or perform extra checks. +- If a human-only step appears or two attempts make no progress, request help instead of + brute-forcing. -1. **Define success first** — one concrete, observable condition derived from the user's words, `purpose`, or the last meaningful step in a trace (e.g. "form submitted", "item added to cart", "playback started"). -2. **Take the shortest path** — snapshot → act → at most one check. Do not wander, re-try unrelated actions, or stack exploratory steps. -3. **Stop as soon as success is reached** — run `bsk session stop ` immediately unless the user explicitly asked to keep the session open (e.g. "don't close yet", "keep browsing"). -4. **No post-success work** — once the goal is met, do not click, refresh, navigate, re-search, switch tabs, or "double-check" that it worked. Further verification is a new task. -5. **When blocked, pause — do not brute-force** — if the page requires human input (login, captcha, OTP, payment confirmation) or an action fails twice with no progress, call `bsk request-help` instead of retrying blindly. See **Ask the human for help** below. -6. **When unsure** — at most one extra `bsk snapshot`. If success looks met, stop. If not, ask the user; do not keep clicking. +With a trace, follow its semantic target information and values in order, but treat its refs as +record-local hints. Stop when its purpose or last meaningful effect is satisfied. A trace guides the +task; it does not expand the user's goal or authorize additional actions. -**With a trace:** replay steps in order using `target` role/name/tag and raw `value`/`selection` fields. After the last step (or when its `effect.navigated_to` / success hint is satisfied), apply rules 3–4 immediately. The trace guides execution; it does not extend control beyond the goal. +## Observe, act, observe -**Without a trace:** the user's request *is* the success condition. Satisfying it ends the task — same stop rules apply. +Use this default loop: -## Core interaction loop - -Write operations only affect tabs in the **Agent Window** (or tabs you **borrowed** into it). - -``` +```text bsk navigate --session -bsk observe --session → primary semantic VOM view; reveals hover/focus surfaces -bsk snapshot --session → static aria tree fallback when VOM is insufficient -bsk hover @e3 --session → reveal hover-triggered menus before re-observing/clicking -bsk click @e4 --session → or bsk fill, bsk select, bsk press -bsk observe --session → again after navigation / DOM change +bsk observe --session +bsk click|hover|fill|select|press ... --session +bsk observe --session # after navigation or a meaningful DOM change ``` -**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page. +Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also +make them stale. Observe again before the next interaction. -Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`). +An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. +The listed items are labels, not usable refs: hover the trigger, observe again, then act on the +revealed item's own ref. Do not click the trigger itself unless the user wants the trigger's action. +`[has-submenu]` and `[expanded]` mark the same kind of trigger without listing what it hides. -When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover --session `, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action. +`bsk observe` does not hover the page on its own. Reach for `--probe-hover` when a control you have +good reason to expect is absent **and** no marker points at a trigger — that combination is what a +CSS-only hover menu looks like from here. It hovers a bounded set of likely triggers, so it costs a +few seconds and touches the live page; once you know which element hides the menu, `bsk hover ` +is cheaper and more precise. -## Observation priority +Escalate page reading only as needed: -Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question: +1. `bsk observe` for normal semantic understanding, text, controls, and refs. +2. `bsk observe --probe-hover` once when an expected control is missing and no marker points at a + trigger. +3. `bsk snapshot` when a stricter static accessibility tree is more useful. +4. `bsk get-html` for exact markup or hidden metadata that semantic views cannot provide. +5. `bsk screenshot` for layout, styling, canvas, images, or requested visual evidence. -1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery -2. `bsk snapshot` — strict static accessibility tree fallback -3. `bsk get-html` — when hidden DOM, metadata, or markup details are required -4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab. +Do not start with raw HTML or screenshots merely to discover ordinary controls. When interaction is +needed, obtain a fresh observation before acting on screenshot or HTML findings. -Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page. +## Respect the Agent Window boundary -## Sandbox rules +Normal page writes affect only Agent Window tabs. To operate a user tab, first list it with +`bsk tab list --scope user --session `, then `bsk tab borrow `. Return it immediately +after the relevant step with `bsk tab return `; never invent a tab id or keep a personal tab +borrowed across unrelated work. -| Rule | Detail | -|------|--------| -| Agent Window | `bsk tab create`, `bsk navigate`, `bsk click`, etc. work on agent tabs by default | -| User tabs | Read-only until borrowed: `bsk tab list --session --scope user` then `bsk tab borrow --session ` | -| Return borrowed tabs | Call `bsk tab return --session ` when finished; unreturned tabs are **auto-returned** on `bsk session stop` | -| Writes off-agent | Commands that mutate the page fail if the tab is not in the Agent Window — borrow or create a tab first | +## Ask the human when needed -## Global flags +Use `bsk request-help` for login, captcha, OTP, payment confirmation, consent, or another step the +user must complete. Give a precise prompt and pass fresh `--target` refs/selectors when concrete +controls can be highlighted. Use completion criteria only when the page has a clear stable success +signal. -| Flag | Purpose | -|------|---------| -| `--json` | Machine-readable JSON on stdout (errors too) | -| `--quiet` | Suppress informational stderr | -| `-v` / `-vv` | More verbose logging | +The result `outcome` is one of `continued`, `completed`, `cancelled`, `timed_out`, or `disabled` +(`navigated` is deprecated — never treat navigation as a completion signal). Resume only after +`continued` or `completed`. Treat `cancelled` as rejection, and `timed_out` or `disabled` as a +blocker rather than a reason to retry. After control returns, run a fresh `bsk observe` before +reasoning about the page or using refs. -Auto-update is **on by default** — the background daemon upgrades `bsk` itself when a new release is available (postponed while a session is active). Set `BSK_AUTO_UPDATE=off` to disable it and upgrade manually with `bsk update`. +## Command inventory -Command-specific flags (timeouts, `--tab-id`, `--wait-until`, …): **`bsk --help`** +This list of names is complete. Never invent a command outside it; read +`bsk --help` for flags instead of guessing them. -## CLI command reference (one line each) - -Details and flags: **`bsk --help`** - -### Diagnostics - -| Command | Summary | -|---------|---------| -| `bsk status` | Connection health, connected browsers, active sessions | -| `bsk doctor` | Deep diagnostics and repair hints | -| `bsk browsers` | List connected browser instances (ids, labels, versions) | - -### Session - -| Command | Summary | -|---------|---------| -| `bsk session start` | Open Agent Window (`--width`/`--height` for initial size); prints **4-letter session id** | -| `bsk session start --no-focus` | Open Agent Window in the background without stealing focus | -| `bsk session stop ` | End session, close Agent Window, auto-return borrowed tabs | -| `bsk session stop --all` | Stop every active session | -| `bsk session list` | List active sessions | - -### Window (require `--session `) - -| Command | Summary | -|---------|---------| -| `bsk window resize` | Resize the Agent Window (`--width`, `--height`; 100..=7680 CSS px) | - -### Device emulation — `bsk emulate` (requires `--session `) - -Emulate a mobile device environment on the agent tab via CDP — viewport, User-Agent, and touch — to debug a page's mobile layout and behaviour: - -```bash -bsk emulate --session --device iphone-14 -bsk emulate --session --width 390 --height 844 --dpr 3 --mobile --ua "Mozilla/5.0 (iPhone…" --touch -bsk emulate --session --off +```text +session start|stop|list browsers status doctor update logs +navigate navigate-back navigate-forward reload wait-for-navigation wait-ms +observe snapshot get-html screenshot console network +click hover fill select press evaluate +tab list|create|close|select|borrow|return window resize emulate +upload download request-help record start|stop ``` -- Presets (`--device`): `iphone-14`, `iphone-14-pro-max`, `iphone-se`, `pixel-7`, `galaxy-s23`, `ipad-mini`, `galaxy-tab-s8`. Manual flags (`--width`/`--height`/`--dpr`/`--mobile`/`--ua`/`--accept-language`/`--touch`/`--max-touch-points`) also work without a preset, or override individual preset fields; `--no-mobile`/`--no-touch` turn a preset's mobile viewport / touch emulation back off. -- Repeated runs merge field by field onto the tab's current emulation state — only the flags you pass change. E.g. after `--device iphone-14`, `bsk emulate --session --width 390 --height 844` keeps the preset's dpr (`3`) and mobile viewport. (The extension remembers the state per tab; after an extension reload the next run applies only the fields it carries.) -- `--off` clears every override (viewport, UA, touch) and the remembered state, restoring the tab's real environment. -- Scope: overrides apply to **one tab only** (CDP per-target) and are **not inherited by new tabs** — re-run `bsk emulate` after opening or switching to another tab (default target is the session's active tab; `--tab-id` overrides). -- Emulation covers viewport/UA/touch only; it does not throttle the network, fake geolocation, or synthesise real touch-event streams. +Required flags that are easy to get wrong: -### Tabs (require `--session `) - -| Command | Summary | -|---------|---------| -| `bsk tab list` | List tabs (`--scope user\|agent\|all`, default `all`) | -| `bsk tab create` | New tab in Agent Window (`--url`, `--no-active`, `--index`) | -| `bsk tab close ` | Close an agent tab | -| `bsk tab select ` | Focus an agent tab | -| `bsk tab borrow ` | Move a user tab into the Agent Window | -| `bsk tab return ` | Return a borrowed tab to its original window | - -### Observation (require `--session` unless noted) - -| Command | Summary | -|---------|---------| -| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs | -| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces | -| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) | -| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) | - -### Console & network debugging (read-only; require `--session`) - -| Command | Summary | -|---------|---------| -| `bsk console` | Buffered page console messages, JS exceptions, and browser log entries (`--include-stack` for stack traces) | -| `bsk network` | Buffered network responses (status, method, URL, MIME/resource type) and failures (`net::ERR_*` reason) | - -Both capture from the moment the tab is attached and read a bounded per-tab buffer: `--since ` pages from a cursor (`next_since` in the result), `--limit` (default 50, max 200), `--max-text-chars` (default 1000, max 4096), `--tab-id` to target a non-active tab. Both are strictly read-only — they never intercept or modify traffic, and request/response headers, bodies, and timings are not captured. - -### Navigation - -| Command | Summary | -|---------|---------| -| `bsk navigate ` | Go to URL in agent tab (`--wait-until`, `--timeout`) | -| `bsk navigate-back` | History back one step | -| `bsk navigate-forward` | History forward one step | -| `bsk reload` | Reload current tab (`--hard` bypass cache) | - -(`bsk navigate back` / `bsk navigate forward` are equivalent subcommands.) - -### Interaction - -| Command | Summary | -|---------|---------| -| `bsk click ` | Click element (`--button`, `--click-count`, `--modifiers`) | -| `bsk hover ` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) | -| `bsk fill --value ` | Clear and type into input | -| `bsk select --value ` | Set `