From 6800f8fdfe1aeee29a8acf6d6a2632ab2aebc4b8 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Thu, 20 Aug 2026 16:51:50 +0800 Subject: [PATCH 01/12] feat(bsk): support upload and download --- Cargo.lock | 1 + Cargo.toml | 1 + apps/extension/PRIVACY.md | 9 +- .../src/tools/__tests__/file-transfer.test.ts | 144 +++++ apps/extension/src/tools/dispatcher.ts | 38 ++ apps/extension/src/tools/download.ts | 138 +++++ apps/extension/src/tools/upload.ts | 148 ++++++ apps/extension/src/transport/types.ts | 43 ++ apps/extension/wxt.config.ts | 1 + crates/bsk-cli/Cargo.toml | 1 + crates/bsk-cli/skill/SKILL.md | 13 + crates/bsk-cli/src/cli/download.rs | 159 ++++++ crates/bsk-cli/src/cli/interaction.rs | 2 +- crates/bsk-cli/src/cli/mod.rs | 10 + crates/bsk-cli/src/cli/upload.rs | 180 +++++++ crates/bsk-cli/src/daemon/file_transfer.rs | 493 ++++++++++++++++++ crates/bsk-cli/src/daemon/ipc.rs | 173 +++++- crates/bsk-cli/src/daemon/mod.rs | 1 + crates/bsk-cli/src/daemon/start.rs | 11 +- crates/bsk-cli/src/daemon/state.rs | 6 + crates/bsk-cli/src/daemon/ws.rs | 3 + crates/bsk-cli/src/main.rs | 2 + crates/bsk-cli/tests/cli_parse.rs | 39 ++ .../schema/tool_download_params.json | 47 ++ .../schema/tool_download_result.json | 62 +++ .../schema/tool_upload_params.json | 70 +++ .../schema/tool_upload_result.json | 33 ++ crates/bsk-protocol/src/bin/dump-schema.rs | 4 + crates/bsk-protocol/src/method.rs | 22 + .../bsk-protocol/src/tools/file_transfer.rs | 170 ++++++ crates/bsk-protocol/src/tools/mod.rs | 2 + docs/architecture.md | 8 + skill/SKILL.md | 13 + 33 files changed, 2038 insertions(+), 9 deletions(-) create mode 100644 apps/extension/src/tools/__tests__/file-transfer.test.ts create mode 100644 apps/extension/src/tools/download.ts create mode 100644 apps/extension/src/tools/upload.ts create mode 100644 crates/bsk-cli/src/cli/download.rs create mode 100644 crates/bsk-cli/src/cli/upload.rs create mode 100644 crates/bsk-cli/src/daemon/file_transfer.rs create mode 100644 crates/bsk-protocol/schema/tool_download_params.json create mode 100644 crates/bsk-protocol/schema/tool_download_result.json create mode 100644 crates/bsk-protocol/schema/tool_upload_params.json create mode 100644 crates/bsk-protocol/schema/tool_upload_result.json create mode 100644 crates/bsk-protocol/src/tools/file_transfer.rs diff --git a/Cargo.lock b/Cargo.lock index 9aa9d94..443d462 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -137,6 +137,7 @@ name = "bsk" version = "0.1.10" dependencies = [ "anyhow", + "base64", "bsk-protocol", "clap", "console", diff --git a/Cargo.toml b/Cargo.toml index 007dbac..3af54d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ flate2 = "1.1.9" tar = "0.4.46" zip = { version = "7.2.0", default-features = false, features = ["deflate"] } sha2 = "0.11.0" +base64 = "0.22" tempfile = "3" nix = { version = "0.29", features = ["signal", "process"] } rand = "0.8" diff --git a/apps/extension/PRIVACY.md b/apps/extension/PRIVACY.md index 9517a3c..38e7968 100644 --- a/apps/extension/PRIVACY.md +++ b/apps/extension/PRIVACY.md @@ -1,6 +1,6 @@ # BrowserSkill — Privacy Policy -**Last updated:** May 25, 2026 +**Last updated:** August 20, 2026 This Privacy Policy describes how the **BrowserSkill** browser extension (the "Extension") handles information when you install and use it. BrowserSkill is published as part of the open-source [BrowserSkill](https://github.com/Tencent/BrowserSkill) project. The source code is publicly auditable. @@ -14,7 +14,7 @@ BrowserSkill is a local automation bridge that lets AI coding agents (such as Cu ## 2. Single Purpose -The Extension's single purpose is to expose browser automation primitives (navigation, DOM observation, screenshots, clicks, form filling, tab management) to a locally running BrowserSkill daemon over a WebSocket connection on `127.0.0.1`, so that an AI agent invoked by the user can interact with web pages on the user's behalf. +The Extension's single purpose is to expose browser automation primitives (navigation, DOM observation, screenshots, clicks, form filling, task-scoped file transfer, and tab management) to a locally running BrowserSkill daemon over a WebSocket connection on `127.0.0.1`, so that an AI agent invoked by the user can interact with web pages on the user's behalf. ## 3. Data the Extension Accesses @@ -26,6 +26,7 @@ Depending on the commands the user (via their AI agent) sends to the local daemo | **User input simulated by the agent** | Mouse clicks, keystrokes, and form values that the AI agent dispatches through the Chrome DevTools Protocol (CDP). | Required to perform automation actions the user has asked the agent to do. | | **Tab and window metadata** | Tab IDs, URLs, titles, window IDs of the Agent Window and any tabs the user explicitly authorizes. | Required to target automation commands at the correct tab/window. | | **Local extension storage** | A randomly generated 8-character instance ID and an optional user-supplied label. | Used so the local daemon can recognize this browser instance across reconnects. No personal data is stored. | +| **File transfers requested by the agent** | Local files explicitly supplied to `bsk upload`, and the file created by a single `bsk download` action. | Required to attach a task file to a web page or return a browser-generated download to the invoking local agent. | | **OS notifications** | Permission to display a system notification when the agent requests to "borrow" one of the user's existing tabs. | Required to obtain explicit, per-tab user consent before the agent touches any pre-existing tab. | ## 4. Data the Extension Does **Not** Collect @@ -34,7 +35,7 @@ BrowserSkill does **not**: - Send any data to remote servers, the Extension's authors, or any third party. - Call any LLM, AI, or cloud API. The Extension contains no API keys, model identifiers, or remote endpoints. -- Read or transmit cookies, browsing history, bookmarks, downloads, saved passwords, or autofill data. +- Read or transmit cookies, browsing history, bookmarks, saved passwords, or autofill data. It observes only the download initiated by an active `bsk download` call, not download history generally. - Use webcam, microphone, geolocation, or any device sensor. - Include analytics, telemetry, crash reporting, advertising SDKs, or fingerprinting code. - Track users across websites or across sessions. @@ -50,6 +51,7 @@ The Extension requests the following Chrome permissions. Each is used solely for - **`alarms`** — Periodically wake the service worker to keep the local WebSocket connection alive. - **`idle`** — Detect when the device returns from idle/locked so the Extension can promptly re-establish the local WebSocket connection after the machine wakes. No idle data is stored or transmitted. - **`notifications`** — Show a system notification to obtain user approval before the agent borrows a user-owned tab. +- **`downloads`** — Observe and, on cancellation, stop the one browser download initiated by an active `bsk download` command. It is not used to enumerate download history. - **`storage`** — Persist a random instance ID and optional label in `chrome.storage.local`. - **Host permission ``** — Inject a small status overlay (showing "Agent Active") on pages controlled by the agent, and enable automation across whatever sites the user directs the agent to. The Extension does **not** read or transmit page content from sites the agent is not actively driving. @@ -61,6 +63,7 @@ All Extension activity stays on the user's local device. The only network traffi - The instance ID and optional label persist in `chrome.storage.local` until the user uninstalls the Extension or clears extension storage. - Page content, screenshots, DOM snapshots, and other observed data are returned to the local daemon in response to commands and are **not retained by the Extension**. They live only as long as the agent's tool call. +- Upload and download bytes are staged by the local daemon in a private, session-scoped directory. Download staging is removed after it is copied to the requested destination. Upload staging is retained until the session ends so a later form submission can still read the attached file. Remaining staging is removed when the session ends or disconnects, or when the daemon next starts after a crash. ## 8. User Control diff --git a/apps/extension/src/tools/__tests__/file-transfer.test.ts b/apps/extension/src/tools/__tests__/file-transfer.test.ts new file mode 100644 index 0000000..16e1eef --- /dev/null +++ b/apps/extension/src/tools/__tests__/file-transfer.test.ts @@ -0,0 +1,144 @@ +import { describe, expect, it, vi } from "vitest"; +import { SessionManager } from "@/session-manager/manager"; +import { type DownloadsApi, handleDownload } from "../download"; +import type { CdpRunner } from "../shared"; +import { handleUpload } from "../upload"; + +function sessions() { + return new SessionManager({ + agentWindow: { + create: vi.fn(async () => 100), + remove: vi.fn(async () => {}), + ensureActiveTab: vi.fn(async () => {}), + }, + }); +} + +function tabsApi() { + return { + get: vi.fn( + async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, + ), + query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), + }; +} + +function fakeEvent void>() { + const listeners = new Set(); + return { + addListener: (listener: T) => listeners.add(listener), + removeListener: (listener: T) => listeners.delete(listener), + hasListener: (listener: T) => listeners.has(listener), + hasListeners: () => listeners.size > 0, + emit: (...args: Parameters) => { + for (const listener of listeners) listener(...args); + }, + }; +} + +describe("file transfer tools", () => { + it("arms chooser interception before clicking and injects only staged paths", async () => { + const manager = sessions(); + const ctx = await manager.start("s1"); + ctx.refStore.set("e3", 123, { tabId: 4 }); + let eventHandler: Parameters>[0] | undefined; + const calls: Array<{ method: string; params?: object }> = []; + const send = vi.fn(async (_tabId: number, method: string, params?: object) => { + calls.push({ method, params }); + if (method === "Page.getLayoutMetrics") + return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; + if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] }; + if ( + method === "Input.dispatchMouseEvent" && + (params as { type?: string }).type === "mousePressed" + ) { + eventHandler?.({ tabId: 4 }, "Page.fileChooserOpened", { + backendNodeId: 456, + mode: "selectMultiple", + }); + } + return {}; + }); + const cdp: CdpRunner = { + onEvent: (handler) => { + eventHandler = handler; + return { dispose: vi.fn() }; + }, + send: send as unknown as CdpRunner["send"], + }; + + const result = await handleUpload( + manager, + { + session_id: "s1", + ref: "@e3", + files: [ + { transfer_id: "tr_1", name: "one.png", staged_path: "/private/stage/one" }, + { transfer_id: "tr_2", name: "two.png", staged_path: "/private/stage/two" }, + ], + }, + { cdp, tabsApi: tabsApi() }, + ); + + expect(result).toMatchObject({ tab_id: 4, file_names: ["one.png", "two.png"] }); + expect(calls[0]).toMatchObject({ + method: "Page.setInterceptFileChooserDialog", + params: { enabled: true }, + }); + expect(calls).toContainEqual({ + method: "DOM.setFileInputFiles", + params: { files: ["/private/stage/one", "/private/stage/two"], backendNodeId: 456 }, + }); + }); + + it("captures the one download created under daemon staging", async () => { + const manager = sessions(); + const ctx = await manager.start("s1"); + ctx.refStore.set("e3", 123, { tabId: 4 }); + const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>(); + const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>(); + const item = { + id: 9, + filename: "/private/stage/tr_1/result.zip", + state: "complete", + fileSize: 12, + totalBytes: 12, + mime: "application/zip", + danger: "safe", + } as chrome.downloads.DownloadItem; + const downloads: DownloadsApi = { + onCreated: onCreated as unknown as DownloadsApi["onCreated"], + onChanged: onChanged as unknown as DownloadsApi["onChanged"], + search: vi.fn(async () => [item]), + cancel: vi.fn(async () => {}), + }; + const send = vi.fn(async (_tabId: number, method: string, params?: object) => { + if (method === "Page.getLayoutMetrics") + return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; + if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] }; + if ( + method === "Input.dispatchMouseEvent" && + (params as { type?: string }).type === "mousePressed" + ) { + onCreated.emit(item); + } + return {}; + }); + const cdp: CdpRunner = { + send: send as unknown as CdpRunner["send"], + }; + + const result = await handleDownload( + manager, + { session_id: "s1", ref: "@e3", staging_path: "/private/stage/tr_1" }, + { cdp, tabsApi: tabsApi(), downloads }, + ); + + expect(result).toMatchObject({ + tab_id: 4, + suggested_filename: "result.zip", + byte_size: 12, + staged_path: item.filename, + }); + }); +}); diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index e0a5ac4..c7cca85 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -4,6 +4,7 @@ import type { Transport } from "@/transport/transport"; import type { ClickParams, ConsoleParams, + DownloadParams, EmulateParams, EvaluateParams, FillParams, @@ -28,10 +29,12 @@ import type { ScreenshotParams, SelectParams, SnapshotParams, + UploadParams, WaitForNavigationParams, } from "@/transport/types"; import { isRequestFrame } from "@/transport/types"; import { handleConsole } from "./console"; +import { handleDownload } from "./download"; import { type EmulateCdpRunner, handleEmulate } from "./emulate"; import { handleEvaluate } from "./evaluate"; import { handleRequestHelp } from "./human-loop"; @@ -79,6 +82,7 @@ import { type TabReturnParams, type TabSelectParams, } from "./tabs"; +import { handleUpload } from "./upload"; import { handleWaitForNavigation } from "./waits"; import { handleWindowResize, type WindowResizeParams } from "./window"; @@ -513,6 +517,38 @@ export class ToolDispatcher { ), signal, ); + case "tool.upload": + return this.withHoverReleaseForRequest( + req.params as UploadParams, + () => + this.cdp + ? handleUpload(this.sessions, req.params as UploadParams, { + cdp: this.cdp, + tabsApi: chromeTabsApi, + signal, + }) + : Promise.resolve({ + code: "unsupported", + message: "upload requires CDP", + } satisfies RpcError), + signal, + ); + case "tool.download": + return this.withHoverReleaseForRequest( + req.params as DownloadParams, + () => + this.cdp + ? handleDownload(this.sessions, req.params as DownloadParams, { + cdp: this.cdp, + tabsApi: chromeTabsApi, + signal, + }) + : Promise.resolve({ + code: "unsupported", + message: "download requires CDP", + } satisfies RpcError), + signal, + ); case "tool.evaluate": return handleEvaluate( this.sessions, @@ -719,6 +755,8 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { case "tool.fill": case "tool.press": case "tool.select": + case "tool.upload": + case "tool.download": case "tool.evaluate": case "tool.observe": case "tool.request_help": diff --git a/apps/extension/src/tools/download.ts b/apps/extension/src/tools/download.ts new file mode 100644 index 0000000..e1b942e --- /dev/null +++ b/apps/extension/src/tools/download.ts @@ -0,0 +1,138 @@ +// One-click/one-file browser download capture. chrome.downloads events are +// browser-global, so a coordinator prevents two bsk sessions from racing. + +import type { SessionManager } from "@/session-manager/manager"; +import type { ClickParams, DownloadParams, DownloadResult, RpcError } from "@/transport/types"; +import { handleClick, type InteractionDeps } from "./interaction"; +import { enforceAgentWindow, isRpcError, lookupSession, resolveTargetTab } from "./shared"; + +export interface DownloadsApi { + onCreated: chrome.events.Event<(item: chrome.downloads.DownloadItem) => void>; + onChanged: chrome.events.Event<(delta: chrome.downloads.DownloadDelta) => void>; + search(query: chrome.downloads.DownloadQuery): Promise; + cancel(downloadId: number): Promise; +} + +const chromeDownloadsApi: DownloadsApi = { + get onCreated() { + return chrome.downloads.onCreated; + }, + get onChanged() { + return chrome.downloads.onChanged; + }, + search: (query) => chrome.downloads.search(query), + cancel: (id) => chrome.downloads.cancel(id), +}; + +let downloadActive = false; + +export interface DownloadDeps extends InteractionDeps { + downloads?: DownloadsApi; +} + +export async function handleDownload( + manager: SessionManager, + params: DownloadParams, + deps: DownloadDeps, +): Promise { + if (downloadActive) return { code: "invalid_params", message: "another bsk download is active" }; + downloadActive = true; + const downloads = deps.downloads ?? chromeDownloadsApi; + let capturedId: number | undefined; + let createdListener: ((item: chrome.downloads.DownloadItem) => void) | undefined; + let changedListener: ((delta: chrome.downloads.DownloadDelta) => void) | undefined; + let timer: ReturnType | undefined; + try { + const ctx = lookupSession(manager, params, "download"); + if (isRpcError(ctx)) return ctx; + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, "download"); + if (denied) return denied; + if (!params.staging_path) + return { code: "invalid_params", message: "download requires daemon staging" }; + const normalisePath = (path: string) => path.replaceAll("\\", "/"); + const normalisedStaging = normalisePath(params.staging_path); + const stagingPrefix = normalisedStaging.endsWith("/") + ? normalisedStaging + : `${normalisedStaging}/`; + let rejectCompleted!: (error: Error) => void; + const completed = new Promise((resolve, reject) => { + rejectCompleted = reject; + createdListener = (item) => { + if (!normalisePath(item.filename).startsWith(stagingPrefix)) return; + if (capturedId !== undefined && capturedId !== item.id) { + reject(new Error("download trigger produced more than one file")); + return; + } + capturedId = item.id; + if (item.state === "complete") resolve(item); + }; + changedListener = async (delta) => { + if (capturedId === undefined || delta.id !== capturedId) return; + if (delta.state?.current === "interrupted" || delta.error?.current) { + reject(new Error(delta.error?.current ?? "download interrupted")); + return; + } + if (delta.state?.current === "complete") { + const [item] = await downloads.search({ id: delta.id }); + if (item) resolve(item); + else reject(new Error("completed download disappeared")); + } + }; + downloads.onCreated.addListener(createdListener); + downloads.onChanged.addListener(changedListener); + timer = setTimeout( + () => reject(new Error("download did not complete before timeout")), + params.timeout_ms ?? 120_000, + ); + }); + const onAbort = () => { + if (capturedId !== undefined) void downloads.cancel(capturedId).catch(() => undefined); + rejectCompleted(new DOMException("aborted", "AbortError")); + }; + deps.signal?.addEventListener("abort", onAbort, { once: true }); + try { + await deps.cdp.send(target.tabId, "Page.setDownloadBehavior", { + behavior: "allow", + downloadPath: params.staging_path, + }); + const clickParams: ClickParams = { + session_id: params.session_id, + ref: params.ref, + selector: params.selector, + tab_id: params.tab_id, + timeout_ms: params.timeout_ms, + }; + const clicked = await handleClick(manager, clickParams, deps); + if (isRpcError(clicked)) { + void completed.catch(() => undefined); + return clicked; + } + const item = await completed; + return { + tab_id: target.tabId, + used_ref: clicked.used_ref, + used_selector: clicked.used_selector, + suggested_filename: item.filename.split(/[\\/]/).pop() ?? "download", + byte_size: item.fileSize >= 0 ? item.fileSize : item.totalBytes, + mime: item.mime || undefined, + danger: item.danger, + staged_path: item.filename, + }; + } finally { + deps.signal?.removeEventListener("abort", onAbort); + try { + await deps.cdp.send(target.tabId, "Page.setDownloadBehavior", { behavior: "default" }); + } catch {} + } + } catch (err) { + if (err instanceof DOMException && err.name === "AbortError") throw err; + return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err) }; + } finally { + if (timer) clearTimeout(timer); + if (createdListener) downloads.onCreated.removeListener(createdListener); + if (changedListener) downloads.onChanged.removeListener(changedListener); + downloadActive = false; + } +} diff --git a/apps/extension/src/tools/upload.ts b/apps/extension/src/tools/upload.ts new file mode 100644 index 0000000..afee918 --- /dev/null +++ b/apps/extension/src/tools/upload.ts @@ -0,0 +1,148 @@ +// Atomic file-chooser upload: arm interception before clicking, then attach +// daemon-staged files to the chooser node. No local path is accepted from an +// agent-facing call; staged_path is injected by the daemon. + +import type { CdpTarget } from "@/browser-driver/frame-graph"; +import type { SessionManager } from "@/session-manager/manager"; +import type { ClickParams, RpcError, UploadParams, UploadResult } from "@/transport/types"; +import { handleClick, type InteractionDeps } from "./interaction"; +import { + type CdpRunner, + enforceAgentWindow, + isRpcError, + lookupSession, + resolveTargetTab, + sendToCdpTarget, +} from "./shared"; +import { resolveSnapshotRef } from "./snapshot-ref"; + +const DEFAULT_TIMEOUT_MS = 120_000; + +export interface UploadDeps extends InteractionDeps { + cdp: CdpRunner; +} + +function chooserTarget( + params: UploadParams, + tabId: number, + manager: SessionManager, +): CdpTarget | RpcError { + const ctx = manager.get(params.session_id); + if (!ctx) return { code: "not_found", message: `session ${params.session_id} unknown` }; + if (params.ref) { + const ref = resolveSnapshotRef(ctx, params.ref, tabId); + if (isRpcError(ref)) return ref; + return { tabId, ...(ref.cdpSessionId ? { sessionId: ref.cdpSessionId } : {}) }; + } + return { tabId }; +} + +export async function handleUpload( + manager: SessionManager, + params: UploadParams, + deps: UploadDeps, +): Promise { + const ctx = lookupSession(manager, params, "upload"); + if (isRpcError(ctx)) return ctx; + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, "upload"); + if (denied) return denied; + if (!deps.cdp?.onEvent) + return { code: "unsupported", message: "CDP event subscription unavailable" }; + if ( + params.files.length === 0 || + params.files.length > 20 || + params.files.some((file) => !file.staged_path) + ) { + return { code: "invalid_params", message: "upload requires daemon-staged files" }; + } + const cdpTarget = chooserTarget(params, target.tabId, manager); + if (isRpcError(cdpTarget)) return cdpTarget; + const timeoutMs = params.timeout_ms ?? DEFAULT_TIMEOUT_MS; + let timer: ReturnType | undefined; + let subscription: { dispose(): void } | undefined; + let resolveChooser!: (value: { + source: { tabId?: number; sessionId?: string }; + backendNodeId: number; + mode?: string; + }) => void; + let rejectChooser!: (error: Error) => void; + const chooser = new Promise<{ + source: { tabId?: number; sessionId?: string }; + backendNodeId: number; + mode?: string; + }>((resolve, reject) => { + resolveChooser = resolve; + rejectChooser = reject; + }); + subscription = deps.cdp.onEvent((source, method, raw) => { + if (method !== "Page.fileChooserOpened" || source.tabId !== target.tabId) return; + if (cdpTarget.sessionId && source.sessionId !== cdpTarget.sessionId) return; + const event = raw as { backendNodeId?: unknown; mode?: unknown }; + if (typeof event.backendNodeId !== "number") { + rejectChooser(new Error("file chooser did not expose an input backendNodeId")); + return; + } + resolveChooser({ + source, + backendNodeId: event.backendNodeId, + ...(typeof event.mode === "string" ? { mode: event.mode } : {}), + }); + }); + timer = setTimeout( + () => rejectChooser(new Error("file chooser did not open before timeout")), + timeoutMs, + ); + const onAbort = () => rejectChooser(new DOMException("aborted", "AbortError")); + deps.signal?.addEventListener("abort", onAbort, { once: true }); + try { + await sendToCdpTarget(deps.cdp, cdpTarget, "Page.setInterceptFileChooserDialog", { + enabled: true, + }); + const clickParams: ClickParams = { + session_id: params.session_id, + ref: params.ref, + selector: params.selector, + tab_id: params.tab_id, + timeout_ms: params.timeout_ms, + }; + const clicked = await handleClick(manager, clickParams, deps); + if (isRpcError(clicked)) { + void chooser.catch(() => undefined); + return clicked; + } + const opened = await chooser; + if (opened.mode === "selectSingle" && params.files.length !== 1) { + return { code: "invalid_params", message: "file chooser accepts exactly one file" }; + } + const sourceTarget: CdpTarget = { + tabId: target.tabId, + ...(opened.source.sessionId ? { sessionId: opened.source.sessionId } : {}), + }; + await sendToCdpTarget(deps.cdp, sourceTarget, "DOM.setFileInputFiles", { + files: params.files.map((file) => file.staged_path as string), + backendNodeId: opened.backendNodeId, + }); + return { + tab_id: target.tabId, + used_ref: clicked.used_ref, + used_selector: clicked.used_selector, + file_names: params.files.map((file) => file.name), + }; + } catch (err) { + if (err instanceof DOMException && err.name === "AbortError") throw err; + return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err) }; + } finally { + if (timer) clearTimeout(timer); + subscription?.dispose(); + deps.signal?.removeEventListener("abort", onAbort); + try { + await sendToCdpTarget(deps.cdp, cdpTarget, "Page.setInterceptFileChooserDialog", { + enabled: false, + }); + } catch { + // Best-effort compensation; the target may have navigated/closed. + } + } +} diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index dc60fe2..3590257 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -500,6 +500,49 @@ export interface SelectResult { dialogs?: JavaScriptDialogInfo[]; } +export interface UploadFile { + transfer_id: string; + name: string; + staged_path?: string; +} + +export interface UploadParams { + session_id: string; + ref?: string; + selector?: string; + tab_id?: number; + files: UploadFile[]; + timeout_ms?: number; +} + +export interface UploadResult { + tab_id: number; + used_ref?: string; + used_selector?: string; + file_names: string[]; +} + +export interface DownloadParams { + session_id: string; + ref?: string; + selector?: string; + tab_id?: number; + timeout_ms?: number; + staging_path?: string; +} + +export interface DownloadResult { + tab_id: number; + used_ref?: string; + used_selector?: string; + suggested_filename: string; + byte_size: number; + mime?: string; + danger?: string; + staged_path?: string; + transfer_id?: string; +} + // -------------------------------------------------------------------------- // M9 tool payloads — evaluate / wait_for_navigation / wait_ms // -------------------------------------------------------------------------- diff --git a/apps/extension/wxt.config.ts b/apps/extension/wxt.config.ts index 6e272d9..ff4fc53 100644 --- a/apps/extension/wxt.config.ts +++ b/apps/extension/wxt.config.ts @@ -30,6 +30,7 @@ export default defineConfig({ permissions: [ "alarms", "debugger", + "downloads", "idle", "notifications", "tabs", diff --git a/crates/bsk-cli/Cargo.toml b/crates/bsk-cli/Cargo.toml index afc980f..1475e4b 100644 --- a/crates/bsk-cli/Cargo.toml +++ b/crates/bsk-cli/Cargo.toml @@ -53,6 +53,7 @@ flate2 = { workspace = true } tar = { workspace = true } zip = { workspace = true } sha2 = { workspace = true } +base64 = { workspace = true } [target.'cfg(unix)'.dependencies] nix = { workspace = true } diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index d6dd1e2..fb6e338 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -206,6 +206,19 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff | `bsk select --value ` | Set `` option values by `value` attribute. Select(SelectArgs), + /// Upload one or more local files through a page file chooser. + Upload(UploadArgs), + + /// Capture one browser download and write it to a local path. + Download(DownloadArgs), + /// Evaluate a JavaScript expression inside the Agent Window. Evaluate(EvaluateArgs), diff --git a/crates/bsk-cli/src/cli/upload.rs b/crates/bsk-cli/src/cli/upload.rs new file mode 100644 index 0000000..ac408e2 --- /dev/null +++ b/crates/bsk-cli/src/cli/upload.rs @@ -0,0 +1,180 @@ +//! `bsk upload` — stage caller-readable files and attach them to a page. + +use std::fs::File; +use std::io::Read; +use std::path::PathBuf; +use std::time::Duration; + +use anyhow::Context; +use base64::Engine; +use bsk_protocol::Method; +use bsk_protocol::tools::{ + TransferBeginParams, TransferBeginResult, TransferChunkParams, TransferChunkResult, + TransferIdParams, TransferReadyResult, TransferReleaseResult, UploadFile, UploadParams, + UploadResult, +}; +use clap::Args; + +use crate::cli::ensure_daemon::ensure_daemon; +use crate::cli::error::{CliError, Format}; +use crate::cli::interaction::split_target; +use crate::cli::navigate::parse_timeout_ms; + +#[derive(Debug, Clone, Args)] +pub struct UploadArgs { + /// Snapshot ref (`@e3`) or CSS selector for the file input / chooser trigger. + pub target: Option, + #[arg(long = "ref")] + pub ref_: Option, + #[arg(long = "selector")] + pub selector: Option, + /// Local file to upload. Repeat for a multiple-file input. + #[arg(long = "file", required = true)] + pub files: Vec, + #[arg(long)] + pub session: String, + #[arg(long = "tab-id")] + pub tab_id: Option, + #[arg(long, default_value = "2m", value_parser = parse_timeout_ms)] + pub timeout: u32, +} + +pub fn dispatch(args: UploadArgs, format: Format) -> Result<(), CliError> { + if args.files.len() > crate::daemon::file_transfer::MAX_UPLOAD_FILES { + return Err(CliError::Local(anyhow::anyhow!( + "upload accepts at most {} files", + crate::daemon::file_transfer::MAX_UPLOAD_FILES + ))); + } + let info = ensure_daemon().context("ensure daemon is running")?; + let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?; + let mut staged = Vec::new(); + let result = (|| { + for path in &args.files { + staged.push(stage_file(&info.sock_path, &args.session, path)?); + } + let params = UploadParams { + session_id: args.session, + ref_, + selector, + tab_id: args.tab_id, + files: staged + .iter() + .map(|(id, name)| UploadFile { + transfer_id: id.clone(), + name: name.clone(), + staged_path: None, + }) + .collect(), + timeout_ms: Some(args.timeout), + }; + crate::cli::business_rpc::call::<_, UploadResult>( + info.sock_path.clone(), + "upload", + Method::ToolUpload, + Some(params), + ipc_timeout(args.timeout), + ) + })(); + if result.is_err() { + for (id, _) in &staged { + let _ = release(&info.sock_path, id); + } + } + let reply = result?; + match format { + Format::Json => println!("{}", serde_json::to_string_pretty(&reply).unwrap()), + Format::Human => println!( + "upload ok tab={} files={}", + reply.tab_id, + reply.file_names.join(", ") + ), + } + Ok(()) +} + +fn stage_file(sock: &PathBuf, session: &str, path: &PathBuf) -> Result<(String, String), CliError> { + let mut file = File::open(path) + .with_context(|| format!("open upload file {}", path.display())) + .map_err(CliError::Local)?; + let meta = file + .metadata() + .with_context(|| format!("read upload file metadata {}", path.display())) + .map_err(CliError::Local)?; + if !meta.is_file() { + return Err(CliError::Local(anyhow::anyhow!( + "upload source is not a regular file: {}", + path.display() + ))); + } + let name = path + .file_name() + .and_then(|v| v.to_str()) + .filter(|v| !v.is_empty()) + .ok_or_else(|| CliError::Local(anyhow::anyhow!("upload file has no valid basename")))? + .to_string(); + let begin: TransferBeginResult = crate::cli::business_rpc::call( + sock.clone(), + "transfer-begin", + Method::TransferBegin, + Some(TransferBeginParams { + session_id: session.to_string(), + name: name.clone(), + byte_size: meta.len(), + }), + Duration::from_secs(10), + )?; + let staged = (|| { + let mut offset = 0u64; + let mut buf = vec![0u8; begin.chunk_size as usize]; + loop { + let n = file.read(&mut buf).map_err(|e| CliError::Local(e.into()))?; + if n == 0 { + break; + } + let reply: TransferChunkResult = crate::cli::business_rpc::call( + sock.clone(), + "transfer-chunk", + Method::TransferChunk, + Some(TransferChunkParams { + transfer_id: begin.transfer_id.clone(), + offset, + data_base64: base64::engine::general_purpose::STANDARD.encode(&buf[..n]), + }), + Duration::from_secs(30), + )?; + offset = reply.next_offset; + } + let _: TransferReadyResult = crate::cli::business_rpc::call( + sock.clone(), + "transfer-finish", + Method::TransferFinish, + Some(TransferIdParams { + transfer_id: begin.transfer_id.clone(), + }), + Duration::from_secs(10), + )?; + Ok::<_, CliError>(()) + })(); + if let Err(err) = staged { + let _ = release(sock, &begin.transfer_id); + return Err(err); + } + Ok((begin.transfer_id, name)) +} + +fn release(sock: &PathBuf, id: &str) -> Result { + crate::cli::business_rpc::call( + sock.clone(), + "transfer-release", + Method::TransferRelease, + Some(TransferIdParams { + transfer_id: id.to_string(), + }), + Duration::from_secs(5), + ) +} + +fn ipc_timeout(timeout_ms: u32) -> Duration { + Duration::from_millis(u64::from(timeout_ms) + 5_000) +} diff --git a/crates/bsk-cli/src/daemon/file_transfer.rs b/crates/bsk-cli/src/daemon/file_transfer.rs new file mode 100644 index 0000000..a3084eb --- /dev/null +++ b/crates/bsk-cli/src/daemon/file_transfer.rs @@ -0,0 +1,493 @@ +//! Session-scoped, content-transparent staging for upload/download tools. +//! +//! The registry deliberately exposes no arbitrary-path operations. Upload +//! bytes arrive from the invoking CLI in bounded chunks; browser downloads +//! land in a daemon-minted directory and are read back by the invoking CLI. + +use std::collections::HashMap; +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; + +use base64::Engine; +use bsk_protocol::tools::{ + TransferBeginParams, TransferBeginResult, TransferChunkParams, TransferChunkResult, + TransferIdParams, TransferReadyResult, TransferReleaseResult, +}; +use bsk_protocol::{ErrorCode, RpcError}; +use uuid::Uuid; + +use super::paths; + +pub const TRANSFER_CHUNK_SIZE: u32 = 512 * 1024; +pub const MAX_TRANSFER_BYTES: u64 = 512 * 1024 * 1024; +pub const MAX_UPLOAD_FILES: usize = 20; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Direction { + Upload, + Download, +} + +#[derive(Debug)] +struct Entry { + session_id: String, + direction: Direction, + path: PathBuf, + expected_size: Option, + written: u64, + ready: bool, +} + +#[derive(Debug)] +pub struct DownloadStaging { + pub transfer_id: String, + pub directory: PathBuf, +} + +#[derive(Debug)] +pub struct TransferRegistry { + root: PathBuf, + initialized: Mutex, + entries: Mutex>, +} + +impl TransferRegistry { + pub fn new() -> anyhow::Result { + Ok(Self { + root: paths::bsk_home()?.join("run").join("transfers"), + initialized: Mutex::new(false), + entries: Mutex::new(HashMap::new()), + }) + } + + #[cfg(test)] + fn at_root(root: PathBuf) -> anyhow::Result { + let registry = Self { + root, + initialized: Mutex::new(false), + entries: Mutex::new(HashMap::new()), + }; + registry.ensure_root()?; + Ok(registry) + } + + pub fn initialize(&self) -> anyhow::Result<()> { + self.ensure_root() + } + + fn ensure_root(&self) -> anyhow::Result<()> { + let mut initialized = self.initialized.lock().unwrap(); + if *initialized { + return Ok(()); + } + if self.root.exists() { + // Transfers never survive a daemon; removing stale directories on + // startup gives crash cleanup without a second persistence model. + let _ = fs::remove_dir_all(&self.root); + } + fs::create_dir_all(&self.root)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&self.root, fs::Permissions::from_mode(0o700))?; + } + *initialized = true; + Ok(()) + } + + pub fn begin_upload(&self, p: TransferBeginParams) -> Result { + self.ensure_root() + .map_err(|err| protocol_error(err.to_string()))?; + if p.byte_size > MAX_TRANSFER_BYTES { + return Err(invalid(format!( + "file size {} exceeds transfer limit {}", + p.byte_size, MAX_TRANSFER_BYTES + ))); + } + let mut entries = self.entries.lock().unwrap(); + let session_bytes: u64 = entries + .values() + .filter(|entry| entry.session_id == p.session_id) + .map(|entry| entry.expected_size.unwrap_or(entry.written)) + .sum(); + if session_bytes.saturating_add(p.byte_size) > MAX_TRANSFER_BYTES { + return Err(invalid(format!( + "session transfer staging exceeds limit {}", + MAX_TRANSFER_BYTES + ))); + } + let id = new_id(); + let dir = self.root.join(&id); + fs::create_dir(&dir).map_err(io_error)?; + set_private_dir(&dir).map_err(io_error)?; + let path = dir.join("payload"); + OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + .map_err(io_error)?; + entries.insert( + id.clone(), + Entry { + session_id: p.session_id, + direction: Direction::Upload, + path, + expected_size: Some(p.byte_size), + written: 0, + ready: false, + }, + ); + Ok(TransferBeginResult { + transfer_id: id, + chunk_size: TRANSFER_CHUNK_SIZE, + }) + } + + pub fn write_chunk(&self, p: TransferChunkParams) -> Result { + let bytes = base64::engine::general_purpose::STANDARD + .decode(p.data_base64) + .map_err(|e| invalid(format!("invalid transfer chunk: {e}")))?; + if bytes.len() > TRANSFER_CHUNK_SIZE as usize { + return Err(invalid("transfer chunk exceeds negotiated size")); + } + let mut entries = self.entries.lock().unwrap(); + let entry = entries + .get_mut(&p.transfer_id) + .ok_or_else(|| not_found("transfer not found"))?; + if entry.direction != Direction::Upload || entry.ready { + return Err(invalid("transfer is not writable")); + } + if p.offset != entry.written { + return Err(invalid(format!( + "non-sequential transfer offset: expected {}, got {}", + entry.written, p.offset + ))); + } + let next = entry.written.saturating_add(bytes.len() as u64); + if next > entry.expected_size.unwrap_or(MAX_TRANSFER_BYTES) { + return Err(invalid("transfer exceeds declared byte size")); + } + let mut file = OpenOptions::new() + .append(true) + .open(&entry.path) + .map_err(io_error)?; + file.write_all(&bytes).map_err(io_error)?; + entry.written = next; + Ok(TransferChunkResult { + next_offset: next, + eof: false, + data_base64: None, + }) + } + + pub fn finish_upload(&self, p: TransferIdParams) -> Result { + let mut entries = self.entries.lock().unwrap(); + let entry = entries + .get_mut(&p.transfer_id) + .ok_or_else(|| not_found("transfer not found"))?; + if entry.direction != Direction::Upload { + return Err(invalid("transfer is not an upload")); + } + if entry.written != entry.expected_size.unwrap_or(entry.written) { + return Err(invalid(format!( + "incomplete transfer: expected {} bytes, received {}", + entry.expected_size.unwrap_or(0), + entry.written + ))); + } + entry.ready = true; + Ok(TransferReadyResult { + transfer_id: p.transfer_id, + byte_size: entry.written, + }) + } + + pub fn resolve_uploads( + &self, + session_id: &str, + ids: &[String], + ) -> Result, RpcError> { + let entries = self.entries.lock().unwrap(); + ids.iter() + .map(|id| { + let entry = entries + .get(id) + .ok_or_else(|| not_found("upload transfer not found"))?; + if entry.session_id != session_id + || entry.direction != Direction::Upload + || !entry.ready + { + return Err(permission( + "upload transfer is outside this session or not ready", + )); + } + Ok(entry.path.clone()) + }) + .collect() + } + + pub fn begin_download(&self, session_id: &str) -> Result { + self.ensure_root() + .map_err(|err| protocol_error(err.to_string()))?; + let id = new_id(); + let dir = self.root.join(&id); + fs::create_dir(&dir).map_err(io_error)?; + set_private_dir(&dir).map_err(io_error)?; + self.entries.lock().unwrap().insert( + id.clone(), + Entry { + session_id: session_id.to_string(), + direction: Direction::Download, + path: dir.clone(), + expected_size: None, + written: 0, + ready: false, + }, + ); + Ok(DownloadStaging { + transfer_id: id, + directory: dir, + }) + } + + pub fn finish_download(&self, id: &str, reported_path: &Path) -> Result { + let canonical = reported_path.canonicalize().map_err(io_error)?; + let mut entries = self.entries.lock().unwrap(); + let entry = entries + .get_mut(id) + .ok_or_else(|| not_found("download transfer not found"))?; + let dir = entry.path.canonicalize().map_err(io_error)?; + if entry.direction != Direction::Download || !canonical.starts_with(&dir) { + return Err(permission("download escaped its staging directory")); + } + let meta = fs::metadata(&canonical).map_err(io_error)?; + if !meta.is_file() || meta.len() > MAX_TRANSFER_BYTES { + return Err(invalid( + "download is not a regular file or exceeds the transfer limit", + )); + } + entry.path = canonical; + entry.written = meta.len(); + entry.expected_size = Some(meta.len()); + entry.ready = true; + Ok(meta.len()) + } + + pub fn read_chunk(&self, p: TransferChunkParams) -> Result { + let entries = self.entries.lock().unwrap(); + let entry = entries + .get(&p.transfer_id) + .ok_or_else(|| not_found("transfer not found"))?; + if entry.direction != Direction::Download || !entry.ready { + return Err(invalid("transfer is not readable")); + } + if p.offset > entry.written { + return Err(invalid("read offset is beyond transfer length")); + } + let mut file = File::open(&entry.path).map_err(io_error)?; + file.seek(SeekFrom::Start(p.offset)).map_err(io_error)?; + let mut buf = vec![0u8; TRANSFER_CHUNK_SIZE as usize]; + let count = file.read(&mut buf).map_err(io_error)?; + buf.truncate(count); + let next = p.offset + count as u64; + Ok(TransferChunkResult { + next_offset: next, + eof: next >= entry.written, + data_base64: Some(base64::engine::general_purpose::STANDARD.encode(buf)), + }) + } + + pub fn release(&self, p: TransferIdParams) -> TransferReleaseResult { + let entry = self.entries.lock().unwrap().remove(&p.transfer_id); + if let Some(entry) = entry { + let dir = if entry.path.is_dir() { + entry.path + } else { + entry.path.parent().unwrap_or(&self.root).to_path_buf() + }; + let _ = fs::remove_dir_all(dir); + TransferReleaseResult { released: true } + } else { + TransferReleaseResult { released: false } + } + } + + pub fn release_session(&self, session_id: &str) { + let ids: Vec = self + .entries + .lock() + .unwrap() + .iter() + .filter(|(_, entry)| entry.session_id == session_id) + .map(|(id, _)| id.clone()) + .collect(); + for id in ids { + self.release(TransferIdParams { transfer_id: id }); + } + } +} + +fn new_id() -> String { + format!("tr_{}", Uuid::new_v4().simple()) +} + +fn set_private_dir(path: &Path) -> std::io::Result<()> { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(path, fs::Permissions::from_mode(0o700))?; + } + Ok(()) +} + +fn invalid(message: impl Into) -> RpcError { + RpcError { + code: ErrorCode::InvalidParams, + message: message.into(), + data: None, + } +} +fn not_found(message: impl Into) -> RpcError { + RpcError { + code: ErrorCode::NotFound, + message: message.into(), + data: None, + } +} +fn permission(message: impl Into) -> RpcError { + RpcError { + code: ErrorCode::PermissionDenied, + message: message.into(), + data: None, + } +} +fn io_error(err: std::io::Error) -> RpcError { + RpcError { + code: ErrorCode::ProtocolError, + message: err.to_string(), + data: None, + } +} + +fn protocol_error(message: impl Into) -> RpcError { + RpcError { + code: ErrorCode::ProtocolError, + message: message.into(), + data: None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn registry() -> (tempfile::TempDir, TransferRegistry) { + let temp = tempfile::tempdir().unwrap(); + let registry = TransferRegistry::at_root(temp.path().join("transfers")).unwrap(); + (temp, registry) + } + + #[test] + fn upload_is_not_resolvable_until_complete_or_from_another_session() { + let (_temp, registry) = registry(); + let begin = registry + .begin_upload(TransferBeginParams { + session_id: "s1".into(), + name: "image.png".into(), + byte_size: 3, + }) + .unwrap(); + assert!( + registry + .resolve_uploads("s1", &[begin.transfer_id.clone()]) + .is_err() + ); + registry + .write_chunk(TransferChunkParams { + transfer_id: begin.transfer_id.clone(), + offset: 0, + data_base64: base64::engine::general_purpose::STANDARD.encode(b"abc"), + }) + .unwrap(); + registry + .finish_upload(TransferIdParams { + transfer_id: begin.transfer_id.clone(), + }) + .unwrap(); + assert!( + registry + .resolve_uploads("s2", &[begin.transfer_id.clone()]) + .is_err() + ); + let [path] = registry + .resolve_uploads("s1", &[begin.transfer_id]) + .unwrap() + .try_into() + .unwrap(); + assert_eq!(fs::read(path).unwrap(), b"abc"); + } + + #[test] + fn download_must_finish_inside_its_minted_directory() { + let (temp, registry) = registry(); + let staging = registry.begin_download("s1").unwrap(); + let outside = temp.path().join("outside.bin"); + fs::write(&outside, b"secret").unwrap(); + assert!( + registry + .finish_download(&staging.transfer_id, &outside) + .is_err() + ); + + let inside = staging.directory.join("result.bin"); + fs::write(&inside, b"result").unwrap(); + assert_eq!( + registry + .finish_download(&staging.transfer_id, &inside) + .unwrap(), + 6 + ); + } + + #[test] + fn releasing_a_session_removes_all_staging() { + let (_temp, registry) = registry(); + let first = registry.begin_download("s1").unwrap(); + let second = registry.begin_download("s2").unwrap(); + registry.release_session("s1"); + assert!(!first.directory.exists()); + assert!(second.directory.exists()); + } + + #[test] + fn upload_staging_is_bounded_across_a_session() { + let (_temp, registry) = registry(); + registry + .begin_upload(TransferBeginParams { + session_id: "s1".into(), + name: "large.bin".into(), + byte_size: MAX_TRANSFER_BYTES, + }) + .unwrap(); + assert!( + registry + .begin_upload(TransferBeginParams { + session_id: "s1".into(), + name: "one-more-byte.bin".into(), + byte_size: 1, + }) + .is_err() + ); + assert!( + registry + .begin_upload(TransferBeginParams { + session_id: "s2".into(), + name: "other-session.bin".into(), + byte_size: 1, + }) + .is_ok() + ); + } +} diff --git a/crates/bsk-cli/src/daemon/ipc.rs b/crates/bsk-cli/src/daemon/ipc.rs index 38d0046..38ee666 100644 --- a/crates/bsk-cli/src/daemon/ipc.rs +++ b/crates/bsk-cli/src/daemon/ipc.rs @@ -27,7 +27,8 @@ use bsk_protocol::system::{ VersionSkewEntry, }; use bsk_protocol::tools::{ - HelpOutcome, RequestHelpResult, ReturnFailure, WaitMsParams, WaitMsResult, + DownloadParams, DownloadResult, HelpOutcome, RequestHelpResult, ReturnFailure, + TransferBeginParams, TransferIdParams, UploadParams, WaitMsParams, WaitMsResult, }; use bsk_protocol::{ CancelParams, CancelResult, ErrorCode, Method, PingResult, ResponseBody, RpcError, RpcId, @@ -223,6 +224,11 @@ pub fn full_handler(status: DaemonStatus, state: Arc) -> RpcHandler Ok(v) => ResponseBody::Ok(v), Err(e) => ResponseBody::Err(e), }, + Method::TransferBegin => handle_transfer_begin(&state, params), + Method::TransferChunk => handle_transfer_chunk(&state, params), + Method::TransferFinish => handle_transfer_finish(&state, params), + Method::TransferRead => handle_transfer_read(&state, params), + Method::TransferRelease => handle_transfer_release(&state, params), Method::ToolTabList | Method::ToolTabCreate | Method::ToolTabClose @@ -246,6 +252,8 @@ pub fn full_handler(status: DaemonStatus, state: Arc) -> RpcHandler | Method::ToolFill | Method::ToolPress | Method::ToolSelect + | Method::ToolUpload + | Method::ToolDownload | Method::ToolEvaluate | Method::ToolWaitForNavigation | Method::ToolRequestHelp @@ -338,24 +346,179 @@ async fn handle_tool_dispatch( }); } }; + let mut params = params; + let mut download_transfer_id: Option = None; + if method == Method::ToolUpload { + let mut upload: UploadParams = match serde_json::from_value(params) { + Ok(v) => v, + Err(err) => return ResponseBody::Err(invalid_params(err.to_string())), + }; + if upload.files.is_empty() || upload.files.len() > super::file_transfer::MAX_UPLOAD_FILES { + return ResponseBody::Err(invalid_params(format!( + "upload requires 1..={} files", + super::file_transfer::MAX_UPLOAD_FILES + ))); + } + let ids: Vec = upload.files.iter().map(|f| f.transfer_id.clone()).collect(); + let paths = match state.transfers.resolve_uploads(&session_id.0, &ids) { + Ok(v) => v, + Err(err) => return ResponseBody::Err(err), + }; + for (file, path) in upload.files.iter_mut().zip(paths) { + file.staged_path = Some(path.to_string_lossy().into_owned()); + } + params = serde_json::to_value(upload).unwrap_or(Value::Null); + } else if method == Method::ToolDownload { + let mut download: DownloadParams = match serde_json::from_value(params) { + Ok(v) => v, + Err(err) => return ResponseBody::Err(invalid_params(err.to_string())), + }; + let staging = match state.transfers.begin_download(&session_id.0) { + Ok(v) => v, + Err(err) => return ResponseBody::Err(err), + }; + download.staging_path = Some(staging.directory.to_string_lossy().into_owned()); + download_transfer_id = Some(staging.transfer_id); + params = serde_json::to_value(download).unwrap_or(Value::Null); + } let entry = inflight_guard.entry(); // `record_stop` must reach the extension while `record_await` holds the // serial busy lock — finishing the recording unblocks await. let outcome = if method == Method::ToolRecordStop { state .tool_queues - .dispatch_unlocked(&session_id, method, params, timeout, Some(entry)) + .dispatch_unlocked(&session_id, method.clone(), params, timeout, Some(entry)) .await } else { state .tool_queues - .dispatch(&session_id, method, params, timeout, Some(entry)) + .dispatch(&session_id, method.clone(), params, timeout, Some(entry)) .await }; drop(inflight_guard); match outcome { + Ok(v) if method == Method::ToolDownload => { + let id = download_transfer_id.expect("download transfer allocated"); + let mut result: DownloadResult = match serde_json::from_value(v) { + Ok(v) => v, + Err(err) => { + state + .transfers + .release(TransferIdParams { transfer_id: id }); + return ResponseBody::Err(RpcError { + code: ErrorCode::ProtocolError, + message: format!("invalid tool.download result: {err}"), + data: None, + }); + } + }; + let Some(path) = result.staged_path.take() else { + state + .transfers + .release(TransferIdParams { transfer_id: id }); + return ResponseBody::Err(RpcError { + code: ErrorCode::ProtocolError, + message: "tool.download returned no staged_path".into(), + data: None, + }); + }; + match state + .transfers + .finish_download(&id, std::path::Path::new(&path)) + { + Ok(size) => { + result.byte_size = size; + result.transfer_id = Some(id); + ResponseBody::Ok(serde_json::to_value(result).unwrap_or(Value::Null)) + } + Err(err) => { + state + .transfers + .release(TransferIdParams { transfer_id: id }); + ResponseBody::Err(err) + } + } + } Ok(v) => ResponseBody::Ok(v), - Err(err) => ResponseBody::Err(err.into_rpc()), + Err(err) => { + if let Some(id) = download_transfer_id { + state + .transfers + .release(TransferIdParams { transfer_id: id }); + } + ResponseBody::Err(err.into_rpc()) + } + } +} + +fn handle_transfer_begin(state: &Arc, params: Value) -> ResponseBody { + let p: TransferBeginParams = match serde_json::from_value(params) { + Ok(v) => v, + Err(err) => return ResponseBody::Err(invalid_params(err.to_string())), + }; + if state + .sessions + .get(&SessionId(p.session_id.clone())) + .is_none() + { + return ResponseBody::Err(RpcError { + code: ErrorCode::NotFound, + message: format!("session {} unknown", p.session_id), + data: None, + }); + } + match state.transfers.begin_upload(p) { + Ok(v) => ResponseBody::Ok(serde_json::to_value(v).unwrap_or(Value::Null)), + Err(err) => ResponseBody::Err(err), + } +} + +fn handle_transfer_chunk(state: &Arc, params: Value) -> ResponseBody { + let p = match serde_json::from_value(params) { + Ok(v) => v, + Err(err) => return ResponseBody::Err(invalid_params(err.to_string())), + }; + match state.transfers.write_chunk(p) { + Ok(v) => ResponseBody::Ok(serde_json::to_value(v).unwrap_or(Value::Null)), + Err(err) => ResponseBody::Err(err), + } +} + +fn handle_transfer_finish(state: &Arc, params: Value) -> ResponseBody { + let p = match serde_json::from_value(params) { + Ok(v) => v, + Err(err) => return ResponseBody::Err(invalid_params(err.to_string())), + }; + match state.transfers.finish_upload(p) { + Ok(v) => ResponseBody::Ok(serde_json::to_value(v).unwrap_or(Value::Null)), + Err(err) => ResponseBody::Err(err), + } +} + +fn handle_transfer_read(state: &Arc, params: Value) -> ResponseBody { + let p = match serde_json::from_value(params) { + Ok(v) => v, + Err(err) => return ResponseBody::Err(invalid_params(err.to_string())), + }; + match state.transfers.read_chunk(p) { + Ok(v) => ResponseBody::Ok(serde_json::to_value(v).unwrap_or(Value::Null)), + Err(err) => ResponseBody::Err(err), + } +} + +fn handle_transfer_release(state: &Arc, params: Value) -> ResponseBody { + let p: TransferIdParams = match serde_json::from_value(params) { + Ok(v) => v, + Err(err) => return ResponseBody::Err(invalid_params(err.to_string())), + }; + ResponseBody::Ok(serde_json::to_value(state.transfers.release(p)).unwrap_or(Value::Null)) +} + +fn invalid_params(message: impl Into) -> RpcError { + RpcError { + code: ErrorCode::InvalidParams, + message: message.into(), + data: None, } } @@ -798,6 +961,7 @@ async fn handle_session_stop( .await { Ok(stop) => { + state.transfers.release_session(&session_id.0); let result = CliSessionStopResult { stopped: vec![session_id.0], failed: Vec::new(), @@ -897,6 +1061,7 @@ async fn handle_session_stop_all( .await { Ok(stop) => { + state.transfers.release_session(&id.0); stopped.push(id.0); returned_tab_ids.extend(stop.returned_tab_ids); return_failures.extend(stop.return_failures); diff --git a/crates/bsk-cli/src/daemon/mod.rs b/crates/bsk-cli/src/daemon/mod.rs index c639f73..3196062 100644 --- a/crates/bsk-cli/src/daemon/mod.rs +++ b/crates/bsk-cli/src/daemon/mod.rs @@ -3,6 +3,7 @@ pub mod abort; pub mod browsers; mod cancel_forward; +pub mod file_transfer; pub mod inflight; pub mod info; pub mod ipc; diff --git a/crates/bsk-cli/src/daemon/start.rs b/crates/bsk-cli/src/daemon/start.rs index 241eb85..cb6460c 100644 --- a/crates/bsk-cli/src/daemon/start.rs +++ b/crates/bsk-cli/src/daemon/start.rs @@ -225,6 +225,10 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> { .with_context(|| format!("bind IPC endpoint {}", sock_path.display()))?; let state = Arc::new(DaemonState::new(cfg.clone())); + state + .transfers + .initialize() + .context("initialize transfer staging")?; let session_idle_task = spawn_session_idle_reaper(Arc::clone(&state)); let browser_liveness_task = spawn_browser_liveness_reaper(Arc::clone(&state)); // Fired by the update check task after a successful auto-update: @@ -448,6 +452,7 @@ pub(crate) fn spawn_browser_liveness_reaper( for s in state.sessions.purge_browser(&client.id) { state.tool_queues.remove(&s.id); state.session_interrupts.drop_session(&s.id); + state.transfers.release_session(&s.id.0); debug!(session = %s.id, "purged session on browser liveness timeout"); } } @@ -485,7 +490,10 @@ pub(crate) fn spawn_session_idle_reaper(state: Arc) -> tokio::task: ) .await { - Ok(_) => info!(session = %session_id, "idle session stopped"), + Ok(_) => { + state.transfers.release_session(&session_id.0); + info!(session = %session_id, "idle session stopped"); + } Err(StopSessionError::SessionBusy | StopSessionError::Stopping) => { debug!(session = %session_id, "idle session still active; retrying later"); } @@ -496,6 +504,7 @@ pub(crate) fn spawn_session_idle_reaper(state: Arc) -> tokio::task: &state.session_interrupts, &session_id, ); + state.transfers.release_session(&session_id.0); } Err(err) => { warn!(session = %session_id, error = %err, "failed to stop idle session"); diff --git a/crates/bsk-cli/src/daemon/state.rs b/crates/bsk-cli/src/daemon/state.rs index ad3b48b..1001f3e 100644 --- a/crates/bsk-cli/src/daemon/state.rs +++ b/crates/bsk-cli/src/daemon/state.rs @@ -7,6 +7,7 @@ use tokio::task::JoinHandle; use super::abort::AbortRegistry; use super::browsers::BrowserRegistry; +use super::file_transfer::TransferRegistry; use super::inflight::ToolInflightRegistry; use super::ipc::IpcHandle; use super::queue::ToolQueueRegistry; @@ -51,6 +52,9 @@ pub struct DaemonState { /// `SessionRegistry` because the signal is a transient runtime /// control state. pub session_interrupts: Arc, + /// Operation-scoped local file staging. The extension only sees paths + /// minted here; agent-facing RPCs use opaque transfer ids. + pub transfers: Arc, } impl DaemonState { @@ -64,6 +68,7 @@ impl DaemonState { )); let abort_registry = Arc::new(AbortRegistry::new()); let session_interrupts = Arc::new(SessionInterruptRegistry::new()); + let transfers = Arc::new(TransferRegistry::new().expect("initialise transfer staging")); Self { config, browsers, @@ -72,6 +77,7 @@ impl DaemonState { abort_registry, tool_inflight, session_interrupts, + transfers, } } } diff --git a/crates/bsk-cli/src/daemon/ws.rs b/crates/bsk-cli/src/daemon/ws.rs index 435b0a9..e571f3d 100644 --- a/crates/bsk-cli/src/daemon/ws.rs +++ b/crates/bsk-cli/src/daemon/ws.rs @@ -295,6 +295,7 @@ async fn drive_connection( for session in state.sessions.purge_browser(&browser_id) { state.tool_queues.remove(&session.id); state.session_interrupts.drop_session(&session.id); + state.transfers.release_session(&session.id.0); debug!(session = %session.id, "purged stale session before browser reconnect"); } let (tx, mut rx) = mpsc::unbounded_channel::(); @@ -407,6 +408,7 @@ async fn drive_connection( for s in state.sessions.purge_browser(&browser_id) { state.tool_queues.remove(&s.id); state.session_interrupts.drop_session(&s.id); + state.transfers.release_session(&s.id.0); debug!(session = %s.id, "purged session on browser disconnect"); } } else { @@ -503,6 +505,7 @@ fn handle_session_window_closed( &state.session_interrupts, &session_id, ) { + state.transfers.release_session(&session_id.0); info!(session = %session_id, "session removed: user closed Agent Window"); } else { debug!(session = %session_id, "session.window_closed for unknown session id"); diff --git a/crates/bsk-cli/src/main.rs b/crates/bsk-cli/src/main.rs index 86c1451..6ac5468 100644 --- a/crates/bsk-cli/src/main.rs +++ b/crates/bsk-cli/src/main.rs @@ -97,6 +97,8 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> { Command::Fill(args) => cli::interaction::dispatch_fill(args, format), Command::Press(args) => cli::interaction::dispatch_press(args, format), Command::Select(args) => cli::interaction::dispatch_select(args, format), + Command::Upload(args) => cli::upload::dispatch(args, format), + Command::Download(args) => cli::download::dispatch(args, format), Command::Evaluate(args) => cli::evaluate::dispatch(args, format), Command::WaitForNavigation(args) => cli::waits::dispatch_wait_for_navigation(args, format), Command::WaitMs(args) => cli::waits::dispatch_wait_ms(args, format), diff --git a/crates/bsk-cli/tests/cli_parse.rs b/crates/bsk-cli/tests/cli_parse.rs index f8cb0f2..ff7c87f 100644 --- a/crates/bsk-cli/tests/cli_parse.rs +++ b/crates/bsk-cli/tests/cli_parse.rs @@ -213,6 +213,45 @@ fn rejects_zero_click_count() { ); } +#[test] +fn parses_upload_with_repeated_files() { + let cli = parse(&[ + "bsk", + "upload", + "@e3", + "--file", + "one.png", + "--file", + "two.png", + "--session", + "s1", + ]); + let Command::Upload(args) = cli.command else { + panic!("expected upload command"); + }; + assert_eq!(args.target.as_deref(), Some("@e3")); + assert_eq!(args.files.len(), 2); +} + +#[test] +fn parses_download_with_exact_output_policy() { + let cli = parse(&[ + "bsk", + "download", + "#export", + "--out", + "result.zip", + "--session", + "s1", + "--overwrite", + ]); + let Command::Download(args) = cli.command else { + panic!("expected download command"); + }; + assert_eq!(args.target.as_deref(), Some("#export")); + assert!(args.overwrite); +} + #[test] fn parses_record_start_with_browser_and_url() { let cli = parse(&[ diff --git a/crates/bsk-protocol/schema/tool_download_params.json b/crates/bsk-protocol/schema/tool_download_params.json new file mode 100644 index 0000000..ec3c844 --- /dev/null +++ b/crates/bsk-protocol/schema/tool_download_params.json @@ -0,0 +1,47 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "DownloadParams", + "type": "object", + "required": [ + "session_id" + ], + "properties": { + "ref": { + "type": [ + "string", + "null" + ] + }, + "selector": { + "type": [ + "string", + "null" + ] + }, + "session_id": { + "type": "string" + }, + "staging_path": { + "description": "Daemon-injected destination directory for the browser process.", + "type": [ + "string", + "null" + ] + }, + "tab_id": { + "type": [ + "integer", + "null" + ], + "format": "int64" + }, + "timeout_ms": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0.0 + } + } +} diff --git a/crates/bsk-protocol/schema/tool_download_result.json b/crates/bsk-protocol/schema/tool_download_result.json new file mode 100644 index 0000000..ea2ac7c --- /dev/null +++ b/crates/bsk-protocol/schema/tool_download_result.json @@ -0,0 +1,62 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "DownloadResult", + "type": "object", + "required": [ + "byte_size", + "suggested_filename", + "tab_id" + ], + "properties": { + "byte_size": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "danger": { + "type": [ + "string", + "null" + ] + }, + "mime": { + "type": [ + "string", + "null" + ] + }, + "staged_path": { + "description": "Extension-internal completed path, stripped by the daemon.", + "type": [ + "string", + "null" + ] + }, + "suggested_filename": { + "type": "string" + }, + "tab_id": { + "type": "integer", + "format": "int64" + }, + "transfer_id": { + "description": "Opaque id returned by the daemon to the CLI.", + "type": [ + "string", + "null" + ] + }, + "used_ref": { + "type": [ + "string", + "null" + ] + }, + "used_selector": { + "type": [ + "string", + "null" + ] + } + } +} diff --git a/crates/bsk-protocol/schema/tool_upload_params.json b/crates/bsk-protocol/schema/tool_upload_params.json new file mode 100644 index 0000000..98a0747 --- /dev/null +++ b/crates/bsk-protocol/schema/tool_upload_params.json @@ -0,0 +1,70 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "UploadParams", + "type": "object", + "required": [ + "files", + "session_id" + ], + "properties": { + "files": { + "type": "array", + "items": { + "$ref": "#/definitions/UploadFile" + } + }, + "ref": { + "type": [ + "string", + "null" + ] + }, + "selector": { + "type": [ + "string", + "null" + ] + }, + "session_id": { + "type": "string" + }, + "tab_id": { + "type": [ + "integer", + "null" + ], + "format": "int64" + }, + "timeout_ms": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0.0 + } + }, + "definitions": { + "UploadFile": { + "type": "object", + "required": [ + "name", + "transfer_id" + ], + "properties": { + "name": { + "type": "string" + }, + "staged_path": { + "type": [ + "string", + "null" + ] + }, + "transfer_id": { + "type": "string" + } + } + } + } +} diff --git a/crates/bsk-protocol/schema/tool_upload_result.json b/crates/bsk-protocol/schema/tool_upload_result.json new file mode 100644 index 0000000..192c10d --- /dev/null +++ b/crates/bsk-protocol/schema/tool_upload_result.json @@ -0,0 +1,33 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "UploadResult", + "type": "object", + "required": [ + "file_names", + "tab_id" + ], + "properties": { + "file_names": { + "type": "array", + "items": { + "type": "string" + } + }, + "tab_id": { + "type": "integer", + "format": "int64" + }, + "used_ref": { + "type": [ + "string", + "null" + ] + }, + "used_selector": { + "type": [ + "string", + "null" + ] + } + } +} diff --git a/crates/bsk-protocol/src/bin/dump-schema.rs b/crates/bsk-protocol/src/bin/dump-schema.rs index e1f4e6d..6599dce 100644 --- a/crates/bsk-protocol/src/bin/dump-schema.rs +++ b/crates/bsk-protocol/src/bin/dump-schema.rs @@ -85,6 +85,10 @@ fn main() { dump!(PressResult, "tool_press_result"); dump!(SelectParams, "tool_select_params"); dump!(SelectResult, "tool_select_result"); + dump!(UploadParams, "tool_upload_params"); + dump!(UploadResult, "tool_upload_result"); + dump!(DownloadParams, "tool_download_params"); + dump!(DownloadResult, "tool_download_result"); dump!(SnapshotParams, "tool_snapshot_params"); dump!(SnapshotResult, "tool_snapshot_result"); diff --git a/crates/bsk-protocol/src/method.rs b/crates/bsk-protocol/src/method.rs index 75f8b4d..740498c 100644 --- a/crates/bsk-protocol/src/method.rs +++ b/crates/bsk-protocol/src/method.rs @@ -78,6 +78,10 @@ pub enum Method { ToolPress, #[serde(rename = "tool.select")] ToolSelect, + #[serde(rename = "tool.upload")] + ToolUpload, + #[serde(rename = "tool.download")] + ToolDownload, #[serde(rename = "tool.snapshot")] ToolSnapshot, #[serde(rename = "tool.observe")] @@ -105,6 +109,17 @@ pub enum Method { #[serde(rename = "tool.record_await")] ToolRecordAwait, + #[serde(rename = "transfer.begin")] + TransferBegin, + #[serde(rename = "transfer.chunk")] + TransferChunk, + #[serde(rename = "transfer.finish")] + TransferFinish, + #[serde(rename = "transfer.read")] + TransferRead, + #[serde(rename = "transfer.release")] + TransferRelease, + #[serde(rename = "cancel")] Cancel, } @@ -156,6 +171,8 @@ impl Method { | Method::ToolFill | Method::ToolPress | Method::ToolSelect + | Method::ToolUpload + | Method::ToolDownload | Method::ToolEvaluate // May navigate via optional `url` and changes Agent Window // chrome; gate behind pending-interrupt like other writes. @@ -194,6 +211,11 @@ impl Method { | Method::SystemPing | Method::SystemStatus | Method::BrowserList + | Method::TransferBegin + | Method::TransferChunk + | Method::TransferFinish + | Method::TransferRead + | Method::TransferRelease | Method::Cancel => MethodEffect::ControlPlane, } } diff --git a/crates/bsk-protocol/src/tools/file_transfer.rs b/crates/bsk-protocol/src/tools/file_transfer.rs new file mode 100644 index 0000000..2c2f5b8 --- /dev/null +++ b/crates/bsk-protocol/src/tools/file_transfer.rs @@ -0,0 +1,170 @@ +//! Content-transparent file transfer contracts. +//! +//! Public tool calls carry opaque transfer ids. Filesystem paths are injected +//! by the daemon only on the daemon -> extension hop and are never accepted as +//! an agent-facing source or destination. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct UploadFile { + pub transfer_id: String, + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub staged_path: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct UploadParams { + pub session_id: String, + #[serde( + rename = "ref", + alias = "ref_", + default, + skip_serializing_if = "Option::is_none" + )] + pub ref_: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub selector: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tab_id: Option, + pub files: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub timeout_ms: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct UploadResult { + pub tab_id: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub used_ref: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub used_selector: Option, + pub file_names: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct DownloadParams { + pub session_id: String, + #[serde( + rename = "ref", + alias = "ref_", + default, + skip_serializing_if = "Option::is_none" + )] + pub ref_: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub selector: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tab_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub timeout_ms: Option, + /// Daemon-injected destination directory for the browser process. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub staging_path: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct DownloadResult { + pub tab_id: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub used_ref: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub used_selector: Option, + pub suggested_filename: String, + pub byte_size: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mime: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub danger: Option, + /// Extension-internal completed path, stripped by the daemon. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub staged_path: Option, + /// Opaque id returned by the daemon to the CLI. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub transfer_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct TransferBeginParams { + pub session_id: String, + pub name: String, + pub byte_size: u64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct TransferBeginResult { + pub transfer_id: String, + pub chunk_size: u32, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct TransferChunkParams { + pub transfer_id: String, + pub offset: u64, + pub data_base64: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct TransferChunkResult { + pub next_offset: u64, + #[serde(default)] + pub eof: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub data_base64: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct TransferIdParams { + pub transfer_id: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct TransferReadyResult { + pub transfer_id: String, + pub byte_size: u64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct TransferReleaseResult { + pub released: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn upload_agent_shape_omits_daemon_path_and_uses_ref_wire_name() { + let value = serde_json::to_value(UploadParams { + session_id: "s1".into(), + ref_: Some("@e3".into()), + selector: None, + tab_id: None, + files: vec![UploadFile { + transfer_id: "tr_1".into(), + name: "image.png".into(), + staged_path: None, + }], + timeout_ms: None, + }) + .unwrap(); + assert_eq!(value["ref"], "@e3"); + assert!(value["files"][0].get("staged_path").is_none()); + } + + #[test] + fn download_agent_shape_omits_daemon_path() { + let value = serde_json::to_value(DownloadParams { + session_id: "s1".into(), + ref_: None, + selector: Some("#export".into()), + tab_id: None, + timeout_ms: None, + staging_path: None, + }) + .unwrap(); + assert!(value.get("staging_path").is_none()); + } +} diff --git a/crates/bsk-protocol/src/tools/mod.rs b/crates/bsk-protocol/src/tools/mod.rs index 88347b3..9e841bb 100644 --- a/crates/bsk-protocol/src/tools/mod.rs +++ b/crates/bsk-protocol/src/tools/mod.rs @@ -3,6 +3,7 @@ pub mod console; pub mod dialog; pub mod emulate; +pub mod file_transfer; pub mod human_loop; pub mod interaction; pub mod navigation; @@ -21,6 +22,7 @@ pub mod window; pub use console::*; pub use dialog::*; pub use emulate::*; +pub use file_transfer::*; pub use human_loop::*; pub use interaction::*; pub use navigation::*; diff --git a/docs/architecture.md b/docs/architecture.md index 2e02c57..23db98e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -143,6 +143,14 @@ flowchart LR - No credential storage in bsk — cookies stay in the user's browser profile. - `evaluate` restricted to Agent Window tabs in sandbox mode. +### File-transfer boundary + +- The invoking agent/harness decides whether a transfer is authorized and supplies the task-local source or destination path. +- The CLI is the only component that reads an upload source or writes the final download destination. The extension never receives either agent-facing path. +- The daemon issues opaque, session-scoped transfer IDs and stages bounded chunks in a private runtime directory. It injects only its own staging paths into extension RPCs and validates that a completed download remains inside its minted directory. +- The extension only performs the browser-side mechanism: intercept one chooser or capture one download, scoped to the requested session and Agent Window tab. It does not classify content or make policy decisions. +- Download staging is released after the CLI commits the file. Upload staging remains until session teardown because the page may read an attached file only on a later form submission. All staging is released on session stop/browser disconnect and on daemon startup after a crash. Existing download destinations are not overwritten unless the CLI caller explicitly opts in. + ## Repository layout ``` diff --git a/skill/SKILL.md b/skill/SKILL.md index d6dd1e2..fb6e338 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -206,6 +206,19 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff | `bsk select --value ` | Set ``; non-input pickers such as `window.showOpenFilePicker()` return a structured unsupported result for the agent to handle. The extension does not classify content or make policy decisions. - Download staging is released after the CLI commits the file. Upload staging remains until session teardown because the page may read an attached file only on a later form submission. All staging is released on session stop/browser disconnect and on daemon startup after a crash. Existing download destinations are not overwritten unless the CLI caller explicitly opts in. ## Repository layout diff --git a/skill/SKILL.md b/skill/SKILL.md index fb6e338..0e5aee1 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -217,7 +217,13 @@ The agent/harness decides whether a file transfer is appropriate and which local BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted paths reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. Download staging is removed after the CLI copies it; upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive. -Do not use `request-help` merely because a native file chooser or browser download is involved; use these commands. Ask for help only when the intended upload file is unavailable/unreadable to the agent, the page requires genuinely human-only input, or the browser reports the transfer mechanism as unsupported. +Do not use `request-help` merely because a native file chooser or browser download is involved; try these commands first. For upload failures, use the structured error instead of retrying blindly: + +- `reason=unsupported_file_chooser` means the page opened a non-input picker such as `window.showOpenFilePicker()`. Do not retry `upload`; if human help is available, call `request-help` and tell the user the exact original file path to choose. The staged daemon path is an internal implementation detail and must not be shown to the user. +- `reason=file_chooser_control_failed` means BrowserSkill could not establish or complete the browser-side transaction. Do not retry the same action blindly; use `request-help` when available. +- `reason=file_chooser_not_opened` means the click completed without opening a chooser. Observe once to verify that the target is the actual upload action (for example, after expanding a menu), then retry with the corrected target or ask for help. + +If `request-help` returns `outcome="disabled"`, do not retry it. Stop gracefully and report that the workflow requires a non-input file picker or unavailable local file. ### Scripting & timing From 14e5bbe4fc8f0e489175ed302e50bfe7673e9677 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Fri, 21 Aug 2026 11:14:51 +0800 Subject: [PATCH 03/12] feat(bsk): fix upload function that cannot bypass layout --- .../src/tools/__tests__/dispatcher.test.ts | 88 +++++ .../src/tools/__tests__/file-transfer.test.ts | 322 +++++++++-------- apps/extension/src/tools/dispatcher.ts | 2 + apps/extension/src/tools/download-capture.ts | 211 +++++++++++ apps/extension/src/tools/download.ts | 149 +++----- apps/extension/src/tools/file-chooser.ts | 265 -------------- .../src/tools/file-input-transaction.ts | 339 ++++++++++++++++++ apps/extension/src/tools/interaction.ts | 2 +- apps/extension/src/tools/upload.ts | 34 +- apps/extension/src/transport/types.ts | 11 +- crates/bsk-cli/skill/SKILL.md | 15 +- crates/bsk-cli/src/cli/download.rs | 4 +- crates/bsk-cli/src/cli/render_error.rs | 60 ++-- crates/bsk-cli/src/daemon/file_transfer.rs | 94 ++++- crates/bsk-cli/src/daemon/ipc.rs | 8 +- .../schema/tool_download_params.json | 14 +- .../schema/tool_download_result.json | 14 +- .../bsk-protocol/src/tools/file_transfer.rs | 12 +- docs/architecture.md | 4 +- skill/SKILL.md | 15 +- 20 files changed, 1033 insertions(+), 630 deletions(-) create mode 100644 apps/extension/src/tools/download-capture.ts delete mode 100644 apps/extension/src/tools/file-chooser.ts create mode 100644 apps/extension/src/tools/file-input-transaction.ts diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index c05cdb2..113fc20 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -232,6 +232,94 @@ describe("ToolDispatcher", () => { }); }); + it("bypasses and restores the control overlay for an upload trigger click", async () => { + const sendMessage = vi.fn(async () => undefined); + vi.stubGlobal("chrome", { + tabs: { + get: vi.fn(async () => ({ id: 7, windowId: 4242, active: true })), + query: vi.fn(async () => [{ id: 7, windowId: 4242, active: true }]), + sendMessage, + }, + }); + const { transport, sent, deliver } = fakeTransport(); + const sessions = new SessionManager({ + agentWindow: { + create: vi.fn(async () => 4242), + remove: vi.fn(async () => {}), + ensureActiveTab: vi.fn(async () => {}), + }, + }); + const ctx = await sessions.start("aa11"); + ctx.refStore.set("e1", 123, { tabId: 7 }); + const send = vi.fn(async (_tabId: number, method: string, params?: object) => { + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } } as T; + } + if (method === "DOM.getContentQuads") { + return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] } as T; + } + if (method === "DOM.describeNode") return { node: { backendNodeId: 456 } } as T; + if (method === "Runtime.evaluate") { + const expression = (params as { expression?: string }).expression ?? ""; + if (expression.includes("overlayDetails")) { + return { result: { value: { hitIndex: 0 } } } as T; + } + if (expression.includes("overlayHostPresent")) { + return { + result: { + value: { overlayHostPresent: true, overlayHostConnected: true }, + }, + } as T; + } + if (expression.includes("count:")) { + return { result: { value: { count: 1, multiple: false } } } as T; + } + if (expression.includes("?.inputs[0]")) { + return { result: { objectId: "input-object" } } as T; + } + return { result: { value: true } } as T; + } + return {} as T; + }); + const cdp = { + send, + detachSession: vi.fn(async () => {}), + ensureNetworkCapture: vi.fn(async () => {}), + networkEntriesSince: vi.fn(() => ({ + tab_id: 7, + entries: [], + next_since: 0, + truncated: false, + })), + setDeviceMetricsOverride: vi.fn(async () => {}), + clearDeviceMetricsOverride: vi.fn(async () => {}), + setUserAgentOverride: vi.fn(async () => {}), + setTouchEmulationEnabled: vi.fn(async () => {}), + }; + const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp }); + dispatcher.start(); + + deliver( + makeRequest("tool.upload", { + session_id: "aa11", + ref: "@e1", + files: [{ transfer_id: "tr_1", name: "test.png", staged_path: "/stage/test.png" }], + }), + ); + await flushMicrotasks(); + await vi.waitFor(() => expect(sent).toHaveLength(1)); + + expect(sent[0]).toMatchObject({ result: { tab_id: 7, file_names: ["test.png"] } }); + expect(sendMessage).toHaveBeenNthCalledWith(1, 7, { + type: "bh-automation-bypass", + enabled: true, + }); + expect(sendMessage).toHaveBeenNthCalledWith(2, 7, { + type: "bh-automation-bypass", + enabled: false, + }); + }); + it("detaches CDP state before stopping a session", async () => { const { transport, sent, deliver } = fakeTransport(); const sessions = new SessionManager({ diff --git a/apps/extension/src/tools/__tests__/file-transfer.test.ts b/apps/extension/src/tools/__tests__/file-transfer.test.ts index 182744b..75d0b7e 100644 --- a/apps/extension/src/tools/__tests__/file-transfer.test.ts +++ b/apps/extension/src/tools/__tests__/file-transfer.test.ts @@ -1,7 +1,8 @@ import { describe, expect, it, vi } from "vitest"; import { SessionManager } from "@/session-manager/manager"; import { type DownloadsApi, handleDownload } from "../download"; -import { uploadThroughFileChooser } from "../file-chooser"; +import { captureBrowserDownload } from "../download-capture"; +import { uploadThroughActivatedFileInput } from "../file-input-transaction"; import type { CdpRunner } from "../shared"; import { handleUpload } from "../upload"; @@ -24,49 +25,57 @@ function tabsApi() { }; } -function fakeEvent void>() { +function fakeEvent unknown>() { const listeners = new Set(); return { addListener: (listener: T) => listeners.add(listener), removeListener: (listener: T) => listeners.delete(listener), - hasListener: (listener: T) => listeners.has(listener), - hasListeners: () => listeners.size > 0, emit: (...args: Parameters) => { for (const listener of listeners) listener(...args); }, }; } +function uploadCdp(options: { inputCount?: number; multiple?: boolean; pickerCall?: string } = {}) { + const calls: Array<{ method: string; params?: object }> = []; + const send = vi.fn(async (_tabId: number, method: string, params?: object) => { + calls.push({ method, params }); + if (method === "Page.getLayoutMetrics") + return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; + if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] }; + if (method === "Runtime.evaluate") { + const expression = (params as { expression?: string }).expression ?? ""; + if (expression.includes("count:")) { + return { + result: { + value: { + count: options.inputCount ?? 1, + multiple: options.multiple ?? true, + pickerCall: options.pickerCall, + }, + }, + }; + } + if (expression.includes("?.inputs[0]")) { + return { result: { objectId: "input-object" } }; + } + return { result: { value: true } }; + } + if (method === "DOM.describeNode") return { node: { backendNodeId: 456 } }; + return {}; + }); + return { + calls, + cdp: { send: send as unknown as CdpRunner["send"] } satisfies CdpRunner, + }; +} + describe("file transfer tools", () => { - it("enables chooser observation before clicking and injects only staged paths", async () => { + it("captures the file input activated by the requested click and injects only staged paths", async () => { const manager = sessions(); const ctx = await manager.start("s1"); ctx.refStore.set("e3", 123, { tabId: 4 }); - let eventHandler: Parameters>[0] | undefined; - const calls: Array<{ method: string; params?: object }> = []; - const send = vi.fn(async (_tabId: number, method: string, params?: object) => { - calls.push({ method, params }); - if (method === "Page.getLayoutMetrics") - return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; - if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] }; - if ( - method === "Input.dispatchMouseEvent" && - (params as { type?: string }).type === "mousePressed" - ) { - eventHandler?.({ tabId: 4 }, "Page.fileChooserOpened", { - backendNodeId: 456, - mode: "selectMultiple", - }); - } - return {}; - }); - const cdp: CdpRunner = { - onEvent: (handler) => { - eventHandler = handler; - return { dispose: vi.fn() }; - }, - send: send as unknown as CdpRunner["send"], - }; + const { cdp, calls } = uploadCdp(); const result = await handleUpload( manager, @@ -82,61 +91,60 @@ describe("file transfer tools", () => { ); expect(result).toMatchObject({ tab_id: 4, file_names: ["one.png", "two.png"] }); - expect(calls[0]).toMatchObject({ - method: "Page.enable", - params: { enableFileChooserOpenedEvent: true }, - }); + const armCall = calls.find( + (call) => + call.method === "Runtime.evaluate" && + (call.params as { expression?: string }).expression?.includes("showOpenFilePicker"), + ); + expect(armCall).toBeDefined(); + expect((armCall?.params as { expression?: string }).expression).toContain( + "event.preventDefault()", + ); expect(calls).toContainEqual({ method: "DOM.setFileInputFiles", params: { files: ["/private/stage/one", "/private/stage/two"], backendNodeId: 456 }, }); - expect(calls.at(-1)).toEqual({ - method: "Page.enable", - params: { enableFileChooserOpenedEvent: false }, - }); }); - it("reports File System Access pickers as an explicit manual fallback", async () => { - let eventHandler: Parameters>[0] | undefined; - const cdp: CdpRunner = { - onEvent: (handler) => { - eventHandler = handler; - return { dispose: vi.fn() }; - }, - send: vi.fn(async () => ({})) as unknown as CdpRunner["send"], - }; - - const result = await uploadThroughFileChooser({ + it("fails immediately when the trigger does not activate a file input", async () => { + const { cdp } = uploadCdp({ inputCount: 0 }); + const result = await uploadThroughActivatedFileInput({ cdp, target: { tabId: 4 }, files: ["/private/stage/one"], timeoutMs: 100, - trigger: async () => { - eventHandler?.({ tabId: 4 }, "Page.fileChooserOpened", { mode: "selectSingle" }); - return { tab_id: 4, x: 10, y: 10 }; - }, + trigger: async () => ({ tab_id: 4, x: 10, y: 10 }), }); - expect(result).toEqual({ + expect(result).toMatchObject({ code: "unsupported", - message: "this file chooser is not backed by an input element", - data: { - reason: "unsupported_file_chooser", - chooser_kind: "non_input", - manual_fallback_available: true, - }, + data: { reason: "file_input_not_activated", phase: "resolve_input" }, }); }); - it("bounds a stuck chooser-control command and resets uncertain CDP state", async () => { - const detach = vi.fn(async () => {}); + it("reports a File System Access picker without waiting for a timeout", async () => { + const { cdp } = uploadCdp({ inputCount: 0, pickerCall: "showOpenFilePicker" }); + const result = await uploadThroughActivatedFileInput({ + cdp, + target: { tabId: 4 }, + files: ["/private/stage/one"], + timeoutMs: 100, + trigger: async () => ({ tab_id: 4, x: 10, y: 10 }), + }); + + expect(result).toMatchObject({ + code: "unsupported", + message: "upload trigger invoked showOpenFilePicker instead of an input[type=file]", + data: { reason: "file_input_not_activated", phase: "resolve_input" }, + }); + }); + + it("bounds a stuck file-input probe", async () => { const cdp: CdpRunner = { - onEvent: () => ({ dispose: vi.fn() }), send: vi.fn(() => new Promise(() => {})) as unknown as CdpRunner["send"], - detach, }; - const result = await uploadThroughFileChooser({ + const result = await uploadThroughActivatedFileInput({ cdp, target: { tabId: 4 }, files: ["/private/stage/one"], @@ -146,95 +154,49 @@ describe("file transfer tools", () => { expect(result).toMatchObject({ code: "timeout", - data: { reason: "file_chooser_control_failed", phase: "enable_events" }, - }); - expect(detach).toHaveBeenCalledWith(4); - }); - - it("distinguishes a completed click that did not open a chooser", async () => { - const send = vi.fn(async () => ({})) as unknown as CdpRunner["send"]; - const cdp: CdpRunner = { - onEvent: () => ({ dispose: vi.fn() }), - send, - }; - - const result = await uploadThroughFileChooser({ - cdp, - target: { tabId: 4 }, - files: ["/private/stage/one"], - timeoutMs: 5, - trigger: async () => ({ tab_id: 4, x: 10, y: 10 }), - }); - - expect(result).toMatchObject({ - code: "timeout", - data: { reason: "file_chooser_not_opened", phase: "await_event" }, - }); - expect(send).toHaveBeenLastCalledWith(4, "Page.enable", { - enableFileChooserOpenedEvent: false, + data: { reason: "file_input_probe_failed", phase: "arm_input_probe" }, }); }); - it("accepts chooser events only from the exact CDP target", async () => { - let eventHandler: Parameters>[0] | undefined; - const setFiles = vi.fn(); - const cdp: CdpRunner = { - onEvent: (handler) => { - eventHandler = handler; - return { dispose: vi.fn() }; - }, - send: vi.fn(async () => ({})) as unknown as CdpRunner["send"], - sendToTarget: vi.fn(async (_target, method, params) => { - if (method === "DOM.setFileInputFiles") setFiles(params); - return {}; - }) as unknown as NonNullable, - }; - - const result = await uploadThroughFileChooser({ - cdp, - target: { tabId: 4, sessionId: "child-a" }, - files: ["/private/stage/one"], - timeoutMs: 100, - trigger: async () => { - eventHandler?.({ tabId: 4, sessionId: "child-b" }, "Page.fileChooserOpened", { - backendNodeId: 111, - }); - eventHandler?.({ tabId: 4, sessionId: "child-a" }, "Page.fileChooserOpened", { - backendNodeId: 222, - }); - return { tab_id: 4, x: 10, y: 10 }; - }, - }); - - expect(result).toMatchObject({ click: { tab_id: 4 } }); - expect(setFiles).toHaveBeenCalledOnce(); - expect(setFiles).toHaveBeenCalledWith({ - files: ["/private/stage/one"], - backendNodeId: 222, - }); - }); - - it("captures the one download created under daemon staging", async () => { + it("routes one exact-target download through a browser-relative capability", async () => { const manager = sessions(); const ctx = await manager.start("s1"); ctx.refStore.set("e3", 123, { tabId: 4 }); const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>(); const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>(); - const item = { + const onDeterminingFilename = + fakeEvent< + ( + item: chrome.downloads.DownloadItem, + suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void, + ) => void | true + >(); + const initial = { id: 9, - filename: "/private/stage/tr_1/result.zip", - state: "complete", - fileSize: 12, + url: "https://example.test/result.zip", + finalUrl: "https://example.test/result.zip", + filename: "result.zip", + state: "in_progress", + fileSize: -1, totalBytes: 12, mime: "application/zip", danger: "safe", } as chrome.downloads.DownloadItem; + const completed = { + ...initial, + filename: "/profile/Downloads/BrowserSkill/tr_1/result.zip", + state: "complete", + fileSize: 12, + } as chrome.downloads.DownloadItem; const downloads: DownloadsApi = { - onCreated: onCreated as unknown as DownloadsApi["onCreated"], - onChanged: onChanged as unknown as DownloadsApi["onChanged"], - search: vi.fn(async () => [item]), + onCreated, + onChanged, + onDeterminingFilename, + search: vi.fn(async () => [completed]), cancel: vi.fn(async () => {}), }; + let cdpEvent: Parameters>[0] | undefined; + let suggested: chrome.downloads.DownloadFilenameSuggestion | undefined; const send = vi.fn(async (_tabId: number, method: string, params?: object) => { if (method === "Page.getLayoutMetrics") return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; @@ -243,25 +205,103 @@ describe("file transfer tools", () => { method === "Input.dispatchMouseEvent" && (params as { type?: string }).type === "mousePressed" ) { - onCreated.emit(item); + cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", { + url: initial.url, + suggestedFilename: "result.zip", + }); + await new Promise((resolve) => { + onDeterminingFilename.emit(initial, (value) => { + suggested = value; + resolve(); + }); + }); + onCreated.emit(completed); } return {}; }); const cdp: CdpRunner = { send: send as unknown as CdpRunner["send"], + onEvent: (handler) => { + cdpEvent = handler; + return { dispose: vi.fn() }; + }, }; const result = await handleDownload( manager, - { session_id: "s1", ref: "@e3", staging_path: "/private/stage/tr_1" }, + { session_id: "s1", ref: "@e3", browser_relative_dir: "BrowserSkill/tr_1" }, { cdp, tabsApi: tabsApi(), downloads }, ); + expect(suggested).toEqual({ + filename: "BrowserSkill/tr_1/result.zip", + conflictAction: "overwrite", + }); expect(result).toMatchObject({ tab_id: 4, suggested_filename: "result.zip", byte_size: 12, - staged_path: item.filename, + browser_path: completed.filename, + }); + }); + + it("does not claim a download without an intent from the exact target", async () => { + const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>(); + const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>(); + const onDeterminingFilename = + fakeEvent< + ( + item: chrome.downloads.DownloadItem, + suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void, + ) => void | true + >(); + const downloads: DownloadsApi = { + onCreated, + onChanged, + onDeterminingFilename, + search: vi.fn(async () => []), + cancel: vi.fn(async () => {}), + }; + let cdpEvent: Parameters>[0] | undefined; + const cdp: CdpRunner = { + send: vi.fn(async () => ({})) as unknown as CdpRunner["send"], + onEvent: (handler) => { + cdpEvent = handler; + return { dispose: vi.fn() }; + }, + }; + const unrelated = { + id: 17, + url: "https://example.test/unrelated.zip", + finalUrl: "https://example.test/unrelated.zip", + filename: "unrelated.zip", + state: "in_progress", + } as chrome.downloads.DownloadItem; + let defaultSuggestionCalled = false; + + const result = await captureBrowserDownload({ + cdp, + target: { tabId: 4, sessionId: "expected-child" }, + downloads, + browserRelativeDir: "BrowserSkill/tr_1", + timeoutMs: 5, + trigger: async () => { + cdpEvent?.({ tabId: 4, sessionId: "other-child" }, "Page.downloadWillBegin", { + url: unrelated.url, + suggestedFilename: unrelated.filename, + }); + onDeterminingFilename.emit(unrelated, (suggestion) => { + defaultSuggestionCalled = suggestion === undefined; + }); + return { tab_id: 4, x: 10, y: 10 }; + }, + }); + + expect(defaultSuggestionCalled).toBe(true); + expect(downloads.cancel).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + code: "cdp_failed", + data: { reason: "download_capture_failed" }, }); }); }); diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index c7cca85..f6366f2 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -526,6 +526,7 @@ export class ToolDispatcher { cdp: this.cdp, tabsApi: chromeTabsApi, signal, + bypassOverlay, }) : Promise.resolve({ code: "unsupported", @@ -542,6 +543,7 @@ export class ToolDispatcher { cdp: this.cdp, tabsApi: chromeTabsApi, signal, + bypassOverlay, }) : Promise.resolve({ code: "unsupported", diff --git a/apps/extension/src/tools/download-capture.ts b/apps/extension/src/tools/download-capture.ts new file mode 100644 index 0000000..afab991 --- /dev/null +++ b/apps/extension/src/tools/download-capture.ts @@ -0,0 +1,211 @@ +// Transaction-scoped capture for one web download. Chrome owns the actual +// download and writes only beneath its Downloads root; BrowserSkill supplies a +// daemon-minted relative directory and reports the completed absolute path +// back to the daemon for validated import. + +import type { CdpTarget } from "@/browser-driver/frame-graph"; +import type { ClickResult, RpcError } from "@/transport/types"; +import { type CdpRunner, isRpcError } from "./shared"; + +type DeterminingFilenameListener = ( + item: chrome.downloads.DownloadItem, + suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void, +) => void | true; + +interface ListenerEvent { + addListener(listener: T): void; + removeListener(listener: T): void; +} + +export interface DownloadsApi { + onCreated: ListenerEvent<(item: chrome.downloads.DownloadItem) => void>; + onChanged: ListenerEvent<(delta: chrome.downloads.DownloadDelta) => void>; + onDeterminingFilename: ListenerEvent; + search(query: chrome.downloads.DownloadQuery): Promise; + cancel(downloadId: number): Promise; +} + +export const chromeDownloadsApi: DownloadsApi = { + get onCreated() { + return chrome.downloads.onCreated; + }, + get onChanged() { + return chrome.downloads.onChanged; + }, + get onDeterminingFilename() { + return chrome.downloads.onDeterminingFilename; + }, + search: (query) => chrome.downloads.search(query), + cancel: (id) => chrome.downloads.cancel(id), +}; + +export interface DownloadCaptureOptions { + cdp: CdpRunner; + target: CdpTarget; + downloads: DownloadsApi; + browserRelativeDir: string; + timeoutMs: number; + signal?: AbortSignal; + trigger(): Promise; +} + +export interface DownloadCaptureResult { + click: ClickResult; + item: chrome.downloads.DownloadItem; +} + +function safeBasename(filename: string): string { + const basename = filename.split(/[\\/]/).pop()?.trim(); + return basename && basename !== "." && basename !== ".." ? basename : "download"; +} + +function captureError(message: string): RpcError { + return { + code: "cdp_failed", + message, + data: { reason: "download_capture_failed" }, + }; +} + +function sameTarget(source: { tabId?: number; sessionId?: string }, target: CdpTarget): boolean { + return source.tabId === target.tabId && source.sessionId === target.sessionId; +} + +function matchesIntent( + item: chrome.downloads.DownloadItem, + intent: { url: string; suggestedFilename: string }, +): boolean { + const urlMatches = item.url === intent.url || item.finalUrl === intent.url; + return urlMatches && safeBasename(item.filename) === safeBasename(intent.suggestedFilename); +} + +export async function captureBrowserDownload( + options: DownloadCaptureOptions, +): Promise { + let capturedId: number | undefined; + let intent: { url: string; suggestedFilename: string } | undefined; + const createdItems = new Map(); + let settled = false; + let succeeded = false; + let timer: ReturnType | undefined; + let rejectCompletion!: (error: Error) => void; + let resolveCompletion!: (item: chrome.downloads.DownloadItem) => void; + + const completion = new Promise((resolve, reject) => { + resolveCompletion = resolve; + rejectCompletion = reject; + }); + const fail = (error: Error) => { + if (settled) return; + settled = true; + rejectCompletion(error); + }; + const complete = (item: chrome.downloads.DownloadItem) => { + if (settled) return; + settled = true; + resolveCompletion(item); + }; + const claim = (item: chrome.downloads.DownloadItem): boolean => { + if (capturedId === undefined) { + capturedId = item.id; + return true; + } + if (capturedId === item.id) return true; + void options.downloads.cancel(item.id).catch(() => undefined); + fail(new Error("download trigger produced more than one file")); + return false; + }; + + const determiningListener: DeterminingFilenameListener = (item, suggest) => { + try { + if (!intent || !matchesIntent(item, intent) || !claim(item)) { + suggest(); + return; + } + suggest({ + filename: `${options.browserRelativeDir}/${safeBasename(intent.suggestedFilename)}`, + conflictAction: "overwrite", + }); + const created = createdItems.get(item.id); + if (created?.state === "complete") complete(created); + } catch (err) { + suggest(); + fail(err instanceof Error ? err : new Error(String(err))); + } + }; + const createdListener = (item: chrome.downloads.DownloadItem) => { + createdItems.set(item.id, item); + if (capturedId !== item.id) return; + if (item.state === "interrupted") { + fail(new Error(item.error ?? "download interrupted")); + } else if (item.state === "complete") { + complete(item); + } + }; + const changedListener = async (delta: chrome.downloads.DownloadDelta) => { + if (capturedId === undefined || delta.id !== capturedId || settled) return; + if (delta.state?.current === "interrupted" || delta.error?.current) { + fail(new Error(delta.error?.current ?? "download interrupted")); + return; + } + if (delta.state?.current === "complete") { + try { + const [item] = await options.downloads.search({ id: delta.id }); + if (item) complete(item); + else fail(new Error("completed download disappeared")); + } catch (err) { + fail(err instanceof Error ? err : new Error(String(err))); + } + } + }; + const onAbort = () => { + fail(new DOMException("aborted", "AbortError")); + }; + const cdpSubscription = options.cdp.onEvent?.((source, method, raw) => { + if (method !== "Page.downloadWillBegin" || !sameTarget(source, options.target)) return; + const event = raw as { url?: unknown; suggestedFilename?: unknown }; + if (typeof event.url !== "string" || typeof event.suggestedFilename !== "string") return; + if (intent) { + fail(new Error("download trigger produced more than one browser download intent")); + return; + } + intent = { url: event.url, suggestedFilename: event.suggestedFilename }; + }); + if (!cdpSubscription) { + return captureError("CDP download intent subscription unavailable"); + } + + options.downloads.onDeterminingFilename.addListener(determiningListener); + options.downloads.onCreated.addListener(createdListener); + options.downloads.onChanged.addListener(changedListener); + options.signal?.addEventListener("abort", onAbort, { once: true }); + timer = setTimeout( + () => fail(new Error("download did not complete before timeout")), + options.timeoutMs, + ); + + try { + const triggered = await options.trigger(); + if (isRpcError(triggered)) { + void completion.catch(() => undefined); + return triggered; + } + const item = await completion; + succeeded = true; + return { click: triggered, item }; + } catch (err) { + if (err instanceof DOMException && err.name === "AbortError") throw err; + return captureError(err instanceof Error ? err.message : String(err)); + } finally { + settled = true; + if (timer) clearTimeout(timer); + if (!succeeded && capturedId !== undefined) { + await options.downloads.cancel(capturedId).catch(() => undefined); + } + options.signal?.removeEventListener("abort", onAbort); + options.downloads.onDeterminingFilename.removeListener(determiningListener); + options.downloads.onCreated.removeListener(createdListener); + options.downloads.onChanged.removeListener(changedListener); + cdpSubscription.dispose(); + } +} diff --git a/apps/extension/src/tools/download.ts b/apps/extension/src/tools/download.ts index e1b942e..d337315 100644 --- a/apps/extension/src/tools/download.ts +++ b/apps/extension/src/tools/download.ts @@ -1,31 +1,16 @@ -// One-click/one-file browser download capture. chrome.downloads events are -// browser-global, so a coordinator prevents two bsk sessions from racing. +// Download orchestration: validate session/tab ownership, then delegate one +// browser-global chrome.downloads transaction to download-capture.ts. import type { SessionManager } from "@/session-manager/manager"; import type { ClickParams, DownloadParams, DownloadResult, RpcError } from "@/transport/types"; -import { handleClick, type InteractionDeps } from "./interaction"; +import { captureBrowserDownload, chromeDownloadsApi, type DownloadsApi } from "./download-capture"; +import { handleClick, type InteractionDeps, resolveBackendNode } from "./interaction"; import { enforceAgentWindow, isRpcError, lookupSession, resolveTargetTab } from "./shared"; -export interface DownloadsApi { - onCreated: chrome.events.Event<(item: chrome.downloads.DownloadItem) => void>; - onChanged: chrome.events.Event<(delta: chrome.downloads.DownloadDelta) => void>; - search(query: chrome.downloads.DownloadQuery): Promise; - cancel(downloadId: number): Promise; -} - -const chromeDownloadsApi: DownloadsApi = { - get onCreated() { - return chrome.downloads.onCreated; - }, - get onChanged() { - return chrome.downloads.onChanged; - }, - search: (query) => chrome.downloads.search(query), - cancel: (id) => chrome.downloads.cancel(id), -}; - let downloadActive = false; +export type { DownloadsApi } from "./download-capture"; + export interface DownloadDeps extends InteractionDeps { downloads?: DownloadsApi; } @@ -37,11 +22,6 @@ export async function handleDownload( ): Promise { if (downloadActive) return { code: "invalid_params", message: "another bsk download is active" }; downloadActive = true; - const downloads = deps.downloads ?? chromeDownloadsApi; - let capturedId: number | undefined; - let createdListener: ((item: chrome.downloads.DownloadItem) => void) | undefined; - let changedListener: ((delta: chrome.downloads.DownloadDelta) => void) | undefined; - let timer: ReturnType | undefined; try { const ctx = lookupSession(manager, params, "download"); if (isRpcError(ctx)) return ctx; @@ -49,90 +29,43 @@ export async function handleDownload( if (isRpcError(target)) return target; const denied = enforceAgentWindow(ctx, target, "download"); if (denied) return denied; - if (!params.staging_path) - return { code: "invalid_params", message: "download requires daemon staging" }; - const normalisePath = (path: string) => path.replaceAll("\\", "/"); - const normalisedStaging = normalisePath(params.staging_path); - const stagingPrefix = normalisedStaging.endsWith("/") - ? normalisedStaging - : `${normalisedStaging}/`; - let rejectCompleted!: (error: Error) => void; - const completed = new Promise((resolve, reject) => { - rejectCompleted = reject; - createdListener = (item) => { - if (!normalisePath(item.filename).startsWith(stagingPrefix)) return; - if (capturedId !== undefined && capturedId !== item.id) { - reject(new Error("download trigger produced more than one file")); - return; - } - capturedId = item.id; - if (item.state === "complete") resolve(item); - }; - changedListener = async (delta) => { - if (capturedId === undefined || delta.id !== capturedId) return; - if (delta.state?.current === "interrupted" || delta.error?.current) { - reject(new Error(delta.error?.current ?? "download interrupted")); - return; - } - if (delta.state?.current === "complete") { - const [item] = await downloads.search({ id: delta.id }); - if (item) resolve(item); - else reject(new Error("completed download disappeared")); - } - }; - downloads.onCreated.addListener(createdListener); - downloads.onChanged.addListener(changedListener); - timer = setTimeout( - () => reject(new Error("download did not complete before timeout")), - params.timeout_ms ?? 120_000, - ); - }); - const onAbort = () => { - if (capturedId !== undefined) void downloads.cancel(capturedId).catch(() => undefined); - rejectCompleted(new DOMException("aborted", "AbortError")); - }; - deps.signal?.addEventListener("abort", onAbort, { once: true }); - try { - await deps.cdp.send(target.tabId, "Page.setDownloadBehavior", { - behavior: "allow", - downloadPath: params.staging_path, - }); - const clickParams: ClickParams = { - session_id: params.session_id, - ref: params.ref, - selector: params.selector, - tab_id: params.tab_id, - timeout_ms: params.timeout_ms, - }; - const clicked = await handleClick(manager, clickParams, deps); - if (isRpcError(clicked)) { - void completed.catch(() => undefined); - return clicked; - } - const item = await completed; - return { - tab_id: target.tabId, - used_ref: clicked.used_ref, - used_selector: clicked.used_selector, - suggested_filename: item.filename.split(/[\\/]/).pop() ?? "download", - byte_size: item.fileSize >= 0 ? item.fileSize : item.totalBytes, - mime: item.mime || undefined, - danger: item.danger, - staged_path: item.filename, - }; - } finally { - deps.signal?.removeEventListener("abort", onAbort); - try { - await deps.cdp.send(target.tabId, "Page.setDownloadBehavior", { behavior: "default" }); - } catch {} + if (!params.browser_relative_dir) { + return { code: "invalid_params", message: "download requires a daemon capability directory" }; } - } catch (err) { - if (err instanceof DOMException && err.name === "AbortError") throw err; - return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err) }; + const address = await resolveBackendNode(deps.cdp, ctx, target, params, "download"); + if (isRpcError(address)) return address; + + const capture = await captureBrowserDownload({ + cdp: deps.cdp, + target: address.cdpTarget, + downloads: deps.downloads ?? chromeDownloadsApi, + browserRelativeDir: params.browser_relative_dir, + timeoutMs: params.timeout_ms ?? 120_000, + signal: deps.signal, + trigger: () => { + const clickParams: ClickParams = { + session_id: params.session_id, + ref: params.ref, + selector: params.selector, + tab_id: params.tab_id, + timeout_ms: params.timeout_ms, + }; + return handleClick(manager, clickParams, deps); + }, + }); + if (isRpcError(capture)) return capture; + const { click, item } = capture; + return { + tab_id: target.tabId, + used_ref: click.used_ref, + used_selector: click.used_selector, + suggested_filename: item.filename.split(/[\\/]/).pop() ?? "download", + byte_size: item.fileSize >= 0 ? item.fileSize : item.totalBytes, + mime: item.mime || undefined, + danger: item.danger, + browser_path: item.filename, + }; } finally { - if (timer) clearTimeout(timer); - if (createdListener) downloads.onCreated.removeListener(createdListener); - if (changedListener) downloads.onChanged.removeListener(changedListener); downloadActive = false; } } diff --git a/apps/extension/src/tools/file-chooser.ts b/apps/extension/src/tools/file-chooser.ts deleted file mode 100644 index ab72241..0000000 --- a/apps/extension/src/tools/file-chooser.ts +++ /dev/null @@ -1,265 +0,0 @@ -// One file-chooser transaction for an upload. Chrome's extension debugger -// bridge may leave a command pending indefinitely, so every state-changing -// command is bounded and an uncertain target is detached before returning. - -import type { CdpTarget } from "@/browser-driver/frame-graph"; -import type { ClickResult, RpcError } from "@/transport/types"; -import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared"; - -const CLEANUP_TIMEOUT_MS = 1_000; - -type ChooserPhase = "enable_events" | "trigger" | "await_event" | "set_files"; - -interface FileChooserOpened { - source: { tabId?: number; sessionId?: string }; - backendNodeId?: number; - mode?: string; -} - -export interface FileChooserUploadOptions { - cdp: CdpRunner; - target: CdpTarget; - files: string[]; - timeoutMs: number; - signal?: AbortSignal; - trigger(timeoutMs: number): Promise; -} - -export interface FileChooserUploadResult { - click: ClickResult; - mode?: string; -} - -class BoundedWaitError extends Error { - constructor( - readonly kind: "timeout" | "aborted", - message: string, - ) { - super(message); - } -} - -function remainingMs(deadline: number): number { - return Math.max(0, deadline - Date.now()); -} - -async function waitBounded( - promise: Promise, - deadline: number, - signal: AbortSignal | undefined, - timeoutMessage: string, -): Promise { - const remaining = remainingMs(deadline); - if (signal?.aborted) throw new BoundedWaitError("aborted", "file chooser operation aborted"); - if (remaining === 0) throw new BoundedWaitError("timeout", timeoutMessage); - - let timer: ReturnType | undefined; - let onAbort: (() => void) | undefined; - const boundary = new Promise((_resolve, reject) => { - timer = setTimeout(() => reject(new BoundedWaitError("timeout", timeoutMessage)), remaining); - if (signal) { - onAbort = () => reject(new BoundedWaitError("aborted", "file chooser operation aborted")); - signal.addEventListener("abort", onAbort, { once: true }); - } - }); - try { - return await Promise.race([promise, boundary]); - } finally { - if (timer) clearTimeout(timer); - if (signal && onAbort) signal.removeEventListener("abort", onAbort); - } -} - -function chooserError( - code: RpcError["code"], - message: string, - reason: "file_chooser_control_failed" | "file_chooser_not_opened", - phase: ChooserPhase, -): RpcError { - return { code, message, data: { reason, phase } }; -} - -function sameTarget(source: FileChooserOpened["source"], target: CdpTarget): boolean { - return source.tabId === target.tabId && source.sessionId === target.sessionId; -} - -async function resetTarget(cdp: CdpRunner, tabId: number): Promise { - if (!cdp.detach) return; - try { - await waitBounded( - cdp.detach(tabId), - Date.now() + CLEANUP_TIMEOUT_MS, - undefined, - "CDP detach timed out", - ); - } catch { - // The target may already have detached or closed. - } -} - -export async function uploadThroughFileChooser( - options: FileChooserUploadOptions, -): Promise { - const deadline = Date.now() + options.timeoutMs; - let resolveOpened!: (value: FileChooserOpened) => void; - const opened = new Promise((resolve) => { - resolveOpened = resolve; - }); - const subscription = options.cdp.onEvent?.((source, method, raw) => { - if (method !== "Page.fileChooserOpened" || !sameTarget(source, options.target)) return; - const event = raw as { backendNodeId?: unknown; mode?: unknown }; - resolveOpened({ - source, - ...(typeof event.backendNodeId === "number" ? { backendNodeId: event.backendNodeId } : {}), - ...(typeof event.mode === "string" ? { mode: event.mode } : {}), - }); - }); - if (!subscription) { - return { code: "unsupported", message: "CDP event subscription unavailable" }; - } - - let eventsEnabled = false; - let targetStateUnknown = false; - let outcome: FileChooserUploadResult | RpcError; - let aborted: BoundedWaitError | undefined; - try { - try { - await waitBounded( - sendToCdpTarget(options.cdp, options.target, "Page.enable", { - enableFileChooserOpenedEvent: true, - }), - deadline, - options.signal, - "enabling file chooser events timed out", - ); - eventsEnabled = true; - } catch (err) { - if (err instanceof BoundedWaitError) { - targetStateUnknown = true; - if (err.kind === "aborted") throw err; - outcome = chooserError( - "timeout", - err.message, - "file_chooser_control_failed", - "enable_events", - ); - } else { - outcome = chooserError( - "cdp_failed", - err instanceof Error ? err.message : String(err), - "file_chooser_control_failed", - "enable_events", - ); - } - return outcome; - } - - let click: ClickResult | RpcError; - try { - click = await waitBounded( - options.trigger(remainingMs(deadline)), - deadline, - options.signal, - "upload trigger timed out", - ); - } catch (err) { - if (err instanceof BoundedWaitError) { - targetStateUnknown = true; - if (err.kind === "aborted") throw err; - return chooserError("timeout", err.message, "file_chooser_control_failed", "trigger"); - } - return chooserError( - "cdp_failed", - err instanceof Error ? err.message : String(err), - "file_chooser_control_failed", - "trigger", - ); - } - if (isRpcError(click)) return click; - - let chooser: FileChooserOpened; - try { - chooser = await waitBounded( - opened, - deadline, - options.signal, - "upload trigger did not open a file chooser", - ); - } catch (err) { - if (err instanceof BoundedWaitError && err.kind === "aborted") throw err; - return chooserError( - "timeout", - err instanceof Error ? err.message : String(err), - "file_chooser_not_opened", - "await_event", - ); - } - - if (chooser.backendNodeId === undefined) { - return { - code: "unsupported", - message: "this file chooser is not backed by an input element", - data: { - reason: "unsupported_file_chooser", - chooser_kind: "non_input", - manual_fallback_available: true, - }, - }; - } - if (chooser.mode === "selectSingle" && options.files.length !== 1) { - return { code: "invalid_params", message: "file chooser accepts exactly one file" }; - } - - try { - await waitBounded( - sendToCdpTarget(options.cdp, options.target, "DOM.setFileInputFiles", { - files: options.files, - backendNodeId: chooser.backendNodeId, - }), - deadline, - options.signal, - "setting file input files timed out", - ); - } catch (err) { - if (err instanceof BoundedWaitError) { - targetStateUnknown = true; - if (err.kind === "aborted") throw err; - return chooserError("timeout", err.message, "file_chooser_control_failed", "set_files"); - } - return chooserError( - "cdp_failed", - err instanceof Error ? err.message : String(err), - "file_chooser_control_failed", - "set_files", - ); - } - outcome = { click, ...(chooser.mode ? { mode: chooser.mode } : {}) }; - } catch (err) { - if (err instanceof BoundedWaitError && err.kind === "aborted") aborted = err; - else throw err; - outcome = { code: "cancelled", message: "file chooser operation aborted" }; - } finally { - subscription.dispose(); - if (targetStateUnknown) { - await resetTarget(options.cdp, options.target.tabId); - } else if (eventsEnabled) { - try { - await waitBounded( - sendToCdpTarget(options.cdp, options.target, "Page.enable", { - enableFileChooserOpenedEvent: false, - }), - Date.now() + CLEANUP_TIMEOUT_MS, - undefined, - "restoring file chooser events timed out", - ); - } catch { - await resetTarget(options.cdp, options.target.tabId); - } - } - } - if (aborted) { - const error = new DOMException(aborted.message, "AbortError"); - throw error; - } - return outcome; -} diff --git a/apps/extension/src/tools/file-input-transaction.ts b/apps/extension/src/tools/file-input-transaction.ts new file mode 100644 index 0000000..50297ad --- /dev/null +++ b/apps/extension/src/tools/file-input-transaction.ts @@ -0,0 +1,339 @@ +// One upload transaction, independent of Page.fileChooserOpened delivery. +// A transaction-scoped DOM listener records the file input actually activated +// by the requested click and cancels its native default action. File System +// Access picker entry points are replaced only for the same transaction, so a +// non-input picker fails promptly without opening an OS dialog. + +import type { CdpTarget } from "@/browser-driver/frame-graph"; +import type { ClickResult, RpcError } from "@/transport/types"; +import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared"; + +const CLEANUP_TIMEOUT_MS = 1_000; + +type UploadPhase = "arm_input_probe" | "trigger" | "resolve_input" | "set_files"; + +interface RuntimeReply { + result?: { + value?: unknown; + objectId?: string; + }; + exceptionDetails?: { + text?: string; + exception?: { description?: string }; + }; +} + +export interface FileInputTransactionOptions { + cdp: CdpRunner; + target: CdpTarget; + files: string[]; + timeoutMs: number; + signal?: AbortSignal; + trigger(timeoutMs: number): Promise; +} + +export interface FileInputTransactionResult { + click: ClickResult; + multiple: boolean; +} + +class BoundedWaitError extends Error { + constructor( + readonly kind: "timeout" | "aborted", + message: string, + ) { + super(message); + } +} + +function remainingMs(deadline: number): number { + return Math.max(0, deadline - Date.now()); +} + +async function waitBounded( + promise: Promise, + deadline: number, + signal: AbortSignal | undefined, + timeoutMessage: string, +): Promise { + const remaining = remainingMs(deadline); + if (signal?.aborted) throw new BoundedWaitError("aborted", "upload transaction aborted"); + if (remaining === 0) throw new BoundedWaitError("timeout", timeoutMessage); + + let timer: ReturnType | undefined; + let onAbort: (() => void) | undefined; + const boundary = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new BoundedWaitError("timeout", timeoutMessage)), remaining); + if (signal) { + onAbort = () => reject(new BoundedWaitError("aborted", "upload transaction aborted")); + signal.addEventListener("abort", onAbort, { once: true }); + } + }); + try { + return await Promise.race([promise, boundary]); + } finally { + if (timer) clearTimeout(timer); + if (signal && onAbort) signal.removeEventListener("abort", onAbort); + } +} + +function uploadError( + code: RpcError["code"], + message: string, + reason: "file_input_probe_failed" | "file_input_not_activated" | "set_file_input_failed", + phase: UploadPhase, +): RpcError { + return { code, message, data: { reason, phase } }; +} + +function runtimeError(reply: RuntimeReply, fallback: string): Error | null { + if (!reply.exceptionDetails) return null; + return new Error( + reply.exceptionDetails.exception?.description ?? reply.exceptionDetails.text ?? fallback, + ); +} + +export async function uploadThroughActivatedFileInput( + options: FileInputTransactionOptions, +): Promise { + const deadline = Date.now() + options.timeoutMs; + const objectGroup = `bsk-upload-${crypto.randomUUID()}`; + const stateKey = `__bskUpload_${crypto.randomUUID().replaceAll("-", "")}`; + const stateKeyLiteral = JSON.stringify(stateKey); + let probeAttempted = false; + + try { + try { + probeAttempted = true; + const armed = await waitBounded( + sendToCdpTarget(options.cdp, options.target, "Runtime.evaluate", { + expression: `(() => { + const key = ${stateKeyLiteral}; + const owner = globalThis; + const doc = document; + const state = { inputs: [], listener: null, pickerCalls: [], pickers: [] }; + Object.defineProperty(owner, key, { value: state, configurable: true }); + state.listener = event => { + const path = typeof event.composedPath === "function" ? event.composedPath() : []; + const candidate = path[0] || event.target; + if (candidate && candidate.nodeType === 1 && + candidate.localName === "input" && candidate.type === "file") { + if (!state.inputs.includes(candidate)) state.inputs.push(candidate); + event.preventDefault(); + } + }; + doc.addEventListener("click", state.listener, true); + const win = doc.defaultView; + for (const name of ["showOpenFilePicker", "showSaveFilePicker", "showDirectoryPicker"]) { + if (!win || typeof win[name] !== "function") continue; + const hadOwn = Object.prototype.hasOwnProperty.call(win, name); + const descriptor = Object.getOwnPropertyDescriptor(win, name); + state.pickers.push({ name, hadOwn, descriptor }); + Object.defineProperty(win, name, { + configurable: true, + enumerable: descriptor?.enumerable ?? true, + writable: true, + value: () => { + state.pickerCalls.push(name); + return Promise.reject( + new DOMException("Picker intercepted by BrowserSkill", "AbortError") + ); + }, + }); + } + return true; + })()`, + objectGroup, + returnByValue: true, + }), + deadline, + options.signal, + "arming file input probe timed out", + ); + const armError = runtimeError(armed, "failed to arm file input probe"); + if (armError) throw armError; + } catch (err) { + if (err instanceof BoundedWaitError && err.kind === "aborted") throw err; + return uploadError( + err instanceof BoundedWaitError ? "timeout" : "cdp_failed", + err instanceof Error ? err.message : String(err), + "file_input_probe_failed", + "arm_input_probe", + ); + } + + let click: ClickResult | RpcError; + try { + click = await waitBounded( + options.trigger(remainingMs(deadline)), + deadline, + options.signal, + "upload trigger timed out", + ); + } catch (err) { + if (err instanceof BoundedWaitError && err.kind === "aborted") throw err; + return uploadError( + err instanceof BoundedWaitError ? "timeout" : "cdp_failed", + err instanceof Error ? err.message : String(err), + "file_input_probe_failed", + "trigger", + ); + } + if (isRpcError(click)) return click; + + try { + const summary = await waitBounded( + sendToCdpTarget(options.cdp, options.target, "Runtime.evaluate", { + expression: `(() => { + const s = globalThis[${stateKeyLiteral}]; + return s + ? { count: s.inputs.length, multiple: s.inputs[0]?.multiple === true, + pickerCall: s.pickerCalls[0] } + : { count: 0, multiple: false }; + })()`, + returnByValue: true, + }), + deadline, + options.signal, + "resolving activated file input timed out", + ); + const summaryError = runtimeError(summary, "failed to inspect activated file input"); + if (summaryError) throw summaryError; + const value = summary.result?.value as + | { count?: unknown; multiple?: unknown; pickerCall?: unknown } + | undefined; + const count = typeof value?.count === "number" ? value.count : 0; + const multiple = value?.multiple === true; + if (typeof value?.pickerCall === "string") { + return uploadError( + "unsupported", + `upload trigger invoked ${value.pickerCall} instead of an input[type=file]`, + "file_input_not_activated", + "resolve_input", + ); + } + if (count !== 1) { + return uploadError( + "unsupported", + count === 0 + ? "upload trigger did not activate an input[type=file]" + : "upload trigger activated more than one input[type=file]", + "file_input_not_activated", + "resolve_input", + ); + } + if (!multiple && options.files.length !== 1) { + return { code: "invalid_params", message: "file input accepts exactly one file" }; + } + + const input = await waitBounded( + sendToCdpTarget(options.cdp, options.target, "Runtime.evaluate", { + expression: `globalThis[${stateKeyLiteral}]?.inputs[0]`, + objectGroup, + returnByValue: false, + }), + deadline, + options.signal, + "resolving activated file input object timed out", + ); + const inputError = runtimeError(input, "failed to resolve activated file input object"); + if (inputError) throw inputError; + const inputObjectId = input.result?.objectId; + if (!inputObjectId) throw new Error("activated file input returned no objectId"); + + const described = await waitBounded( + sendToCdpTarget<{ node?: { backendNodeId?: number } }>( + options.cdp, + options.target, + "DOM.describeNode", + { objectId: inputObjectId }, + ), + deadline, + options.signal, + "describing activated file input timed out", + ); + const backendNodeId = described.node?.backendNodeId; + if (typeof backendNodeId !== "number") { + throw new Error("DOM.describeNode returned no file input backendNodeId"); + } + + try { + await waitBounded( + sendToCdpTarget(options.cdp, options.target, "DOM.setFileInputFiles", { + files: options.files, + backendNodeId, + }), + deadline, + options.signal, + "setting file input files timed out", + ); + } catch (err) { + if (err instanceof BoundedWaitError && err.kind === "aborted") throw err; + return uploadError( + err instanceof BoundedWaitError ? "timeout" : "cdp_failed", + err instanceof Error ? err.message : String(err), + "set_file_input_failed", + "set_files", + ); + } + return { click, multiple }; + } catch (err) { + if (err instanceof BoundedWaitError && err.kind === "aborted") throw err; + return uploadError( + err instanceof BoundedWaitError ? "timeout" : "cdp_failed", + err instanceof Error ? err.message : String(err), + "file_input_probe_failed", + "resolve_input", + ); + } + } catch (err) { + if (err instanceof BoundedWaitError && err.kind === "aborted") { + return { code: "cancelled", message: "upload transaction aborted" }; + } + throw err; + } finally { + if (probeAttempted) { + try { + await waitBounded( + sendToCdpTarget(options.cdp, options.target, "Runtime.evaluate", { + expression: `(() => { + const key = ${stateKeyLiteral}; + const state = globalThis[key]; + if (state?.listener) { + document.removeEventListener("click", state.listener, true); + } + const win = document.defaultView; + if (win) { + for (const picker of state?.pickers || []) { + try { + if (picker.hadOwn && picker.descriptor) { + Object.defineProperty(win, picker.name, picker.descriptor); + } else { + delete win[picker.name]; + } + } catch {} + } + } + delete globalThis[key]; + })()`, + }), + Date.now() + CLEANUP_TIMEOUT_MS, + undefined, + "cleaning file input probe timed out", + ); + } catch { + // Navigation may have invalidated the object; its document is gone too. + } + } + try { + await waitBounded( + sendToCdpTarget(options.cdp, options.target, "Runtime.releaseObjectGroup", { + objectGroup, + }), + Date.now() + CLEANUP_TIMEOUT_MS, + undefined, + "releasing upload object group timed out", + ); + } catch {} + } +} diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index 182cabd..3ea1373 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -128,7 +128,7 @@ async function wait(ms: number, signal?: AbortSignal): Promise { * `RpcError` if the caller supplied neither (or both), or if neither * lookup matched. */ -async function resolveBackendNode( +export async function resolveBackendNode( cdp: CdpRunner, ctx: SessionContext, target: { tabId: number }, diff --git a/apps/extension/src/tools/upload.ts b/apps/extension/src/tools/upload.ts index e131392..6622bc4 100644 --- a/apps/extension/src/tools/upload.ts +++ b/apps/extension/src/tools/upload.ts @@ -1,11 +1,11 @@ // Upload orchestration: validate the session-scoped request, resolve its click -// target, then delegate the browser protocol transaction to file-chooser.ts. +// target, then delegate the browser protocol transaction to the file-input +// transaction module. -import type { CdpTarget } from "@/browser-driver/frame-graph"; import type { SessionManager } from "@/session-manager/manager"; import type { ClickParams, RpcError, UploadParams, UploadResult } from "@/transport/types"; -import { uploadThroughFileChooser } from "./file-chooser"; -import { handleClick, type InteractionDeps } from "./interaction"; +import { uploadThroughActivatedFileInput } from "./file-input-transaction"; +import { handleClick, type InteractionDeps, resolveBackendNode } from "./interaction"; import { type CdpRunner, enforceAgentWindow, @@ -13,7 +13,6 @@ import { lookupSession, resolveTargetTab, } from "./shared"; -import { resolveSnapshotRef } from "./snapshot-ref"; const DEFAULT_TIMEOUT_MS = 120_000; @@ -21,21 +20,6 @@ export interface UploadDeps extends InteractionDeps { cdp: CdpRunner; } -function chooserTarget( - params: UploadParams, - tabId: number, - manager: SessionManager, -): CdpTarget | RpcError { - const ctx = manager.get(params.session_id); - if (!ctx) return { code: "not_found", message: `session ${params.session_id} unknown` }; - if (params.ref) { - const ref = resolveSnapshotRef(ctx, params.ref, tabId); - if (isRpcError(ref)) return ref; - return { tabId, ...(ref.cdpSessionId ? { sessionId: ref.cdpSessionId } : {}) }; - } - return { tabId }; -} - export async function handleUpload( manager: SessionManager, params: UploadParams, @@ -47,8 +31,6 @@ export async function handleUpload( if (isRpcError(target)) return target; const denied = enforceAgentWindow(ctx, target, "upload"); if (denied) return denied; - if (!deps.cdp?.onEvent) - return { code: "unsupported", message: "CDP event subscription unavailable" }; if ( params.files.length === 0 || params.files.length > 20 || @@ -56,12 +38,12 @@ export async function handleUpload( ) { return { code: "invalid_params", message: "upload requires daemon-staged files" }; } - const cdpTarget = chooserTarget(params, target.tabId, manager); - if (isRpcError(cdpTarget)) return cdpTarget; + const address = await resolveBackendNode(deps.cdp, ctx, target, params, "upload"); + if (isRpcError(address)) return address; const timeoutMs = params.timeout_ms ?? DEFAULT_TIMEOUT_MS; - const transaction = await uploadThroughFileChooser({ + const transaction = await uploadThroughActivatedFileInput({ cdp: deps.cdp, - target: cdpTarget, + target: address.cdpTarget, files: params.files.map((file) => file.staged_path as string), timeoutMs, signal: deps.signal, diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index c161169..4504ff1 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -33,9 +33,10 @@ export type RpcErrorReason = | "restricted_tab_url" | "borrow_conflict" | "screenshot_capture_failed" - | "file_chooser_control_failed" - | "file_chooser_not_opened" - | "unsupported_file_chooser" + | "file_input_probe_failed" + | "file_input_not_activated" + | "set_file_input_failed" + | "download_capture_failed" | "cleanup_failed"; export interface RpcErrorData { @@ -531,7 +532,7 @@ export interface DownloadParams { selector?: string; tab_id?: number; timeout_ms?: number; - staging_path?: string; + browser_relative_dir?: string; } export interface DownloadResult { @@ -542,7 +543,7 @@ export interface DownloadResult { byte_size: number; mime?: string; danger?: string; - staged_path?: string; + browser_path?: string; transfer_id?: string; } diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index 0e5aee1..b13f631 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -210,20 +210,21 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff | Command | Summary | |---------|---------| -| `bsk upload --file ` | Click one file chooser and attach an agent-readable local file (`--file` is repeatable) | +| `bsk upload --file ` | Click one upload trigger and attach an agent-readable local file (`--file` is repeatable) | | `bsk download --out ` | Click one download trigger and copy the single completed file to an exact local path (`--overwrite` is opt-in) | The agent/harness decides whether a file transfer is appropriate and which local path belongs to the task. Treat upload as disclosure of that file to the current website, and download as accepting website-controlled bytes onto the local filesystem. Use only paths that are necessary for the user's bounded goal. -BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted paths reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. Download staging is removed after the CLI copies it; upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive. +BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted capabilities reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. Upload captures the file input actually activated by the requested click and assigns only the staged file paths. Download requires an exact-target browser intent before it claims one Chrome download, routes it through a daemon-minted relative directory, and lets the daemon validate and import it. Upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive. -Do not use `request-help` merely because a native file chooser or browser download is involved; try these commands first. For upload failures, use the structured error instead of retrying blindly: +Do not use `request-help` merely because a native file chooser or browser download is involved; try these commands first. For transfer failures, use the structured error instead of retrying blindly: -- `reason=unsupported_file_chooser` means the page opened a non-input picker such as `window.showOpenFilePicker()`. Do not retry `upload`; if human help is available, call `request-help` and tell the user the exact original file path to choose. The staged daemon path is an internal implementation detail and must not be shown to the user. -- `reason=file_chooser_control_failed` means BrowserSkill could not establish or complete the browser-side transaction. Do not retry the same action blindly; use `request-help` when available. -- `reason=file_chooser_not_opened` means the click completed without opening a chooser. Observe once to verify that the target is the actual upload action (for example, after expanding a menu), then retry with the corrected target or ask for help. +- `reason=file_input_not_activated` means the requested click did not activate exactly one ``; the page may use a non-input picker such as `window.showOpenFilePicker()`. Do not retry blindly. If human help is available, call `request-help` and tell the user the exact original local path to choose. The staged daemon path is internal and must not be shown to the user. +- `reason=file_input_probe_failed` means BrowserSkill could not safely establish the browser-side upload transaction. Do not repeat the same action; use `request-help` when available. +- `reason=set_file_input_failed` means BrowserSkill found the activated file input but Chrome rejected the staged path or assignment. Check the extension's file-URL access permission; otherwise use `request-help`. +- `reason=download_capture_failed` means BrowserSkill could not attribute exactly one completed download to the requested target. Do not retry blindly or accept an unrelated browser download; use `request-help` when available. -If `request-help` returns `outcome="disabled"`, do not retry it. Stop gracefully and report that the workflow requires a non-input file picker or unavailable local file. +If `request-help` returns `outcome="disabled"`, do not retry it. Stop gracefully and report the transfer mechanism that requires human intervention. ### Scripting & timing diff --git a/crates/bsk-cli/src/cli/download.rs b/crates/bsk-cli/src/cli/download.rs index ededf99..863206d 100644 --- a/crates/bsk-cli/src/cli/download.rs +++ b/crates/bsk-cli/src/cli/download.rs @@ -55,7 +55,7 @@ pub fn dispatch(args: DownloadArgs, format: Format) -> Result<(), CliError> { selector, tab_id: args.tab_id, timeout_ms: Some(args.timeout), - staging_path: None, + browser_relative_dir: None, }; let reply: DownloadResult = crate::cli::business_rpc::call( info.sock_path.clone(), @@ -82,7 +82,7 @@ pub fn dispatch(args: DownloadArgs, format: Format) -> Result<(), CliError> { value["path"] = serde_json::json!(args.out.to_string_lossy()); if let Some(obj) = value.as_object_mut() { obj.remove("transfer_id"); - obj.remove("staged_path"); + obj.remove("browser_path"); } println!("{}", serde_json::to_string_pretty(&value).unwrap()); } diff --git a/crates/bsk-cli/src/cli/render_error.rs b/crates/bsk-cli/src/cli/render_error.rs index 2be4682..bb2a0a6 100644 --- a/crates/bsk-cli/src/cli/render_error.rs +++ b/crates/bsk-cli/src/cli/render_error.rs @@ -49,9 +49,10 @@ pub mod reason { pub const TAB_NOT_ACTIVE: &str = "tab_not_active"; pub const BORROW_CONFLICT: &str = "borrow_conflict"; pub const SCREENSHOT_CAPTURE_FAILED: &str = "screenshot_capture_failed"; - pub const FILE_CHOOSER_CONTROL_FAILED: &str = "file_chooser_control_failed"; - pub const FILE_CHOOSER_NOT_OPENED: &str = "file_chooser_not_opened"; - pub const UNSUPPORTED_FILE_CHOOSER: &str = "unsupported_file_chooser"; + pub const FILE_INPUT_PROBE_FAILED: &str = "file_input_probe_failed"; + pub const FILE_INPUT_NOT_ACTIVATED: &str = "file_input_not_activated"; + pub const SET_FILE_INPUT_FAILED: &str = "set_file_input_failed"; + pub const DOWNLOAD_CAPTURE_FAILED: &str = "download_capture_failed"; pub const SESSION_BUSY: &str = crate::rpc_reason::SESSION_BUSY; pub const RECORD_START_PAGE_UNREACHABLE: &str = "record_start_page_unreachable"; } @@ -283,26 +284,33 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend ), exit_code: base.exit_code, }, - (ErrorCode::Unsupported, reason::UNSUPPORTED_FILE_CHOOSER) => RenderInfo { - summary: "the page opened a non-input file picker", + (ErrorCode::Unsupported, reason::FILE_INPUT_NOT_ACTIVATED) => RenderInfo { + summary: "the upload trigger did not activate a file input", hint: Some( - "do not retry upload; use `bsk request-help` and tell the user the exact original local path to select, or stop if human help is disabled", + "the page may use a non-input picker such as showOpenFilePicker(); do not retry blindly — use `bsk request-help` with the exact original local path, or stop if human help is disabled", ), exit_code: base.exit_code, }, - (ErrorCode::Timeout | ErrorCode::CdpFailed, reason::FILE_CHOOSER_CONTROL_FAILED) => { + (ErrorCode::Timeout | ErrorCode::CdpFailed, reason::FILE_INPUT_PROBE_FAILED) => { RenderInfo { - summary: "the browser file-chooser transaction could not complete", + summary: "the browser upload transaction could not be established", hint: Some( "do not retry the same upload blindly; use `bsk request-help` when human help is available", ), exit_code: base.exit_code, } } - (ErrorCode::Timeout, reason::FILE_CHOOSER_NOT_OPENED) => RenderInfo { - summary: "the upload action did not open a file chooser", + (ErrorCode::Timeout | ErrorCode::CdpFailed, reason::SET_FILE_INPUT_FAILED) => RenderInfo { + summary: "the browser could not attach the staged file to the input", hint: Some( - "observe once after expanding any upload menu, then retry with the actual upload action or use `bsk request-help`", + "check that BrowserSkill has Chrome's 'Allow access to file URLs' permission; otherwise use `bsk request-help`", + ), + exit_code: base.exit_code, + }, + (ErrorCode::CdpFailed, reason::DOWNLOAD_CAPTURE_FAILED) => RenderInfo { + summary: "the browser download could not be attributed or completed", + hint: Some( + "do not retry blindly; the target browser may not expose a reliable download intent signal", ), exit_code: base.exit_code, }, @@ -446,24 +454,26 @@ mod tests { } #[test] - fn file_chooser_reasons_render_actionable_fallbacks() { - let unsupported = serde_json::json!({ "reason": reason::UNSUPPORTED_FILE_CHOOSER }); + fn file_transfer_reasons_render_actionable_fallbacks() { + let unsupported = serde_json::json!({ "reason": reason::FILE_INPUT_NOT_ACTIVATED }); let info = info_for_error(ErrorCode::Unsupported, Some(&unsupported)); - assert_eq!(info.summary, "the page opened a non-input file picker"); - assert!(info.hint.unwrap().contains("exact original local path")); - - let control = serde_json::json!({ "reason": reason::FILE_CHOOSER_CONTROL_FAILED }); - let info = info_for_error(ErrorCode::Timeout, Some(&control)); - assert!(info.summary.contains("transaction could not complete")); - assert!(info.hint.unwrap().contains("do not retry")); - - let not_opened = serde_json::json!({ "reason": reason::FILE_CHOOSER_NOT_OPENED }); - let info = info_for_error(ErrorCode::Timeout, Some(¬_opened)); assert_eq!( info.summary, - "the upload action did not open a file chooser" + "the upload trigger did not activate a file input" ); - assert!(info.hint.unwrap().contains("expanding any upload menu")); + assert!(info.hint.unwrap().contains("exact original local path")); + + let control = serde_json::json!({ "reason": reason::FILE_INPUT_PROBE_FAILED }); + let info = info_for_error(ErrorCode::Timeout, Some(&control)); + assert!( + info.summary + .contains("transaction could not be established") + ); + assert!(info.hint.unwrap().contains("do not retry")); + + let download = serde_json::json!({ "reason": reason::DOWNLOAD_CAPTURE_FAILED }); + let info = info_for_error(ErrorCode::CdpFailed, Some(&download)); + assert!(info.summary.contains("download could not be attributed")); } #[test] diff --git a/crates/bsk-cli/src/daemon/file_transfer.rs b/crates/bsk-cli/src/daemon/file_transfer.rs index a3084eb..71123e1 100644 --- a/crates/bsk-cli/src/daemon/file_transfer.rs +++ b/crates/bsk-cli/src/daemon/file_transfer.rs @@ -43,7 +43,7 @@ struct Entry { #[derive(Debug)] pub struct DownloadStaging { pub transfer_id: String, - pub directory: PathBuf, + pub browser_relative_dir: String, } #[derive(Debug)] @@ -247,32 +247,70 @@ impl TransferRegistry { }, ); Ok(DownloadStaging { + browser_relative_dir: format!("BrowserSkill/{id}"), transfer_id: id, - directory: dir, }) } - pub fn finish_download(&self, id: &str, reported_path: &Path) -> Result { - let canonical = reported_path.canonicalize().map_err(io_error)?; + pub fn import_download(&self, id: &str, reported_path: &Path) -> Result { let mut entries = self.entries.lock().unwrap(); let entry = entries .get_mut(id) .ok_or_else(|| not_found("download transfer not found"))?; - let dir = entry.path.canonicalize().map_err(io_error)?; - if entry.direction != Direction::Download || !canonical.starts_with(&dir) { - return Err(permission("download escaped its staging directory")); + if entry.direction != Direction::Download || !reported_path.is_absolute() { + return Err(permission( + "download path is outside its browser capability", + )); } + let expected_parent = Path::new("BrowserSkill").join(id); + let reported_parent = reported_path + .parent() + .ok_or_else(|| permission("download path has no parent directory"))?; + if !reported_parent.ends_with(&expected_parent) { + return Err(permission( + "download escaped its browser capability directory", + )); + } + for path in [reported_path, reported_parent] { + if fs::symlink_metadata(path) + .map_err(io_error)? + .file_type() + .is_symlink() + { + return Err(permission("download capability path contains a symlink")); + } + } + let canonical = reported_path.canonicalize().map_err(io_error)?; let meta = fs::metadata(&canonical).map_err(io_error)?; if !meta.is_file() || meta.len() > MAX_TRANSFER_BYTES { return Err(invalid( "download is not a regular file or exceeds the transfer limit", )); } - entry.path = canonical; - entry.written = meta.len(); - entry.expected_size = Some(meta.len()); + + let destination = entry.path.join("payload"); + let source = File::open(&canonical).map_err(io_error)?; + let mut destination_file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&destination) + .map_err(io_error)?; + let copied = std::io::copy( + &mut source.take(MAX_TRANSFER_BYTES.saturating_add(1)), + &mut destination_file, + ) + .map_err(io_error)?; + if copied > MAX_TRANSFER_BYTES { + let _ = fs::remove_file(&destination); + return Err(invalid("download exceeds the transfer limit")); + } + entry.path = destination; + entry.written = copied; + entry.expected_size = Some(copied); entry.ready = true; - Ok(meta.len()) + let _ = fs::remove_file(&canonical); + let _ = fs::remove_dir(reported_parent); + Ok(copied) } pub fn read_chunk(&self, p: TransferChunkParams) -> Result { @@ -430,25 +468,45 @@ mod tests { } #[test] - fn download_must_finish_inside_its_minted_directory() { + fn download_must_import_from_its_browser_capability_directory() { let (temp, registry) = registry(); let staging = registry.begin_download("s1").unwrap(); let outside = temp.path().join("outside.bin"); fs::write(&outside, b"secret").unwrap(); assert!( registry - .finish_download(&staging.transfer_id, &outside) + .import_download(&staging.transfer_id, &outside) .is_err() ); - let inside = staging.directory.join("result.bin"); + let browser_dir = temp + .path() + .join("Downloads") + .join("BrowserSkill") + .join(&staging.transfer_id); + fs::create_dir_all(&browser_dir).unwrap(); + let inside = browser_dir.join("result.bin"); fs::write(&inside, b"result").unwrap(); assert_eq!( registry - .finish_download(&staging.transfer_id, &inside) + .import_download(&staging.transfer_id, &inside) .unwrap(), 6 ); + assert!(!inside.exists()); + let chunk = registry + .read_chunk(TransferChunkParams { + transfer_id: staging.transfer_id, + offset: 0, + data_base64: String::new(), + }) + .unwrap(); + assert_eq!( + base64::engine::general_purpose::STANDARD + .decode(chunk.data_base64.unwrap()) + .unwrap(), + b"result" + ); } #[test] @@ -456,9 +514,11 @@ mod tests { let (_temp, registry) = registry(); let first = registry.begin_download("s1").unwrap(); let second = registry.begin_download("s2").unwrap(); + let first_directory = registry.root.join(&first.transfer_id); + let second_directory = registry.root.join(&second.transfer_id); registry.release_session("s1"); - assert!(!first.directory.exists()); - assert!(second.directory.exists()); + assert!(!first_directory.exists()); + assert!(second_directory.exists()); } #[test] diff --git a/crates/bsk-cli/src/daemon/ipc.rs b/crates/bsk-cli/src/daemon/ipc.rs index 64a03d3..9a6d0bd 100644 --- a/crates/bsk-cli/src/daemon/ipc.rs +++ b/crates/bsk-cli/src/daemon/ipc.rs @@ -381,7 +381,7 @@ async fn handle_tool_dispatch( Ok(v) => v, Err(err) => return ResponseBody::Err(err), }; - download.staging_path = Some(staging.directory.to_string_lossy().into_owned()); + download.browser_relative_dir = Some(staging.browser_relative_dir); download_transfer_id = Some(staging.transfer_id); params = serde_json::to_value(download).unwrap_or(Value::Null); } @@ -416,19 +416,19 @@ async fn handle_tool_dispatch( }); } }; - let Some(path) = result.staged_path.take() else { + let Some(path) = result.browser_path.take() else { state .transfers .release(TransferIdParams { transfer_id: id }); return ResponseBody::Err(RpcError { code: ErrorCode::ProtocolError, - message: "tool.download returned no staged_path".into(), + message: "tool.download returned no browser_path".into(), data: None, }); }; match state .transfers - .finish_download(&id, std::path::Path::new(&path)) + .import_download(&id, std::path::Path::new(&path)) { Ok(size) => { result.byte_size = size; diff --git a/crates/bsk-protocol/schema/tool_download_params.json b/crates/bsk-protocol/schema/tool_download_params.json index ec3c844..b6487cc 100644 --- a/crates/bsk-protocol/schema/tool_download_params.json +++ b/crates/bsk-protocol/schema/tool_download_params.json @@ -6,6 +6,13 @@ "session_id" ], "properties": { + "browser_relative_dir": { + "description": "Daemon-injected relative directory beneath Chrome's Downloads root.", + "type": [ + "string", + "null" + ] + }, "ref": { "type": [ "string", @@ -21,13 +28,6 @@ "session_id": { "type": "string" }, - "staging_path": { - "description": "Daemon-injected destination directory for the browser process.", - "type": [ - "string", - "null" - ] - }, "tab_id": { "type": [ "integer", diff --git a/crates/bsk-protocol/schema/tool_download_result.json b/crates/bsk-protocol/schema/tool_download_result.json index ea2ac7c..726ca50 100644 --- a/crates/bsk-protocol/schema/tool_download_result.json +++ b/crates/bsk-protocol/schema/tool_download_result.json @@ -8,6 +8,13 @@ "tab_id" ], "properties": { + "browser_path": { + "description": "Extension-internal completed Chrome download path, stripped by the daemon.", + "type": [ + "string", + "null" + ] + }, "byte_size": { "type": "integer", "format": "uint64", @@ -25,13 +32,6 @@ "null" ] }, - "staged_path": { - "description": "Extension-internal completed path, stripped by the daemon.", - "type": [ - "string", - "null" - ] - }, "suggested_filename": { "type": "string" }, diff --git a/crates/bsk-protocol/src/tools/file_transfer.rs b/crates/bsk-protocol/src/tools/file_transfer.rs index 2c2f5b8..f2517ce 100644 --- a/crates/bsk-protocol/src/tools/file_transfer.rs +++ b/crates/bsk-protocol/src/tools/file_transfer.rs @@ -60,9 +60,9 @@ pub struct DownloadParams { pub tab_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub timeout_ms: Option, - /// Daemon-injected destination directory for the browser process. + /// Daemon-injected relative directory beneath Chrome's Downloads root. #[serde(default, skip_serializing_if = "Option::is_none")] - pub staging_path: Option, + pub browser_relative_dir: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] @@ -78,9 +78,9 @@ pub struct DownloadResult { pub mime: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub danger: Option, - /// Extension-internal completed path, stripped by the daemon. + /// Extension-internal completed Chrome download path, stripped by the daemon. #[serde(default, skip_serializing_if = "Option::is_none")] - pub staged_path: Option, + pub browser_path: Option, /// Opaque id returned by the daemon to the CLI. #[serde(default, skip_serializing_if = "Option::is_none")] pub transfer_id: Option, @@ -162,9 +162,9 @@ mod tests { selector: Some("#export".into()), tab_id: None, timeout_ms: None, - staging_path: None, + browser_relative_dir: None, }) .unwrap(); - assert!(value.get("staging_path").is_none()); + assert!(value.get("browser_relative_dir").is_none()); } } diff --git a/docs/architecture.md b/docs/architecture.md index 2560016..d86748d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -147,8 +147,8 @@ flowchart LR - The invoking agent/harness decides whether a transfer is authorized and supplies the task-local source or destination path. - The CLI is the only component that reads an upload source or writes the final download destination. The extension never receives either agent-facing path. -- The daemon issues opaque, session-scoped transfer IDs and stages bounded chunks in a private runtime directory. It injects only its own staging paths into extension RPCs and validates that a completed download remains inside its minted directory. -- The extension only performs the browser-side mechanism: handle one file-chooser transaction or capture one download, scoped to the requested session and exact Agent Window target. Automatic upload is limited to choosers backed by ``; non-input pickers such as `window.showOpenFilePicker()` return a structured unsupported result for the agent to handle. The extension does not classify content or make policy decisions. +- The daemon issues opaque, session-scoped transfer IDs and stages bounded chunks in a private runtime directory. For upload it injects only its private staged paths. For download it mints a relative Chrome directory capability, validates the reported completed path against that capability, rejects symlinks and oversized/non-regular files, then imports the bytes into private staging. +- The extension only performs the browser-side mechanism, scoped to the requested session and exact Agent Window target. Upload installs a transaction-scoped DOM guard that records the one `` actually activated by the requested click while cancelling its native default action, then uses `DOM.setFileInputFiles`. File System Access picker entry points are intercepted only for that transaction and return a structured fallback. Download requires an exact-target `Page.downloadWillBegin` intent before a matching `chrome.downloads` item is routed into the daemon-minted relative directory. The extension does not classify content or make policy decisions. - Download staging is released after the CLI commits the file. Upload staging remains until session teardown because the page may read an attached file only on a later form submission. All staging is released on session stop/browser disconnect and on daemon startup after a crash. Existing download destinations are not overwritten unless the CLI caller explicitly opts in. ## Repository layout diff --git a/skill/SKILL.md b/skill/SKILL.md index 0e5aee1..b13f631 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -210,20 +210,21 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff | Command | Summary | |---------|---------| -| `bsk upload --file ` | Click one file chooser and attach an agent-readable local file (`--file` is repeatable) | +| `bsk upload --file ` | Click one upload trigger and attach an agent-readable local file (`--file` is repeatable) | | `bsk download --out ` | Click one download trigger and copy the single completed file to an exact local path (`--overwrite` is opt-in) | The agent/harness decides whether a file transfer is appropriate and which local path belongs to the task. Treat upload as disclosure of that file to the current website, and download as accepting website-controlled bytes onto the local filesystem. Use only paths that are necessary for the user's bounded goal. -BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted paths reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. Download staging is removed after the CLI copies it; upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive. +BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted capabilities reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. Upload captures the file input actually activated by the requested click and assigns only the staged file paths. Download requires an exact-target browser intent before it claims one Chrome download, routes it through a daemon-minted relative directory, and lets the daemon validate and import it. Upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive. -Do not use `request-help` merely because a native file chooser or browser download is involved; try these commands first. For upload failures, use the structured error instead of retrying blindly: +Do not use `request-help` merely because a native file chooser or browser download is involved; try these commands first. For transfer failures, use the structured error instead of retrying blindly: -- `reason=unsupported_file_chooser` means the page opened a non-input picker such as `window.showOpenFilePicker()`. Do not retry `upload`; if human help is available, call `request-help` and tell the user the exact original file path to choose. The staged daemon path is an internal implementation detail and must not be shown to the user. -- `reason=file_chooser_control_failed` means BrowserSkill could not establish or complete the browser-side transaction. Do not retry the same action blindly; use `request-help` when available. -- `reason=file_chooser_not_opened` means the click completed without opening a chooser. Observe once to verify that the target is the actual upload action (for example, after expanding a menu), then retry with the corrected target or ask for help. +- `reason=file_input_not_activated` means the requested click did not activate exactly one ``; the page may use a non-input picker such as `window.showOpenFilePicker()`. Do not retry blindly. If human help is available, call `request-help` and tell the user the exact original local path to choose. The staged daemon path is internal and must not be shown to the user. +- `reason=file_input_probe_failed` means BrowserSkill could not safely establish the browser-side upload transaction. Do not repeat the same action; use `request-help` when available. +- `reason=set_file_input_failed` means BrowserSkill found the activated file input but Chrome rejected the staged path or assignment. Check the extension's file-URL access permission; otherwise use `request-help`. +- `reason=download_capture_failed` means BrowserSkill could not attribute exactly one completed download to the requested target. Do not retry blindly or accept an unrelated browser download; use `request-help` when available. -If `request-help` returns `outcome="disabled"`, do not retry it. Stop gracefully and report that the workflow requires a non-input file picker or unavailable local file. +If `request-help` returns `outcome="disabled"`, do not retry it. Stop gracefully and report the transfer mechanism that requires human intervention. ### Scripting & timing From 76d447caa071043d0d084bb858c8ce8e57c17e7f Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Fri, 21 Aug 2026 14:11:13 +0800 Subject: [PATCH 04/12] fix(bsk): upload function safety --- crates/bsk-cli/src/daemon/file_transfer.rs | 98 +++++++++++++++++++++- 1 file changed, 95 insertions(+), 3 deletions(-) diff --git a/crates/bsk-cli/src/daemon/file_transfer.rs b/crates/bsk-cli/src/daemon/file_transfer.rs index 71123e1..354eec1 100644 --- a/crates/bsk-cli/src/daemon/file_transfer.rs +++ b/crates/bsk-cli/src/daemon/file_transfer.rs @@ -7,7 +7,7 @@ use std::collections::HashMap; use std::fs::{self, File, OpenOptions}; use std::io::{Read, Seek, SeekFrom, Write}; -use std::path::{Path, PathBuf}; +use std::path::{Component, Path, PathBuf}; use std::sync::Mutex; use base64::Engine; @@ -100,6 +100,7 @@ impl TransferRegistry { pub fn begin_upload(&self, p: TransferBeginParams) -> Result { self.ensure_root() .map_err(|err| protocol_error(err.to_string()))?; + let basename = safe_upload_basename(&p.name)?; if p.byte_size > MAX_TRANSFER_BYTES { return Err(invalid(format!( "file size {} exceeds transfer limit {}", @@ -122,12 +123,13 @@ impl TransferRegistry { let dir = self.root.join(&id); fs::create_dir(&dir).map_err(io_error)?; set_private_dir(&dir).map_err(io_error)?; - let path = dir.join("payload"); - OpenOptions::new() + let path = dir.join(basename); + let file = OpenOptions::new() .write(true) .create_new(true) .open(&path) .map_err(io_error)?; + set_private_file(&file).map_err(io_error)?; entries.insert( id.clone(), Entry { @@ -380,6 +382,26 @@ fn set_private_dir(path: &Path) -> std::io::Result<()> { Ok(()) } +fn set_private_file(file: &File) -> std::io::Result<()> { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + file.set_permissions(fs::Permissions::from_mode(0o600))?; + } + Ok(()) +} + +fn safe_upload_basename(name: &str) -> Result<&str, RpcError> { + if name.is_empty() || name.contains(['/', '\\', '\0']) { + return Err(invalid("upload name must be a safe basename")); + } + let mut components = Path::new(name).components(); + match (components.next(), components.next()) { + (Some(Component::Normal(_)), None) => Ok(name), + _ => Err(invalid("upload name must be a safe basename")), + } +} + fn invalid(message: impl Into) -> RpcError { RpcError { code: ErrorCode::InvalidParams, @@ -464,9 +486,79 @@ mod tests { .unwrap() .try_into() .unwrap(); + assert_eq!(path.file_name().unwrap(), "image.png"); assert_eq!(fs::read(path).unwrap(), b"abc"); } + #[test] + fn upload_name_must_be_one_safe_path_component() { + let (_temp, registry) = registry(); + for name in ["", ".", "..", "../secret", "folder/file", "folder\\file"] { + let result = registry.begin_upload(TransferBeginParams { + session_id: "s1".into(), + name: name.into(), + byte_size: 0, + }); + assert!(result.is_err(), "accepted unsafe upload name"); + } + assert!(registry.entries.lock().unwrap().is_empty()); + assert!(fs::read_dir(®istry.root).unwrap().next().is_none()); + } + + #[test] + fn same_upload_names_are_isolated_by_transfer_directory() { + let (_temp, registry) = registry(); + let first = registry + .begin_upload(TransferBeginParams { + session_id: "s1".into(), + name: "image.png".into(), + byte_size: 0, + }) + .unwrap(); + let second = registry + .begin_upload(TransferBeginParams { + session_id: "s1".into(), + name: "image.png".into(), + byte_size: 0, + }) + .unwrap(); + let entries = registry.entries.lock().unwrap(); + let first_path = &entries[&first.transfer_id].path; + let second_path = &entries[&second.transfer_id].path; + assert_ne!(first_path.parent(), second_path.parent()); + assert_eq!(first_path.file_name().unwrap(), "image.png"); + assert_eq!(second_path.file_name().unwrap(), "image.png"); + } + + #[cfg(unix)] + #[test] + fn upload_staging_is_private() { + use std::os::unix::fs::PermissionsExt; + + let (_temp, registry) = registry(); + let begin = registry + .begin_upload(TransferBeginParams { + session_id: "s1".into(), + name: "private.png".into(), + byte_size: 0, + }) + .unwrap(); + let entries = registry.entries.lock().unwrap(); + let path = &entries[&begin.transfer_id].path; + assert_eq!( + fs::metadata(path.parent().unwrap()) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o700 + ); + assert_eq!( + fs::metadata(path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + #[test] fn download_must_import_from_its_browser_capability_directory() { let (temp, registry) = registry(); From a86cbf724f207e3c8163d67052bb20c95ee8e152 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Fri, 21 Aug 2026 14:32:29 +0800 Subject: [PATCH 05/12] fix(bsk): fix ci --- crates/bsk-cli/src/cli/download.rs | 4 ++-- crates/bsk-cli/src/cli/upload.rs | 14 +++++++------- crates/bsk-cli/src/daemon/file_transfer.rs | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/crates/bsk-cli/src/cli/download.rs b/crates/bsk-cli/src/cli/download.rs index 863206d..043b4d1 100644 --- a/crates/bsk-cli/src/cli/download.rs +++ b/crates/bsk-cli/src/cli/download.rs @@ -91,7 +91,7 @@ pub fn dispatch(args: DownloadArgs, format: Format) -> Result<(), CliError> { Ok(()) } -fn write_transfer(sock: &PathBuf, id: &str, out: &Path, overwrite: bool) -> Result<(), CliError> { +fn write_transfer(sock: &Path, id: &str, out: &Path, overwrite: bool) -> Result<(), CliError> { let parent = out .parent() .filter(|p| !p.as_os_str().is_empty()) @@ -107,7 +107,7 @@ fn write_transfer(sock: &PathBuf, id: &str, out: &Path, overwrite: bool) -> Resu let mut offset = 0u64; loop { let chunk: TransferChunkResult = crate::cli::business_rpc::call( - sock.clone(), + sock.to_path_buf(), "transfer-read", Method::TransferRead, Some(TransferChunkParams { diff --git a/crates/bsk-cli/src/cli/upload.rs b/crates/bsk-cli/src/cli/upload.rs index ac408e2..9614e0b 100644 --- a/crates/bsk-cli/src/cli/upload.rs +++ b/crates/bsk-cli/src/cli/upload.rs @@ -2,7 +2,7 @@ use std::fs::File; use std::io::Read; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::time::Duration; use anyhow::Context; @@ -93,7 +93,7 @@ pub fn dispatch(args: UploadArgs, format: Format) -> Result<(), CliError> { Ok(()) } -fn stage_file(sock: &PathBuf, session: &str, path: &PathBuf) -> Result<(String, String), CliError> { +fn stage_file(sock: &Path, session: &str, path: &PathBuf) -> Result<(String, String), CliError> { let mut file = File::open(path) .with_context(|| format!("open upload file {}", path.display())) .map_err(CliError::Local)?; @@ -114,7 +114,7 @@ fn stage_file(sock: &PathBuf, session: &str, path: &PathBuf) -> Result<(String, .ok_or_else(|| CliError::Local(anyhow::anyhow!("upload file has no valid basename")))? .to_string(); let begin: TransferBeginResult = crate::cli::business_rpc::call( - sock.clone(), + sock.to_path_buf(), "transfer-begin", Method::TransferBegin, Some(TransferBeginParams { @@ -133,7 +133,7 @@ fn stage_file(sock: &PathBuf, session: &str, path: &PathBuf) -> Result<(String, break; } let reply: TransferChunkResult = crate::cli::business_rpc::call( - sock.clone(), + sock.to_path_buf(), "transfer-chunk", Method::TransferChunk, Some(TransferChunkParams { @@ -146,7 +146,7 @@ fn stage_file(sock: &PathBuf, session: &str, path: &PathBuf) -> Result<(String, offset = reply.next_offset; } let _: TransferReadyResult = crate::cli::business_rpc::call( - sock.clone(), + sock.to_path_buf(), "transfer-finish", Method::TransferFinish, Some(TransferIdParams { @@ -163,9 +163,9 @@ fn stage_file(sock: &PathBuf, session: &str, path: &PathBuf) -> Result<(String, Ok((begin.transfer_id, name)) } -fn release(sock: &PathBuf, id: &str) -> Result { +fn release(sock: &Path, id: &str) -> Result { crate::cli::business_rpc::call( - sock.clone(), + sock.to_path_buf(), "transfer-release", Method::TransferRelease, Some(TransferIdParams { diff --git a/crates/bsk-cli/src/daemon/file_transfer.rs b/crates/bsk-cli/src/daemon/file_transfer.rs index 354eec1..b63ca50 100644 --- a/crates/bsk-cli/src/daemon/file_transfer.rs +++ b/crates/bsk-cli/src/daemon/file_transfer.rs @@ -461,7 +461,7 @@ mod tests { .unwrap(); assert!( registry - .resolve_uploads("s1", &[begin.transfer_id.clone()]) + .resolve_uploads("s1", std::slice::from_ref(&begin.transfer_id)) .is_err() ); registry @@ -478,7 +478,7 @@ mod tests { .unwrap(); assert!( registry - .resolve_uploads("s2", &[begin.transfer_id.clone()]) + .resolve_uploads("s2", std::slice::from_ref(&begin.transfer_id)) .is_err() ); let [path] = registry From bf59753aa3113bfaa56f92e21015bfb9b5453543 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Fri, 21 Aug 2026 16:31:47 +0800 Subject: [PATCH 06/12] feat(bsk): harden browser file transfer transactions --- apps/extension/PRIVACY.md | 2 +- .../src/tools/__tests__/dispatcher.test.ts | 13 + .../src/tools/__tests__/file-transfer.test.ts | 300 ++++++++++- apps/extension/src/tools/download-capture.ts | 256 ++++++--- apps/extension/src/tools/download.ts | 23 +- apps/extension/src/tools/errors.ts | 28 +- .../src/tools/file-input-transaction.ts | 494 +++++++++++------- apps/extension/src/tools/interaction.ts | 46 +- apps/extension/src/tools/upload.ts | 23 +- apps/extension/src/transport/types.ts | 9 + crates/bsk-cli/Cargo.toml | 1 + crates/bsk-cli/skill/SKILL.md | 5 +- crates/bsk-cli/src/cli/atomic_output.rs | 75 +++ crates/bsk-cli/src/cli/download.rs | 20 +- crates/bsk-cli/src/cli/mod.rs | 1 + crates/bsk-cli/src/cli/render_error.rs | 25 + crates/bsk-cli/src/cli/upload.rs | 65 +-- crates/bsk-cli/src/daemon/file_transfer.rs | 133 ++++- crates/bsk-cli/src/daemon/ipc.rs | 3 +- crates/bsk-cli/src/daemon/queue.rs | 253 ++++++++- .../schema/tool_download_params.json | 9 + .../bsk-protocol/src/tools/file_transfer.rs | 6 + docs/architecture.md | 9 +- skill/SKILL.md | 5 +- 24 files changed, 1397 insertions(+), 407 deletions(-) create mode 100644 crates/bsk-cli/src/cli/atomic_output.rs diff --git a/apps/extension/PRIVACY.md b/apps/extension/PRIVACY.md index 38e7968..bfe4675 100644 --- a/apps/extension/PRIVACY.md +++ b/apps/extension/PRIVACY.md @@ -51,7 +51,7 @@ The Extension requests the following Chrome permissions. Each is used solely for - **`alarms`** — Periodically wake the service worker to keep the local WebSocket connection alive. - **`idle`** — Detect when the device returns from idle/locked so the Extension can promptly re-establish the local WebSocket connection after the machine wakes. No idle data is stored or transmitted. - **`notifications`** — Show a system notification to obtain user approval before the agent borrows a user-owned tab. -- **`downloads`** — Observe and, on cancellation, stop the one browser download initiated by an active `bsk download` command. It is not used to enumerate download history. +- **`downloads`** — Correlate and route the one browser download initiated by an active `bsk download` command. If that claimed transaction fails, BrowserSkill cancels an in-progress file or removes its completed temporary browser file. It is not used to enumerate download history or alter unclaimed downloads. - **`storage`** — Persist a random instance ID and optional label in `chrome.storage.local`. - **Host permission ``** — Inject a small status overlay (showing "Agent Active") on pages controlled by the agent, and enable automation across whatever sites the user directs the agent to. The Extension does **not** read or transmit page content from sites the agent is not actively driving. diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index 113fc20..ee566f5 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -258,7 +258,20 @@ describe("ToolDispatcher", () => { if (method === "DOM.getContentQuads") { return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] } as T; } + if (method === "DOM.resolveNode") { + return { object: { objectId: "trigger-object" } } as T; + } if (method === "DOM.describeNode") return { node: { backendNodeId: 456 } } as T; + if (method === "Runtime.callFunctionOn") { + const declaration = (params as { functionDeclaration?: string }).functionDeclaration ?? ""; + if (declaration.includes("count: state.inputs.length")) { + return { result: { value: { count: 1, multiple: false } } } as T; + } + if (declaration.includes("inputs[0]")) { + return { result: { objectId: "input-object" } } as T; + } + return { result: { value: true } } as T; + } if (method === "Runtime.evaluate") { const expression = (params as { expression?: string }).expression ?? ""; if (expression.includes("overlayDetails")) { diff --git a/apps/extension/src/tools/__tests__/file-transfer.test.ts b/apps/extension/src/tools/__tests__/file-transfer.test.ts index 75d0b7e..81dbe01 100644 --- a/apps/extension/src/tools/__tests__/file-transfer.test.ts +++ b/apps/extension/src/tools/__tests__/file-transfer.test.ts @@ -3,6 +3,7 @@ import { SessionManager } from "@/session-manager/manager"; import { type DownloadsApi, handleDownload } from "../download"; import { captureBrowserDownload } from "../download-capture"; import { uploadThroughActivatedFileInput } from "../file-input-transaction"; +import type { ResolvedActionTarget } from "../interaction"; import type { CdpRunner } from "../shared"; import { handleUpload } from "../upload"; @@ -36,37 +37,76 @@ function fakeEvent unknown>() { }; } -function uploadCdp(options: { inputCount?: number; multiple?: boolean; pickerCall?: string } = {}) { +function actionTarget(frameId?: string): ResolvedActionTarget { + return { + tab: { tabId: 4, windowId: 100, active: true }, + backendNodeId: 123, + cdpTarget: { tabId: 4 }, + ...(frameId ? { frameId } : {}), + usedRef: "e3", + }; +} + +function uploadCdp( + options: { + inputCount?: number; + multiple?: boolean; + chooser?: { frameId?: string; backendNodeId?: number; mode?: string }; + pendingResolve?: boolean; + } = {}, +) { const calls: Array<{ method: string; params?: object }> = []; + let cdpEvent: Parameters>[0] | undefined; const send = vi.fn(async (_tabId: number, method: string, params?: object) => { calls.push({ method, params }); + if (method === "Page.setInterceptFileChooserDialog") return {}; if (method === "Page.getLayoutMetrics") return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] }; - if (method === "Runtime.evaluate") { - const expression = (params as { expression?: string }).expression ?? ""; - if (expression.includes("count:")) { + if (method === "DOM.resolveNode") { + if (options.pendingResolve) return new Promise(() => {}); + return { object: { objectId: "trigger-object" } }; + } + if (method === "Runtime.callFunctionOn") { + const declaration = (params as { functionDeclaration?: string }).functionDeclaration ?? ""; + if (declaration.includes("Object.defineProperty")) return { result: { value: true } }; + if (declaration.includes("count: state.inputs.length")) { return { result: { value: { count: options.inputCount ?? 1, multiple: options.multiple ?? true, - pickerCall: options.pickerCall, }, }, }; } - if (expression.includes("?.inputs[0]")) { + if (declaration.includes("inputs[0]")) { return { result: { objectId: "input-object" } }; } return { result: { value: true } }; } if (method === "DOM.describeNode") return { node: { backendNodeId: 456 } }; + if ( + method === "Input.dispatchMouseEvent" && + (params as { type?: string }).type === "mousePressed" && + options.chooser + ) { + cdpEvent?.({ tabId: 4 }, "Page.fileChooserOpened", options.chooser); + } return {}; }); + const cdp: CdpRunner = { + send: send as unknown as CdpRunner["send"], + onEvent: (handler) => { + cdpEvent = handler; + return { dispose: vi.fn() }; + }, + }; return { calls, - cdp: { send: send as unknown as CdpRunner["send"] } satisfies CdpRunner, + emitChooser: (event: { frameId?: string; backendNodeId?: number; mode?: string }) => + cdpEvent?.({ tabId: 4 }, "Page.fileChooserOpened", event), + cdp, }; } @@ -91,15 +131,14 @@ describe("file transfer tools", () => { ); expect(result).toMatchObject({ tab_id: 4, file_names: ["one.png", "two.png"] }); - const armCall = calls.find( - (call) => - call.method === "Runtime.evaluate" && - (call.params as { expression?: string }).expression?.includes("showOpenFilePicker"), - ); - expect(armCall).toBeDefined(); - expect((armCall?.params as { expression?: string }).expression).toContain( - "event.preventDefault()", - ); + expect(calls[0]).toEqual({ + method: "Page.setInterceptFileChooserDialog", + params: { enabled: true }, + }); + expect(calls).toContainEqual({ + method: "Page.setInterceptFileChooserDialog", + params: { enabled: false, cancel: true }, + }); expect(calls).toContainEqual({ method: "DOM.setFileInputFiles", params: { files: ["/private/stage/one", "/private/stage/two"], backendNodeId: 456 }, @@ -110,7 +149,7 @@ describe("file transfer tools", () => { const { cdp } = uploadCdp({ inputCount: 0 }); const result = await uploadThroughActivatedFileInput({ cdp, - target: { tabId: 4 }, + actionTarget: actionTarget(), files: ["/private/stage/one"], timeoutMs: 100, trigger: async () => ({ tab_id: 4, x: 10, y: 10 }), @@ -122,31 +161,80 @@ describe("file transfer tools", () => { }); }); - it("reports a File System Access picker without waiting for a timeout", async () => { - const { cdp } = uploadCdp({ inputCount: 0, pickerCall: "showOpenFilePicker" }); + it("fails before clicking when chooser interception is unavailable", async () => { + const trigger = vi.fn(async () => ({ tab_id: 4, x: 10, y: 10 })); + const cdp: CdpRunner = { + send: vi.fn(async (_tabId: number, method: string) => { + if (method === "Page.setInterceptFileChooserDialog") { + throw new Error("method unavailable"); + } + return {}; + }) as CdpRunner["send"], + }; + const result = await uploadThroughActivatedFileInput({ cdp, - target: { tabId: 4 }, + actionTarget: actionTarget(), files: ["/private/stage/one"], timeoutMs: 100, - trigger: async () => ({ tab_id: 4, x: 10, y: 10 }), + trigger, + }); + + expect(result).toMatchObject({ + code: "cdp_failed", + data: { effect_state: "none", phase: "arm_interception" }, + }); + expect(trigger).not.toHaveBeenCalled(); + }); + + it("uses an exact chooser event as an independent input-location signal", async () => { + const { cdp, calls, emitChooser } = uploadCdp({ inputCount: 0 }); + const result = await uploadThroughActivatedFileInput({ + cdp, + actionTarget: actionTarget("f1"), + files: ["/private/stage/one"], + timeoutMs: 100, + trigger: async () => { + emitChooser({ frameId: "f1", backendNodeId: 789, mode: "selectSingle" }); + return { tab_id: 4, x: 10, y: 10 }; + }, + }); + + expect(result).toMatchObject({ multiple: false }); + expect(calls).toContainEqual({ + method: "DOM.setFileInputFiles", + params: { files: ["/private/stage/one"], backendNodeId: 789 }, + }); + }); + + it("reports a File System Access picker without waiting for a timeout", async () => { + const { cdp, emitChooser } = uploadCdp({ + inputCount: 0, + }); + const result = await uploadThroughActivatedFileInput({ + cdp, + actionTarget: actionTarget("f1"), + files: ["/private/stage/one"], + timeoutMs: 100, + trigger: async () => { + emitChooser({ frameId: "f1", mode: "selectSingle" }); + return { tab_id: 4, x: 10, y: 10 }; + }, }); expect(result).toMatchObject({ code: "unsupported", - message: "upload trigger invoked showOpenFilePicker instead of an input[type=file]", + message: "upload trigger invoked a non-input file picker", data: { reason: "file_input_not_activated", phase: "resolve_input" }, }); }); it("bounds a stuck file-input probe", async () => { - const cdp: CdpRunner = { - send: vi.fn(() => new Promise(() => {})) as unknown as CdpRunner["send"], - }; + const { cdp } = uploadCdp({ pendingResolve: true }); const result = await uploadThroughActivatedFileInput({ cdp, - target: { tabId: 4 }, + actionTarget: actionTarget(), files: ["/private/stage/one"], timeoutMs: 5, trigger: vi.fn(), @@ -158,6 +246,31 @@ describe("file transfer tools", () => { }); }); + it("marks a timed-out file assignment unknown and detaches browser state", async () => { + const fixture = uploadCdp(); + const originalSend = fixture.cdp.send; + const detach = vi.fn(async () => {}); + fixture.cdp.detach = detach; + fixture.cdp.send = vi.fn((tabId: number, method: string, params?: object) => { + if (method === "DOM.setFileInputFiles") return new Promise(() => {}); + return originalSend(tabId, method, params); + }) as CdpRunner["send"]; + + const result = await uploadThroughActivatedFileInput({ + cdp: fixture.cdp, + actionTarget: actionTarget(), + files: ["/private/stage/one"], + timeoutMs: 10, + trigger: async () => ({ tab_id: 4, x: 10, y: 10 }), + }); + + expect(result).toMatchObject({ + code: "timeout", + data: { effect_state: "unknown", phase: "set_files" }, + }); + expect(detach).toHaveBeenCalledWith(4); + }); + it("routes one exact-target download through a browser-relative capability", async () => { const manager = sessions(); const ctx = await manager.start("s1"); @@ -194,6 +307,7 @@ describe("file transfer tools", () => { onDeterminingFilename, search: vi.fn(async () => [completed]), cancel: vi.fn(async () => {}), + removeFile: vi.fn(async () => {}), }; let cdpEvent: Parameters>[0] | undefined; let suggested: chrome.downloads.DownloadFilenameSuggestion | undefined; @@ -261,6 +375,7 @@ describe("file transfer tools", () => { onDeterminingFilename, search: vi.fn(async () => []), cancel: vi.fn(async () => {}), + removeFile: vi.fn(async () => {}), }; let cdpEvent: Parameters>[0] | undefined; const cdp: CdpRunner = { @@ -304,4 +419,137 @@ describe("file transfer tools", () => { data: { reason: "download_capture_failed" }, }); }); + + it("correlates a filename candidate that arrives before the CDP intent", async () => { + const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>(); + const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>(); + const onDeterminingFilename = + fakeEvent< + ( + item: chrome.downloads.DownloadItem, + suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void, + ) => void | true + >(); + const initial = { + id: 21, + url: "https://example.test/candidate-first.bin", + finalUrl: "https://example.test/candidate-first.bin", + filename: "candidate-first.bin", + state: "in_progress", + fileSize: -1, + totalBytes: 4, + bytesReceived: 0, + } as chrome.downloads.DownloadItem; + const complete = { ...initial, state: "complete", fileSize: 4 } as chrome.downloads.DownloadItem; + const downloads: DownloadsApi = { + onCreated, + onChanged, + onDeterminingFilename, + search: vi.fn(async () => [complete]), + cancel: vi.fn(async () => {}), + removeFile: vi.fn(async () => {}), + }; + let cdpEvent: Parameters>[0] | undefined; + const cdp: CdpRunner = { + send: vi.fn(async () => ({})) as CdpRunner["send"], + onEvent: (handler) => { + cdpEvent = handler; + return { dispose: vi.fn() }; + }, + }; + let suggestion: chrome.downloads.DownloadFilenameSuggestion | undefined; + + const result = await captureBrowserDownload({ + cdp, + target: { tabId: 4 }, + downloads, + browserRelativeDir: "BrowserSkill/tr_21", + timeoutMs: 1_000, + trigger: async () => { + const suggested = new Promise((resolve) => { + onDeterminingFilename.emit(initial, (value) => { + suggestion = value; + resolve(); + }); + }); + cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", { + url: initial.url, + suggestedFilename: initial.filename, + }); + await suggested; + onCreated.emit(complete); + return { tab_id: 4, x: 10, y: 10 }; + }, + }); + + expect(suggestion).toEqual({ + filename: "BrowserSkill/tr_21/candidate-first.bin", + conflictAction: "overwrite", + }); + expect(result).toMatchObject({ item: { id: 21, state: "complete" } }); + }); + + it("rejects ambiguous attribution without cancelling either unclaimed download", async () => { + const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>(); + const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>(); + const onDeterminingFilename = + fakeEvent< + ( + item: chrome.downloads.DownloadItem, + suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void, + ) => void | true + >(); + const downloads: DownloadsApi = { + onCreated, + onChanged, + onDeterminingFilename, + search: vi.fn(async () => []), + cancel: vi.fn(async () => {}), + removeFile: vi.fn(async () => {}), + }; + let cdpEvent: Parameters>[0] | undefined; + const cdp: CdpRunner = { + send: vi.fn(async () => ({})) as CdpRunner["send"], + onEvent: (handler) => { + cdpEvent = handler; + return { dispose: vi.fn() }; + }, + }; + const defaults: number[] = []; + const candidate = (id: number) => + ({ + id, + url: "https://example.test/same.bin", + finalUrl: "https://example.test/same.bin", + filename: "same.bin", + state: "in_progress", + }) as chrome.downloads.DownloadItem; + + const result = await captureBrowserDownload({ + cdp, + target: { tabId: 4 }, + downloads, + browserRelativeDir: "BrowserSkill/tr_ambiguous", + timeoutMs: 100, + trigger: async () => { + cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", { + url: "https://example.test/same.bin", + suggestedFilename: "same.bin", + }); + for (const id of [31, 32]) { + onDeterminingFilename.emit(candidate(id), (value) => { + if (value === undefined) defaults.push(id); + }); + } + return { tab_id: 4, x: 10, y: 10 }; + }, + }); + + expect(defaults.sort()).toEqual([31, 32]); + expect(downloads.cancel).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + code: "cdp_failed", + data: { effect_state: "unknown", phase: "attribution" }, + }); + }); }); diff --git a/apps/extension/src/tools/download-capture.ts b/apps/extension/src/tools/download-capture.ts index afab991..6d909b2 100644 --- a/apps/extension/src/tools/download-capture.ts +++ b/apps/extension/src/tools/download-capture.ts @@ -1,12 +1,16 @@ -// Transaction-scoped capture for one web download. Chrome owns the actual -// download and writes only beneath its Downloads root; BrowserSkill supplies a -// daemon-minted relative directory and reports the completed absolute path -// back to the daemon for validated import. +// Order-independent coordinator for one browser download. CDP supplies the +// exact target/frame intent while chrome.downloads supplies the download id +// and filename routing hook; neither event is assumed to arrive first. import type { CdpTarget } from "@/browser-driver/frame-graph"; -import type { ClickResult, RpcError } from "@/transport/types"; +import type { ClickResult, RpcError, TransferEffectState } from "@/transport/types"; +import { transferError } from "./errors"; import { type CdpRunner, isRpcError } from "./shared"; +const CORRELATION_GRACE_MS = 750; +const UNIQUE_SETTLE_MS = 50; +const SIZE_POLL_MS = 250; + type DeterminingFilenameListener = ( item: chrome.downloads.DownloadItem, suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void, @@ -23,6 +27,7 @@ export interface DownloadsApi { onDeterminingFilename: ListenerEvent; search(query: chrome.downloads.DownloadQuery): Promise; cancel(downloadId: number): Promise; + removeFile(downloadId: number): Promise; } export const chromeDownloadsApi: DownloadsApi = { @@ -37,13 +42,16 @@ export const chromeDownloadsApi: DownloadsApi = { }, search: (query) => chrome.downloads.search(query), cancel: (id) => chrome.downloads.cancel(id), + removeFile: (id) => chrome.downloads.removeFile(id), }; export interface DownloadCaptureOptions { cdp: CdpRunner; target: CdpTarget; + expectedFrameId?: string; downloads: DownloadsApi; browserRelativeDir: string; + maxByteSize?: number; timeoutMs: number; signal?: AbortSignal; trigger(): Promise; @@ -54,43 +62,70 @@ export interface DownloadCaptureResult { item: chrome.downloads.DownloadItem; } +interface DownloadIntent { + url: string; + suggestedFilename: string; + frameId?: string; +} + +interface DownloadCandidate { + item: chrome.downloads.DownloadItem; + suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void; + suggested: boolean; + graceTimer: ReturnType; +} + function safeBasename(filename: string): string { const basename = filename.split(/[\\/]/).pop()?.trim(); return basename && basename !== "." && basename !== ".." ? basename : "download"; } -function captureError(message: string): RpcError { - return { - code: "cdp_failed", - message, - data: { reason: "download_capture_failed" }, - }; -} - function sameTarget(source: { tabId?: number; sessionId?: string }, target: CdpTarget): boolean { return source.tabId === target.tabId && source.sessionId === target.sessionId; } -function matchesIntent( - item: chrome.downloads.DownloadItem, - intent: { url: string; suggestedFilename: string }, -): boolean { +function matchesIntent(item: chrome.downloads.DownloadItem, intent: DownloadIntent): boolean { const urlMatches = item.url === intent.url || item.finalUrl === intent.url; return urlMatches && safeBasename(item.filename) === safeBasename(intent.suggestedFilename); } +function knownSize(item: chrome.downloads.DownloadItem): number | undefined { + if (item.fileSize >= 0) return item.fileSize; + if (item.totalBytes >= 0) return item.totalBytes; + return undefined; +} + +function captureError( + message: string, + effectState: TransferEffectState, + phase: string, + cleanupFailed = false, +): RpcError { + return transferError("cdp_failed", "download_capture_failed", message, { + effectState, + phase, + ...(cleanupFailed ? { cleanupState: "failed" } : {}), + }); +} + export async function captureBrowserDownload( options: DownloadCaptureOptions, ): Promise { + let click: ClickResult | undefined; + let intent: DownloadIntent | undefined; let capturedId: number | undefined; - let intent: { url: string; suggestedFilename: string } | undefined; - const createdItems = new Map(); + let capturedItem: chrome.downloads.DownloadItem | undefined; let settled = false; let succeeded = false; - let timer: ReturnType | undefined; - let rejectCompletion!: (error: Error) => void; - let resolveCompletion!: (item: chrome.downloads.DownloadItem) => void; + let failureResult: RpcError | undefined; + let uniquenessTimer: ReturnType | undefined; + let operationTimer: ReturnType | undefined; + let sizePoll: ReturnType | undefined; + const candidates = new Map(); + const createdItems = new Map(); + let resolveCompletion!: (item: chrome.downloads.DownloadItem) => void; + let rejectCompletion!: (error: Error) => void; const completion = new Promise((resolve, reject) => { resolveCompletion = resolve; rejectCompletion = reject; @@ -102,40 +137,95 @@ export async function captureBrowserDownload( }; const complete = (item: chrome.downloads.DownloadItem) => { if (settled) return; + const size = knownSize(item); + if (size !== undefined && options.maxByteSize !== undefined && size > options.maxByteSize) { + fail(new Error(`download exceeds transfer limit ${options.maxByteSize}`)); + return; + } settled = true; + capturedItem = item; resolveCompletion(item); }; - const claim = (item: chrome.downloads.DownloadItem): boolean => { - if (capturedId === undefined) { - capturedId = item.id; - return true; + const suggestDefault = (candidate: DownloadCandidate) => { + if (candidate.suggested) return; + candidate.suggested = true; + candidate.suggest(); + }; + const matchingCandidates = (): DownloadCandidate[] => { + const currentIntent = intent; + return currentIntent + ? [...candidates.values()].filter( + (candidate) => !candidate.suggested && matchesIntent(candidate.item, currentIntent), + ) + : []; + }; + + const claimUnique = () => { + uniquenessTimer = undefined; + if (settled || capturedId !== undefined || !intent) return; + const matches = matchingCandidates(); + if (matches.length !== 1) { + if (matches.length > 1) { + for (const candidate of matches) suggestDefault(candidate); + fail(new Error("download attribution is ambiguous")); + } + return; + } + const candidate = matches[0]; + candidate.suggested = true; + clearTimeout(candidate.graceTimer); + capturedId = candidate.item.id; + capturedItem = candidate.item; + candidate.suggest({ + filename: `${options.browserRelativeDir}/${safeBasename(intent.suggestedFilename)}`, + conflictAction: "overwrite", + }); + const size = knownSize(candidate.item); + if (size !== undefined && options.maxByteSize !== undefined && size > options.maxByteSize) { + fail(new Error(`download exceeds transfer limit ${options.maxByteSize}`)); + return; + } + const created = createdItems.get(candidate.item.id); + if (created?.state === "interrupted") { + fail(new Error(created.error ?? "download interrupted")); + } else if (created?.state === "complete") { + complete(created); + } + }; + const reconcile = () => { + if (settled || capturedId !== undefined || !intent) return; + const matches = matchingCandidates(); + if (matches.length > 1) { + for (const candidate of matches) suggestDefault(candidate); + fail(new Error("download attribution is ambiguous")); + return; + } + if (matches.length === 1 && !uniquenessTimer) { + uniquenessTimer = setTimeout(claimUnique, UNIQUE_SETTLE_MS); } - if (capturedId === item.id) return true; - void options.downloads.cancel(item.id).catch(() => undefined); - fail(new Error("download trigger produced more than one file")); - return false; }; const determiningListener: DeterminingFilenameListener = (item, suggest) => { - try { - if (!intent || !matchesIntent(item, intent) || !claim(item)) { - suggest(); - return; - } - suggest({ - filename: `${options.browserRelativeDir}/${safeBasename(intent.suggestedFilename)}`, - conflictAction: "overwrite", - }); - const created = createdItems.get(item.id); - if (created?.state === "complete") complete(created); - } catch (err) { - suggest(); - fail(err instanceof Error ? err : new Error(String(err))); - } + const candidate: DownloadCandidate = { + item, + suggest, + suggested: false, + graceTimer: setTimeout(() => { + suggestDefault(candidate); + candidates.delete(item.id); + if (intent && matchesIntent(item, intent) && capturedId === undefined) { + fail(new Error("download correlation grace elapsed before unique attribution")); + } + }, CORRELATION_GRACE_MS), + }; + candidates.set(item.id, candidate); + reconcile(); + return true; }; const createdListener = (item: chrome.downloads.DownloadItem) => { createdItems.set(item.id, item); if (capturedId !== item.id) return; + capturedItem = item; if (item.state === "interrupted") { fail(new Error(item.error ?? "download interrupted")); } else if (item.state === "complete") { @@ -158,54 +248,100 @@ export async function captureBrowserDownload( } } }; - const onAbort = () => { - fail(new DOMException("aborted", "AbortError")); - }; + const onAbort = () => fail(new DOMException("aborted", "AbortError")); const cdpSubscription = options.cdp.onEvent?.((source, method, raw) => { if (method !== "Page.downloadWillBegin" || !sameTarget(source, options.target)) return; - const event = raw as { url?: unknown; suggestedFilename?: unknown }; + const event = raw as { url?: unknown; suggestedFilename?: unknown; frameId?: unknown }; if (typeof event.url !== "string" || typeof event.suggestedFilename !== "string") return; + if (options.expectedFrameId && event.frameId !== options.expectedFrameId) { + fail(new Error("download originated from a different frame")); + return; + } if (intent) { fail(new Error("download trigger produced more than one browser download intent")); return; } - intent = { url: event.url, suggestedFilename: event.suggestedFilename }; + intent = { + url: event.url, + suggestedFilename: event.suggestedFilename, + ...(typeof event.frameId === "string" ? { frameId: event.frameId } : {}), + }; + reconcile(); }); if (!cdpSubscription) { - return captureError("CDP download intent subscription unavailable"); + return captureError("CDP download intent subscription unavailable", "none", "arm"); } options.downloads.onDeterminingFilename.addListener(determiningListener); options.downloads.onCreated.addListener(createdListener); options.downloads.onChanged.addListener(changedListener); options.signal?.addEventListener("abort", onAbort, { once: true }); - timer = setTimeout( + operationTimer = setTimeout( () => fail(new Error("download did not complete before timeout")), options.timeoutMs, ); + sizePoll = setInterval(() => { + if (capturedId === undefined || settled || options.maxByteSize === undefined) return; + void options.downloads + .search({ id: capturedId }) + .then(([item]) => { + if (!item || settled) return; + capturedItem = item; + if (item.bytesReceived > (options.maxByteSize as number)) { + fail(new Error(`download exceeds transfer limit ${options.maxByteSize}`)); + } + }) + .catch((err) => fail(err instanceof Error ? err : new Error(String(err)))); + }, SIZE_POLL_MS); try { const triggered = await options.trigger(); if (isRpcError(triggered)) { void completion.catch(() => undefined); - return triggered; + const effect: TransferEffectState = capturedId !== undefined ? "committed" : intent ? "unknown" : "none"; + failureResult = { + ...triggered, + data: { ...triggered.data, effect_state: effect, phase: "trigger" }, + }; + return failureResult; } + click = triggered; const item = await completion; succeeded = true; - return { click: triggered, item }; + return { click, item }; } catch (err) { - if (err instanceof DOMException && err.name === "AbortError") throw err; - return captureError(err instanceof Error ? err.message : String(err)); + const effect: TransferEffectState = capturedId !== undefined ? "committed" : click ? "unknown" : "none"; + failureResult = captureError( + err instanceof Error ? err.message : String(err), + effect, + capturedId !== undefined ? "download" : "attribution", + ); + return failureResult; } finally { settled = true; - if (timer) clearTimeout(timer); - if (!succeeded && capturedId !== undefined) { - await options.downloads.cancel(capturedId).catch(() => undefined); - } + if (operationTimer) clearTimeout(operationTimer); + if (uniquenessTimer) clearTimeout(uniquenessTimer); + if (sizePoll) clearInterval(sizePoll); options.signal?.removeEventListener("abort", onAbort); options.downloads.onDeterminingFilename.removeListener(determiningListener); options.downloads.onCreated.removeListener(createdListener); options.downloads.onChanged.removeListener(changedListener); cdpSubscription.dispose(); + for (const candidate of candidates.values()) { + clearTimeout(candidate.graceTimer); + if (candidate.item.id !== capturedId) suggestDefault(candidate); + } + if (!succeeded && capturedId !== undefined) { + const item = capturedItem ?? createdItems.get(capturedId); + try { + if (item?.state === "complete") { + await options.downloads.removeFile(capturedId); + } else { + await options.downloads.cancel(capturedId); + } + } catch { + if (failureResult?.data) failureResult.data.cleanup_state = "failed"; + } + } } } diff --git a/apps/extension/src/tools/download.ts b/apps/extension/src/tools/download.ts index d337315..fb2c586 100644 --- a/apps/extension/src/tools/download.ts +++ b/apps/extension/src/tools/download.ts @@ -2,9 +2,13 @@ // browser-global chrome.downloads transaction to download-capture.ts. import type { SessionManager } from "@/session-manager/manager"; -import type { ClickParams, DownloadParams, DownloadResult, RpcError } from "@/transport/types"; +import type { DownloadParams, DownloadResult, RpcError } from "@/transport/types"; import { captureBrowserDownload, chromeDownloadsApi, type DownloadsApi } from "./download-capture"; -import { handleClick, type InteractionDeps, resolveBackendNode } from "./interaction"; +import { + clickResolvedTarget, + type InteractionDeps, + resolveActionTarget, +} from "./interaction"; import { enforceAgentWindow, isRpcError, lookupSession, resolveTargetTab } from "./shared"; let downloadActive = false; @@ -32,7 +36,7 @@ export async function handleDownload( if (!params.browser_relative_dir) { return { code: "invalid_params", message: "download requires a daemon capability directory" }; } - const address = await resolveBackendNode(deps.cdp, ctx, target, params, "download"); + const address = await resolveActionTarget(deps.cdp, ctx, target, params, "download"); if (isRpcError(address)) return address; const capture = await captureBrowserDownload({ @@ -40,18 +44,11 @@ export async function handleDownload( target: address.cdpTarget, downloads: deps.downloads ?? chromeDownloadsApi, browserRelativeDir: params.browser_relative_dir, + maxByteSize: params.max_byte_size, timeoutMs: params.timeout_ms ?? 120_000, signal: deps.signal, - trigger: () => { - const clickParams: ClickParams = { - session_id: params.session_id, - ref: params.ref, - selector: params.selector, - tab_id: params.tab_id, - timeout_ms: params.timeout_ms, - }; - return handleClick(manager, clickParams, deps); - }, + expectedFrameId: address.frameId, + trigger: () => clickResolvedTarget(ctx, address, {}, deps), }); if (isRpcError(capture)) return capture; const { click, item } = capture; diff --git a/apps/extension/src/tools/errors.ts b/apps/extension/src/tools/errors.ts index 88cfd78..800a765 100644 --- a/apps/extension/src/tools/errors.ts +++ b/apps/extension/src/tools/errors.ts @@ -2,10 +2,23 @@ // rendering. Extension handlers attach reasons here; human-facing copy // lives in bsk-cli `render_error.rs`. -import type { ErrorCode, RpcError, RpcErrorData, RpcErrorReason } from "@/transport/types"; +import type { + ErrorCode, + RpcError, + RpcErrorData, + RpcErrorReason, + TransferCleanupState, + TransferEffectState, +} from "@/transport/types"; export type { RpcErrorData, RpcErrorReason }; +export interface TransferErrorOptions { + effectState: TransferEffectState; + phase: string; + cleanupState?: TransferCleanupState; +} + export function rpcError( code: ErrorCode, reason: RpcErrorReason, @@ -15,3 +28,16 @@ export function rpcError( const data: RpcErrorData = { reason, ...extra }; return { code, message, data }; } + +export function transferError( + code: ErrorCode, + reason: RpcErrorReason, + message: string, + options: TransferErrorOptions, +): RpcError { + return rpcError(code, reason, message, { + effect_state: options.effectState, + phase: options.phase, + ...(options.cleanupState ? { cleanup_state: options.cleanupState } : {}), + }); +} diff --git a/apps/extension/src/tools/file-input-transaction.ts b/apps/extension/src/tools/file-input-transaction.ts index 50297ad..fb3901f 100644 --- a/apps/extension/src/tools/file-input-transaction.ts +++ b/apps/extension/src/tools/file-input-transaction.ts @@ -1,35 +1,43 @@ -// One upload transaction, independent of Page.fileChooserOpened delivery. -// A transaction-scoped DOM listener records the file input actually activated -// by the requested click and cancels its native default action. File System -// Access picker entry points are replaced only for the same transaction, so a -// non-input picker fails promptly without opening an OS dialog. +// One upload transaction with an explicit browser-side commit boundary. +// Chrome interception prevents a native chooser from escaping automation; +// chooser events and a frame-scoped DOM probe are independent input-location +// signals, so event delivery is not required for standard file inputs. -import type { CdpTarget } from "@/browser-driver/frame-graph"; -import type { ClickResult, RpcError } from "@/transport/types"; +import type { ClickResult, RpcError, TransferEffectState } from "@/transport/types"; +import { transferError } from "./errors"; +import type { ResolvedActionTarget } from "./interaction"; import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared"; const CLEANUP_TIMEOUT_MS = 1_000; +const ACTIVATION_GRACE_MS = 1_000; +const PROBE_INTERVAL_MS = 20; -type UploadPhase = "arm_input_probe" | "trigger" | "resolve_input" | "set_files"; +type UploadPhase = + | "arm_interception" + | "arm_input_probe" + | "trigger" + | "resolve_input" + | "set_files" + | "cleanup"; interface RuntimeReply { - result?: { - value?: unknown; - objectId?: string; - }; - exceptionDetails?: { - text?: string; - exception?: { description?: string }; - }; + result?: { value?: unknown; objectId?: string }; + exceptionDetails?: { text?: string; exception?: { description?: string } }; +} + +interface ChooserEvent { + frameId?: string; + backendNodeId?: number; + mode?: "selectSingle" | "selectMultiple"; } export interface FileInputTransactionOptions { cdp: CdpRunner; - target: CdpTarget; + actionTarget: ResolvedActionTarget; files: string[]; timeoutMs: number; signal?: AbortSignal; - trigger(timeoutMs: number): Promise; + trigger(): Promise; } export interface FileInputTransactionResult { @@ -38,10 +46,7 @@ export interface FileInputTransactionResult { } class BoundedWaitError extends Error { - constructor( - readonly kind: "timeout" | "aborted", - message: string, - ) { + constructor(readonly kind: "timeout" | "aborted", message: string) { super(message); } } @@ -77,15 +82,6 @@ async function waitBounded( } } -function uploadError( - code: RpcError["code"], - message: string, - reason: "file_input_probe_failed" | "file_input_not_activated" | "set_file_input_failed", - phase: UploadPhase, -): RpcError { - return { code, message, data: { reason, phase } }; -} - function runtimeError(reply: RuntimeReply, fallback: string): Error | null { if (!reply.exceptionDetails) return null; return new Error( @@ -93,25 +89,124 @@ function runtimeError(reply: RuntimeReply, fallback: string): Error | null { ); } +function transferFailure( + code: RpcError["code"], + reason: "file_input_probe_failed" | "file_input_not_activated" | "set_file_input_failed", + message: string, + effectState: TransferEffectState, + phase: UploadPhase, +): RpcError { + return transferError(code, reason, message, { effectState, phase }); +} + +function enrichFailure(error: RpcError, effectState: TransferEffectState, phase: UploadPhase): RpcError { + return { + ...error, + data: { ...error.data, effect_state: effectState, phase }, + }; +} + +function sameTarget( + source: { tabId?: number; sessionId?: string }, + target: { tabId: number; sessionId?: string }, +): boolean { + return source.tabId === target.tabId && source.sessionId === target.sessionId; +} + +async function delay(ms: number, signal?: AbortSignal): Promise { + await waitBounded( + new Promise((resolve) => setTimeout(resolve, ms)), + Date.now() + ms + 1, + signal, + "upload activation probe timed out", + ); +} + +async function callOnTrigger( + options: FileInputTransactionOptions, + objectId: string, + functionDeclaration: string, + args: unknown[], + returnByValue: boolean, +): Promise { + return sendToCdpTarget(options.cdp, options.actionTarget.cdpTarget, "Runtime.callFunctionOn", { + objectId, + functionDeclaration, + arguments: args.map((value) => ({ value })), + returnByValue, + awaitPromise: false, + }); +} + export async function uploadThroughActivatedFileInput( options: FileInputTransactionOptions, ): Promise { const deadline = Date.now() + options.timeoutMs; + const target = options.actionTarget.cdpTarget; const objectGroup = `bsk-upload-${crypto.randomUUID()}`; const stateKey = `__bskUpload_${crypto.randomUUID().replaceAll("-", "")}`; - const stateKeyLiteral = JSON.stringify(stateKey); - let probeAttempted = false; + const chooserEvents: ChooserEvent[] = []; + let interceptionArmed = false; + let probeArmed = false; + let triggerObjectId: string | undefined; + let outcome: FileInputTransactionResult | RpcError = transferFailure( + "protocol_error", + "file_input_probe_failed", + "upload transaction ended without an outcome", + "none", + "cleanup", + ); + + const chooserSubscription = options.cdp.onEvent?.((source, method, raw) => { + if (method !== "Page.fileChooserOpened" || !sameTarget(source, target)) return; + const event = raw as ChooserEvent; + chooserEvents.push(event); + }); try { try { - probeAttempted = true; + await waitBounded( + sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", { + enabled: true, + }), + deadline, + options.signal, + "arming native file chooser interception timed out", + ); + interceptionArmed = true; + } catch (err) { + outcome = transferFailure( + err instanceof BoundedWaitError ? "timeout" : "cdp_failed", + "file_input_probe_failed", + err instanceof Error ? err.message : String(err), + "none", + "arm_interception", + ); + return outcome; + } + + try { + const resolved = await waitBounded( + sendToCdpTarget<{ object?: { objectId?: string } }>(options.cdp, target, "DOM.resolveNode", { + backendNodeId: options.actionTarget.backendNodeId, + objectGroup, + }), + deadline, + options.signal, + "resolving upload trigger timed out", + ); + triggerObjectId = resolved.object?.objectId; + if (!triggerObjectId) throw new Error("DOM.resolveNode returned no trigger objectId"); + const armed = await waitBounded( - sendToCdpTarget(options.cdp, options.target, "Runtime.evaluate", { - expression: `(() => { - const key = ${stateKeyLiteral}; - const owner = globalThis; - const doc = document; - const state = { inputs: [], listener: null, pickerCalls: [], pickers: [] }; + callOnTrigger( + options, + triggerObjectId, + `function(key) { + const doc = this.ownerDocument; + const owner = doc.defaultView; + if (!owner) return false; + const state = { inputs: [], listener: null }; Object.defineProperty(owner, key, { value: state, configurable: true }); state.listener = event => { const path = typeof event.composedPath === "function" ? event.composedPath() : []; @@ -123,143 +218,169 @@ export async function uploadThroughActivatedFileInput( } }; doc.addEventListener("click", state.listener, true); - const win = doc.defaultView; - for (const name of ["showOpenFilePicker", "showSaveFilePicker", "showDirectoryPicker"]) { - if (!win || typeof win[name] !== "function") continue; - const hadOwn = Object.prototype.hasOwnProperty.call(win, name); - const descriptor = Object.getOwnPropertyDescriptor(win, name); - state.pickers.push({ name, hadOwn, descriptor }); - Object.defineProperty(win, name, { - configurable: true, - enumerable: descriptor?.enumerable ?? true, - writable: true, - value: () => { - state.pickerCalls.push(name); - return Promise.reject( - new DOMException("Picker intercepted by BrowserSkill", "AbortError") - ); - }, - }); - } return true; - })()`, - objectGroup, - returnByValue: true, - }), + }`, + [stateKey], + true, + ), deadline, options.signal, - "arming file input probe timed out", + "arming frame-scoped file input probe timed out", ); const armError = runtimeError(armed, "failed to arm file input probe"); - if (armError) throw armError; + if (armError || armed.result?.value !== true) { + throw armError ?? new Error("file input probe did not arm"); + } + probeArmed = true; } catch (err) { - if (err instanceof BoundedWaitError && err.kind === "aborted") throw err; - return uploadError( + outcome = transferFailure( err instanceof BoundedWaitError ? "timeout" : "cdp_failed", - err instanceof Error ? err.message : String(err), "file_input_probe_failed", + err instanceof Error ? err.message : String(err), + "none", "arm_input_probe", ); + return outcome; } let click: ClickResult | RpcError; try { click = await waitBounded( - options.trigger(remainingMs(deadline)), + options.trigger(), deadline, options.signal, "upload trigger timed out", ); } catch (err) { - if (err instanceof BoundedWaitError && err.kind === "aborted") throw err; - return uploadError( + outcome = transferFailure( err instanceof BoundedWaitError ? "timeout" : "cdp_failed", - err instanceof Error ? err.message : String(err), "file_input_probe_failed", + err instanceof Error ? err.message : String(err), + "none", "trigger", ); + return outcome; + } + if (isRpcError(click)) { + outcome = enrichFailure(click, "none", "trigger"); + return outcome; } - if (isRpcError(click)) return click; try { - const summary = await waitBounded( - sendToCdpTarget(options.cdp, options.target, "Runtime.evaluate", { - expression: `(() => { - const s = globalThis[${stateKeyLiteral}]; - return s - ? { count: s.inputs.length, multiple: s.inputs[0]?.multiple === true, - pickerCall: s.pickerCalls[0] } + const activationDeadline = Math.min(deadline, Date.now() + ACTIVATION_GRACE_MS); + let summary: { count: number; multiple: boolean } = { count: 0, multiple: false }; + while (Date.now() < activationDeadline) { + if (chooserEvents.length > 0) break; + const reply = await callOnTrigger( + options, + triggerObjectId, + `function(key) { + const state = this.ownerDocument.defaultView?.[key]; + return state + ? { count: state.inputs.length, multiple: state.inputs[0]?.multiple === true } : { count: 0, multiple: false }; - })()`, - returnByValue: true, - }), - deadline, - options.signal, - "resolving activated file input timed out", - ); - const summaryError = runtimeError(summary, "failed to inspect activated file input"); - if (summaryError) throw summaryError; - const value = summary.result?.value as - | { count?: unknown; multiple?: unknown; pickerCall?: unknown } - | undefined; - const count = typeof value?.count === "number" ? value.count : 0; - const multiple = value?.multiple === true; - if (typeof value?.pickerCall === "string") { - return uploadError( - "unsupported", - `upload trigger invoked ${value.pickerCall} instead of an input[type=file]`, - "file_input_not_activated", - "resolve_input", + }`, + [stateKey], + true, ); - } - if (count !== 1) { - return uploadError( - "unsupported", - count === 0 - ? "upload trigger did not activate an input[type=file]" - : "upload trigger activated more than one input[type=file]", - "file_input_not_activated", - "resolve_input", - ); - } - if (!multiple && options.files.length !== 1) { - return { code: "invalid_params", message: "file input accepts exactly one file" }; + const summaryError = runtimeError(reply, "failed to inspect activated file input"); + if (summaryError) throw summaryError; + const value = reply.result?.value as { count?: unknown; multiple?: unknown } | undefined; + summary = { + count: typeof value?.count === "number" ? value.count : 0, + multiple: value?.multiple === true, + }; + if (summary.count > 0) break; + await delay(Math.min(PROBE_INTERVAL_MS, remainingMs(activationDeadline)), options.signal); } - const input = await waitBounded( - sendToCdpTarget(options.cdp, options.target, "Runtime.evaluate", { - expression: `globalThis[${stateKeyLiteral}]?.inputs[0]`, - objectGroup, - returnByValue: false, - }), - deadline, - options.signal, - "resolving activated file input object timed out", - ); - const inputError = runtimeError(input, "failed to resolve activated file input object"); - if (inputError) throw inputError; - const inputObjectId = input.result?.objectId; - if (!inputObjectId) throw new Error("activated file input returned no objectId"); + if (chooserEvents.length > 1) { + outcome = transferFailure( + "unsupported", + "file_input_not_activated", + "upload trigger activated more than one file chooser", + "none", + "resolve_input", + ); + return outcome; + } - const described = await waitBounded( - sendToCdpTarget<{ node?: { backendNodeId?: number } }>( + const chooser = chooserEvents[0]; + let backendNodeId: number | undefined; + let multiple = summary.multiple; + if (chooser) { + if ( + options.actionTarget.frameId && + chooser.frameId !== options.actionTarget.frameId + ) { + outcome = transferFailure( + "unsupported", + "file_input_not_activated", + "upload trigger activated a file chooser in a different frame", + "none", + "resolve_input", + ); + return outcome; + } + if (typeof chooser.backendNodeId !== "number") { + outcome = transferFailure( + "unsupported", + "file_input_not_activated", + "upload trigger invoked a non-input file picker", + "none", + "resolve_input", + ); + return outcome; + } + backendNodeId = chooser.backendNodeId; + multiple = chooser.mode === "selectMultiple"; + } else { + if (summary.count !== 1) { + outcome = transferFailure( + "unsupported", + "file_input_not_activated", + summary.count === 0 + ? "upload trigger did not activate an input[type=file]" + : "upload trigger activated more than one input[type=file]", + "none", + "resolve_input", + ); + return outcome; + } + const input = await callOnTrigger( + options, + triggerObjectId, + `function(key) { return this.ownerDocument.defaultView?.[key]?.inputs[0]; }`, + [stateKey], + false, + ); + const inputError = runtimeError(input, "failed to resolve activated file input object"); + if (inputError) throw inputError; + if (!input.result?.objectId) throw new Error("activated file input returned no objectId"); + const described = await sendToCdpTarget<{ node?: { backendNodeId?: number } }>( options.cdp, - options.target, + target, "DOM.describeNode", - { objectId: inputObjectId }, - ), - deadline, - options.signal, - "describing activated file input timed out", - ); - const backendNodeId = described.node?.backendNodeId; - if (typeof backendNodeId !== "number") { - throw new Error("DOM.describeNode returned no file input backendNodeId"); + { objectId: input.result.objectId }, + ); + backendNodeId = described.node?.backendNodeId; + if (typeof backendNodeId !== "number") { + throw new Error("DOM.describeNode returned no file input backendNodeId"); + } + } + + if (!multiple && options.files.length !== 1) { + outcome = enrichFailure( + { code: "invalid_params", message: "file input accepts exactly one file" }, + "none", + "resolve_input", + ); + return outcome; } try { await waitBounded( - sendToCdpTarget(options.cdp, options.target, "DOM.setFileInputFiles", { + sendToCdpTarget(options.cdp, target, "DOM.setFileInputFiles", { files: options.files, backendNodeId, }), @@ -268,72 +389,87 @@ export async function uploadThroughActivatedFileInput( "setting file input files timed out", ); } catch (err) { - if (err instanceof BoundedWaitError && err.kind === "aborted") throw err; - return uploadError( + outcome = transferFailure( err instanceof BoundedWaitError ? "timeout" : "cdp_failed", - err instanceof Error ? err.message : String(err), "set_file_input_failed", + err instanceof Error ? err.message : String(err), + "unknown", "set_files", ); + return outcome; } - return { click, multiple }; + outcome = { click, multiple }; + return outcome; } catch (err) { - if (err instanceof BoundedWaitError && err.kind === "aborted") throw err; - return uploadError( + outcome = transferFailure( err instanceof BoundedWaitError ? "timeout" : "cdp_failed", - err instanceof Error ? err.message : String(err), "file_input_probe_failed", + err instanceof Error ? err.message : String(err), + "none", "resolve_input", ); + return outcome; } - } catch (err) { - if (err instanceof BoundedWaitError && err.kind === "aborted") { - return { code: "cancelled", message: "upload transaction aborted" }; - } - throw err; } finally { - if (probeAttempted) { + chooserSubscription?.dispose(); + let cleanupFailed = false; + if (probeArmed && triggerObjectId) { try { await waitBounded( - sendToCdpTarget(options.cdp, options.target, "Runtime.evaluate", { - expression: `(() => { - const key = ${stateKeyLiteral}; - const state = globalThis[key]; - if (state?.listener) { - document.removeEventListener("click", state.listener, true); - } - const win = document.defaultView; - if (win) { - for (const picker of state?.pickers || []) { - try { - if (picker.hadOwn && picker.descriptor) { - Object.defineProperty(win, picker.name, picker.descriptor); - } else { - delete win[picker.name]; - } - } catch {} - } - } - delete globalThis[key]; - })()`, - }), + callOnTrigger( + options, + triggerObjectId, + `function(key) { + const owner = this.ownerDocument.defaultView; + const state = owner?.[key]; + if (state?.listener) this.ownerDocument.removeEventListener("click", state.listener, true); + if (owner) delete owner[key]; + }`, + [stateKey], + true, + ), Date.now() + CLEANUP_TIMEOUT_MS, undefined, "cleaning file input probe timed out", ); } catch { - // Navigation may have invalidated the object; its document is gone too. + cleanupFailed = true; + } + } + if (interceptionArmed) { + try { + await waitBounded( + sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", { + enabled: false, + cancel: true, + }), + Date.now() + CLEANUP_TIMEOUT_MS, + undefined, + "disabling file chooser interception timed out", + ); + } catch { + cleanupFailed = true; } } try { await waitBounded( - sendToCdpTarget(options.cdp, options.target, "Runtime.releaseObjectGroup", { - objectGroup, - }), + sendToCdpTarget(options.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }), Date.now() + CLEANUP_TIMEOUT_MS, undefined, "releasing upload object group timed out", ); - } catch {} + } catch { + cleanupFailed = true; + } + + const effect = isRpcError(outcome) + ? (outcome.data?.effect_state as TransferEffectState | undefined) + : "committed"; + if (effect === "unknown" || cleanupFailed) { + await options.cdp.detach?.(target.tabId); + } + if (cleanupFailed && isRpcError(outcome)) { + outcome.data = { ...outcome.data, cleanup_state: "failed" }; + } } } diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index 3ea1373..54646ed 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -40,6 +40,7 @@ import { enforceAgentWindow, isRpcError, lookupSession, + type ResolvedTargetTab, resolveTargetTab, } from "./shared"; import { resolveSnapshotRef } from "./snapshot-ref"; @@ -56,6 +57,15 @@ export interface InteractionDeps { keepOverlayBypassAfterHover?: boolean; } +export interface ResolvedActionTarget { + tab: ResolvedTargetTab; + backendNodeId: number; + cdpTarget: CdpTarget; + frameId?: string; + usedRef?: string; + usedSelector?: string; +} + const DEFAULT_TIMEOUT_MS = 30_000; const DEFAULT_HOVER_SETTLE_MS = 200; @@ -215,6 +225,17 @@ export async function resolveBackendNode( } } +export async function resolveActionTarget( + cdp: CdpRunner, + ctx: SessionContext, + target: ResolvedTargetTab, + params: { ref?: string; selector?: string }, + toolName: string, +): Promise { + const node = await resolveBackendNode(cdp, ctx, target, params, toolName); + return isRpcError(node) ? node : { tab: target, ...node }; +} + // --------------------------------------------------------------------------- // tool.click // --------------------------------------------------------------------------- @@ -233,10 +254,19 @@ export async function handleClick( if (isRpcError(target)) return target; const denied = enforceAgentWindow(ctx, target, "click"); if (denied) return denied; - const dialogCursor = markDialogCursor(deps.cdp, target.tabId); + const resolved = await resolveActionTarget(deps.cdp, ctx, target, params, "click"); + if (isRpcError(resolved)) return resolved; + return clickResolvedTarget(ctx, resolved, params, deps); +} - const node = await resolveBackendNode(deps.cdp, ctx, target, params, "click"); - if (isRpcError(node)) return node; +export async function clickResolvedTarget( + ctx: SessionContext, + resolved: ResolvedActionTarget, + params: Pick, + deps: InteractionDeps, +): Promise { + const { tab: target } = resolved; + const dialogCursor = markDialogCursor(deps.cdp, target.tabId); if (throwIfAborted(deps.signal)) { return { code: "cancelled", message: "click aborted" }; @@ -247,9 +277,9 @@ export async function handleClick( deps.cdp, target.tabId, { - target: node.cdpTarget, - backendNodeId: node.backendNodeId, - ...(node.frameId ? { frameId: node.frameId } : {}), + target: resolved.cdpTarget, + backendNodeId: resolved.backendNodeId, + ...(resolved.frameId ? { frameId: resolved.frameId } : {}), }, { scrollIntoView: true }, ); @@ -337,8 +367,8 @@ export async function handleClick( return attachDialogs(deps.cdp, target.tabId, dialogCursor, { tab_id: target.tabId, - used_ref: node.usedRef, - used_selector: node.usedSelector, + used_ref: resolved.usedRef, + used_selector: resolved.usedSelector, x: centre.x, y: centre.y, }); diff --git a/apps/extension/src/tools/upload.ts b/apps/extension/src/tools/upload.ts index 6622bc4..58370e4 100644 --- a/apps/extension/src/tools/upload.ts +++ b/apps/extension/src/tools/upload.ts @@ -3,9 +3,13 @@ // transaction module. import type { SessionManager } from "@/session-manager/manager"; -import type { ClickParams, RpcError, UploadParams, UploadResult } from "@/transport/types"; +import type { RpcError, UploadParams, UploadResult } from "@/transport/types"; import { uploadThroughActivatedFileInput } from "./file-input-transaction"; -import { handleClick, type InteractionDeps, resolveBackendNode } from "./interaction"; +import { + clickResolvedTarget, + type InteractionDeps, + resolveActionTarget, +} from "./interaction"; import { type CdpRunner, enforceAgentWindow, @@ -38,25 +42,16 @@ export async function handleUpload( ) { return { code: "invalid_params", message: "upload requires daemon-staged files" }; } - const address = await resolveBackendNode(deps.cdp, ctx, target, params, "upload"); + const address = await resolveActionTarget(deps.cdp, ctx, target, params, "upload"); if (isRpcError(address)) return address; const timeoutMs = params.timeout_ms ?? DEFAULT_TIMEOUT_MS; const transaction = await uploadThroughActivatedFileInput({ cdp: deps.cdp, - target: address.cdpTarget, + actionTarget: address, files: params.files.map((file) => file.staged_path as string), timeoutMs, signal: deps.signal, - trigger: (remaining) => { - const clickParams: ClickParams = { - session_id: params.session_id, - ref: params.ref, - selector: params.selector, - tab_id: params.tab_id, - timeout_ms: Math.max(1, remaining), - }; - return handleClick(manager, clickParams, deps); - }, + trigger: () => clickResolvedTarget(ctx, address, {}, deps), }); if (isRpcError(transaction)) return transaction; return { diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index 4504ff1..7143cb1 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -37,10 +37,18 @@ export type RpcErrorReason = | "file_input_not_activated" | "set_file_input_failed" | "download_capture_failed" + | "transfer_outcome_unknown" + | "transfer_timeout" | "cleanup_failed"; +export type TransferEffectState = "none" | "committed" | "unknown"; +export type TransferCleanupState = "complete" | "failed"; + export interface RpcErrorData { reason?: RpcErrorReason; + effect_state?: TransferEffectState; + phase?: string; + cleanup_state?: TransferCleanupState; [key: string]: unknown; } @@ -533,6 +541,7 @@ export interface DownloadParams { tab_id?: number; timeout_ms?: number; browser_relative_dir?: string; + max_byte_size?: number; } export interface DownloadResult { diff --git a/crates/bsk-cli/Cargo.toml b/crates/bsk-cli/Cargo.toml index 1475e4b..e24ef88 100644 --- a/crates/bsk-cli/Cargo.toml +++ b/crates/bsk-cli/Cargo.toml @@ -64,6 +64,7 @@ windows-sys = { version = "0.59", features = [ "Win32_Foundation", "Win32_System_Threading", "Win32_Security", + "Win32_Storage_FileSystem", ] } [dev-dependencies] diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index b13f631..8af5383 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -215,7 +215,7 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff The agent/harness decides whether a file transfer is appropriate and which local path belongs to the task. Treat upload as disclosure of that file to the current website, and download as accepting website-controlled bytes onto the local filesystem. Use only paths that are necessary for the user's bounded goal. -BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted capabilities reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. Upload captures the file input actually activated by the requested click and assigns only the staged file paths. Download requires an exact-target browser intent before it claims one Chrome download, routes it through a daemon-minted relative directory, and lets the daemon validate and import it. Upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive. +BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted capabilities reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. Upload intercepts the native chooser for one transaction, locates the input activated in the resolved target's document, and assigns only the staged file paths. Download uniquely correlates one exact-target browser intent with one Chrome download in either event order, routes it through a daemon-minted relative directory, and lets the daemon validate and import it. Upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive. Do not use `request-help` merely because a native file chooser or browser download is involved; try these commands first. For transfer failures, use the structured error instead of retrying blindly: @@ -223,6 +223,9 @@ Do not use `request-help` merely because a native file chooser or browser downlo - `reason=file_input_probe_failed` means BrowserSkill could not safely establish the browser-side upload transaction. Do not repeat the same action; use `request-help` when available. - `reason=set_file_input_failed` means BrowserSkill found the activated file input but Chrome rejected the staged path or assignment. Check the extension's file-URL access permission; otherwise use `request-help`. - `reason=download_capture_failed` means BrowserSkill could not attribute exactly one completed download to the requested target. Do not retry blindly or accept an unrelated browser download; use `request-help` when available. +- `effect_state=none` means BrowserSkill confirmed that no file-transfer effect was committed. Follow the accompanying reason; a corrected target or explicit human fallback may be attempted. +- `effect_state=unknown` means the browser may already have attached or created the file. Do not repeat the transfer. Observe the page if that can establish the result; otherwise stop and report the uncertainty. +- `effect_state=committed` means the browser-side effect occurred even if later completion or cleanup failed. Do not repeat it; continue only after verifying the resulting page/download state. If `request-help` returns `outcome="disabled"`, do not retry it. Stop gracefully and report the transfer mechanism that requires human intervention. diff --git a/crates/bsk-cli/src/cli/atomic_output.rs b/crates/bsk-cli/src/cli/atomic_output.rs new file mode 100644 index 0000000..5efacf2 --- /dev/null +++ b/crates/bsk-cli/src/cli/atomic_output.rs @@ -0,0 +1,75 @@ +//! Atomic visibility boundary for files received into a same-directory temp. + +use std::path::Path; + +pub fn commit(temp: &Path, out: &Path, overwrite: bool) -> std::io::Result<()> { + if !overwrite { + std::fs::hard_link(temp, out)?; + std::fs::remove_file(temp)?; + return Ok(()); + } + replace(temp, out) +} + +#[cfg(unix)] +fn replace(temp: &Path, out: &Path) -> std::io::Result<()> { + // POSIX rename replaces an existing non-directory destination atomically. + std::fs::rename(temp, out) +} + +#[cfg(windows)] +fn replace(temp: &Path, out: &Path) -> std::io::Result<()> { + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::Storage::FileSystem::{ + MOVEFILE_REPLACE_EXISTING, MOVEFILE_WRITE_THROUGH, MoveFileExW, + }; + + let from: Vec = temp.as_os_str().encode_wide().chain(Some(0)).collect(); + let to: Vec = out.as_os_str().encode_wide().chain(Some(0)).collect(); + // SAFETY: both buffers are NUL-terminated and remain alive for the call. + let ok = unsafe { + MoveFileExW( + from.as_ptr(), + to.as_ptr(), + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH, + ) + }; + if ok == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } +} + +#[cfg(not(any(unix, windows)))] +fn replace(temp: &Path, out: &Path) -> std::io::Result<()> { + std::fs::rename(temp, out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn no_replace_never_overwrites_existing_output() { + let dir = tempfile::tempdir().unwrap(); + let temp = dir.path().join("new.part"); + let out = dir.path().join("out.bin"); + std::fs::write(&temp, b"new").unwrap(); + std::fs::write(&out, b"old").unwrap(); + assert!(commit(&temp, &out, false).is_err()); + assert_eq!(std::fs::read(&out).unwrap(), b"old"); + } + + #[test] + fn overwrite_replaces_existing_output_without_predelete() { + let dir = tempfile::tempdir().unwrap(); + let temp = dir.path().join("new.part"); + let out = dir.path().join("out.bin"); + std::fs::write(&temp, b"new").unwrap(); + std::fs::write(&out, b"old").unwrap(); + commit(&temp, &out, true).unwrap(); + assert_eq!(std::fs::read(&out).unwrap(), b"new"); + assert!(!temp.exists()); + } +} diff --git a/crates/bsk-cli/src/cli/download.rs b/crates/bsk-cli/src/cli/download.rs index 043b4d1..5a97b28 100644 --- a/crates/bsk-cli/src/cli/download.rs +++ b/crates/bsk-cli/src/cli/download.rs @@ -15,6 +15,7 @@ use bsk_protocol::tools::{ use clap::Args; use uuid::Uuid; +use crate::cli::atomic_output; use crate::cli::ensure_daemon::ensure_daemon; use crate::cli::error::{CliError, Format}; use crate::cli::interaction::split_target; @@ -56,6 +57,7 @@ pub fn dispatch(args: DownloadArgs, format: Format) -> Result<(), CliError> { tab_id: args.tab_id, timeout_ms: Some(args.timeout), browser_relative_dir: None, + max_byte_size: None, }; let reply: DownloadResult = crate::cli::business_rpc::call( info.sock_path.clone(), @@ -130,22 +132,8 @@ fn write_transfer(sock: &Path, id: &str, out: &Path, overwrite: bool) -> Result< } file.sync_all().map_err(|e| CliError::Local(e.into()))?; drop(file); - if !overwrite { - std::fs::hard_link(&temp, out) - .with_context(|| format!("commit download without overwriting {}", out.display())) - .map_err(CliError::Local)?; - std::fs::remove_file(&temp) - .with_context(|| format!("remove temporary download {}", temp.display())) - .map_err(CliError::Local)?; - return Ok(()); - } - if out.exists() { - std::fs::remove_file(out) - .with_context(|| format!("replace existing output {}", out.display())) - .map_err(CliError::Local)?; - } - std::fs::rename(&temp, out) - .with_context(|| format!("commit download to {}", out.display())) + atomic_output::commit(&temp, out, overwrite) + .with_context(|| format!("atomically commit download to {}", out.display())) .map_err(CliError::Local) })(); if result.is_err() { diff --git a/crates/bsk-cli/src/cli/mod.rs b/crates/bsk-cli/src/cli/mod.rs index 249f556..4776ebe 100644 --- a/crates/bsk-cli/src/cli/mod.rs +++ b/crates/bsk-cli/src/cli/mod.rs @@ -2,6 +2,7 @@ use std::time::Duration; +mod atomic_output; pub mod browser_wait; pub mod browsers; pub mod business_rpc; diff --git a/crates/bsk-cli/src/cli/render_error.rs b/crates/bsk-cli/src/cli/render_error.rs index bb2a0a6..05db2b7 100644 --- a/crates/bsk-cli/src/cli/render_error.rs +++ b/crates/bsk-cli/src/cli/render_error.rs @@ -53,6 +53,8 @@ pub mod reason { pub const FILE_INPUT_NOT_ACTIVATED: &str = "file_input_not_activated"; pub const SET_FILE_INPUT_FAILED: &str = "set_file_input_failed"; pub const DOWNLOAD_CAPTURE_FAILED: &str = "download_capture_failed"; + pub const TRANSFER_OUTCOME_UNKNOWN: &str = "transfer_outcome_unknown"; + pub const TRANSFER_TIMEOUT: &str = "transfer_timeout"; pub const SESSION_BUSY: &str = crate::rpc_reason::SESSION_BUSY; pub const RECORD_START_PAGE_UNREACHABLE: &str = "record_start_page_unreachable"; } @@ -201,6 +203,20 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend return base; }; match (code, reason) { + (_, reason::TRANSFER_OUTCOME_UNKNOWN) => RenderInfo { + summary: "the file transfer outcome could not be confirmed", + hint: Some( + "do not retry the transfer: the browser may already have applied it; inspect the page or stop safely", + ), + exit_code: base.exit_code, + }, + (ErrorCode::Timeout, reason::TRANSFER_TIMEOUT) => RenderInfo { + summary: "the file transfer timed out after browser dispatch", + hint: Some( + "do not retry when effect_state is unknown or committed; inspect the page before taking another action", + ), + exit_code: base.exit_code, + }, (ErrorCode::PermissionDenied, reason::ELEMENT_NOT_VISIBLE) => RenderInfo { summary: "target element has no visible geometry", hint: Some( @@ -474,6 +490,15 @@ mod tests { let download = serde_json::json!({ "reason": reason::DOWNLOAD_CAPTURE_FAILED }); let info = info_for_error(ErrorCode::CdpFailed, Some(&download)); assert!(info.summary.contains("download could not be attributed")); + + let unknown = serde_json::json!({ "reason": reason::TRANSFER_OUTCOME_UNKNOWN }); + let info = info_for_error(ErrorCode::ProtocolError, Some(&unknown)); + assert!(info.summary.contains("outcome could not be confirmed")); + assert!(info.hint.unwrap().contains("do not retry")); + + let timeout = serde_json::json!({ "reason": reason::TRANSFER_TIMEOUT }); + let info = info_for_error(ErrorCode::Timeout, Some(&timeout)); + assert!(info.summary.contains("timed out after browser dispatch")); } #[test] diff --git a/crates/bsk-cli/src/cli/upload.rs b/crates/bsk-cli/src/cli/upload.rs index 9614e0b..7636453 100644 --- a/crates/bsk-cli/src/cli/upload.rs +++ b/crates/bsk-cli/src/cli/upload.rs @@ -49,38 +49,43 @@ pub fn dispatch(args: UploadArgs, format: Format) -> Result<(), CliError> { let info = ensure_daemon().context("ensure daemon is running")?; let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?; let mut staged = Vec::new(); - let result = (|| { - for path in &args.files { - staged.push(stage_file(&info.sock_path, &args.session, path)?); - } - let params = UploadParams { - session_id: args.session, - ref_, - selector, - tab_id: args.tab_id, - files: staged - .iter() - .map(|(id, name)| UploadFile { - transfer_id: id.clone(), - name: name.clone(), - staged_path: None, - }) - .collect(), - timeout_ms: Some(args.timeout), - }; - crate::cli::business_rpc::call::<_, UploadResult>( - info.sock_path.clone(), - "upload", - Method::ToolUpload, - Some(params), - ipc_timeout(args.timeout), - ) - })(); - if result.is_err() { - for (id, _) in &staged { - let _ = release(&info.sock_path, id); + for path in &args.files { + match stage_file(&info.sock_path, &args.session, path) { + Ok(file) => staged.push(file), + Err(err) => { + for (id, _) in &staged { + let _ = release(&info.sock_path, id); + } + return Err(err); + } } } + let params = UploadParams { + session_id: args.session, + ref_, + selector, + tab_id: args.tab_id, + files: staged + .iter() + .map(|(id, name)| UploadFile { + transfer_id: id.clone(), + name: name.clone(), + staged_path: None, + }) + .collect(), + timeout_ms: Some(args.timeout), + }; + let result = crate::cli::business_rpc::call::<_, UploadResult>( + info.sock_path.clone(), + "upload", + Method::ToolUpload, + Some(params), + ipc_timeout(args.timeout), + ); + // Once tool.upload is dispatched, staging ownership belongs to the + // session. A transport timeout cannot prove that Chrome did not attach + // the file, so releasing here could invalidate a late successful attach. + // Session teardown remains the single cleanup boundary after dispatch. let reply = result?; match format { Format::Json => println!("{}", serde_json::to_string_pretty(&reply).unwrap()), diff --git a/crates/bsk-cli/src/daemon/file_transfer.rs b/crates/bsk-cli/src/daemon/file_transfer.rs index b63ca50..397e30b 100644 --- a/crates/bsk-cli/src/daemon/file_transfer.rs +++ b/crates/bsk-cli/src/daemon/file_transfer.rs @@ -40,6 +40,67 @@ struct Entry { ready: bool, } +/// Browser download path after the daemon has verified that it belongs to +/// the capability minted for this transfer. From this point onward the daemon +/// owns cleanup on every error path; unvalidated arbitrary paths are never +/// removed. +#[derive(Debug)] +struct ValidatedBrowserFile { + path: PathBuf, + parent: PathBuf, + cleanup_armed: bool, +} + +impl ValidatedBrowserFile { + fn validate(reported_path: &Path, transfer_id: &str) -> Result { + if !reported_path.is_absolute() { + return Err(permission( + "download path is outside its browser capability", + )); + } + let expected_parent = Path::new("BrowserSkill").join(transfer_id); + let parent = reported_path + .parent() + .ok_or_else(|| permission("download path has no parent directory"))?; + if !parent.ends_with(&expected_parent) { + return Err(permission( + "download escaped its browser capability directory", + )); + } + for path in [reported_path, parent] { + if fs::symlink_metadata(path) + .map_err(io_error)? + .file_type() + .is_symlink() + { + return Err(permission("download capability path contains a symlink")); + } + } + Ok(Self { + path: reported_path.to_path_buf(), + parent: parent.to_path_buf(), + cleanup_armed: true, + }) + } + + fn remove_source(mut self) -> Result<(), RpcError> { + remove_if_present(&self.path).map_err(io_error)?; + remove_dir_if_present(&self.parent).map_err(io_error)?; + self.cleanup_armed = false; + Ok(()) + } +} + +impl Drop for ValidatedBrowserFile { + fn drop(&mut self) { + if !self.cleanup_armed { + return; + } + let _ = remove_if_present(&self.path); + let _ = remove_dir_if_present(&self.parent); + } +} + #[derive(Debug)] pub struct DownloadStaging { pub transfer_id: String, @@ -259,30 +320,11 @@ impl TransferRegistry { let entry = entries .get_mut(id) .ok_or_else(|| not_found("download transfer not found"))?; - if entry.direction != Direction::Download || !reported_path.is_absolute() { - return Err(permission( - "download path is outside its browser capability", - )); + if entry.direction != Direction::Download { + return Err(permission("transfer is not a download capability")); } - let expected_parent = Path::new("BrowserSkill").join(id); - let reported_parent = reported_path - .parent() - .ok_or_else(|| permission("download path has no parent directory"))?; - if !reported_parent.ends_with(&expected_parent) { - return Err(permission( - "download escaped its browser capability directory", - )); - } - for path in [reported_path, reported_parent] { - if fs::symlink_metadata(path) - .map_err(io_error)? - .file_type() - .is_symlink() - { - return Err(permission("download capability path contains a symlink")); - } - } - let canonical = reported_path.canonicalize().map_err(io_error)?; + let browser_file = ValidatedBrowserFile::validate(reported_path, id)?; + let canonical = browser_file.path.canonicalize().map_err(io_error)?; let meta = fs::metadata(&canonical).map_err(io_error)?; if !meta.is_file() || meta.len() > MAX_TRANSFER_BYTES { return Err(invalid( @@ -310,8 +352,7 @@ impl TransferRegistry { entry.written = copied; entry.expected_size = Some(copied); entry.ready = true; - let _ = fs::remove_file(&canonical); - let _ = fs::remove_dir(reported_parent); + browser_file.remove_source()?; Ok(copied) } @@ -391,6 +432,22 @@ fn set_private_file(file: &File) -> std::io::Result<()> { Ok(()) } +fn remove_if_present(path: &Path) -> std::io::Result<()> { + match fs::remove_file(path) { + Ok(()) => Ok(()), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(err) => Err(err), + } +} + +fn remove_dir_if_present(path: &Path) -> std::io::Result<()> { + match fs::remove_dir(path) { + Ok(()) => Ok(()), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(err) => Err(err), + } +} + fn safe_upload_basename(name: &str) -> Result<&str, RpcError> { if name.is_empty() || name.contains(['/', '\\', '\0']) { return Err(invalid("upload name must be a safe basename")); @@ -570,6 +627,7 @@ mod tests { .import_download(&staging.transfer_id, &outside) .is_err() ); + assert_eq!(fs::read(&outside).unwrap(), b"secret"); let browser_dir = temp .path() @@ -601,6 +659,31 @@ mod tests { ); } + #[test] + fn validated_oversized_browser_download_is_removed_on_rejection() { + let (temp, registry) = registry(); + let staging = registry.begin_download("s1").unwrap(); + let browser_dir = temp + .path() + .join("Downloads") + .join("BrowserSkill") + .join(&staging.transfer_id); + fs::create_dir_all(&browser_dir).unwrap(); + let inside = browser_dir.join("oversized.bin"); + File::create(&inside) + .unwrap() + .set_len(MAX_TRANSFER_BYTES + 1) + .unwrap(); + + assert!( + registry + .import_download(&staging.transfer_id, &inside) + .is_err() + ); + assert!(!inside.exists()); + assert!(!browser_dir.exists()); + } + #[test] fn releasing_a_session_removes_all_staging() { let (_temp, registry) = registry(); diff --git a/crates/bsk-cli/src/daemon/ipc.rs b/crates/bsk-cli/src/daemon/ipc.rs index 9a6d0bd..186cade 100644 --- a/crates/bsk-cli/src/daemon/ipc.rs +++ b/crates/bsk-cli/src/daemon/ipc.rs @@ -382,6 +382,7 @@ async fn handle_tool_dispatch( Err(err) => return ResponseBody::Err(err), }; download.browser_relative_dir = Some(staging.browser_relative_dir); + download.max_byte_size = Some(super::file_transfer::MAX_TRANSFER_BYTES); download_transfer_id = Some(staging.transfer_id); params = serde_json::to_value(download).unwrap_or(Value::Null); } @@ -702,7 +703,7 @@ fn tool_dispatch_timeout(params: &Value) -> Result { fn tool_dispatch_transport_timeout(method: &Method, params: &Value) -> Result { tool_dispatch_timeout(params).map(|timeout| { - if method == &Method::ToolUpload { + if matches!(method, Method::ToolUpload | Method::ToolDownload) { timeout.saturating_add(EXTENSION_RESPONSE_GRACE) } else { timeout diff --git a/crates/bsk-cli/src/daemon/queue.rs b/crates/bsk-cli/src/daemon/queue.rs index 05f09e8..6b7366a 100644 --- a/crates/bsk-cli/src/daemon/queue.rs +++ b/crates/bsk-cli/src/daemon/queue.rs @@ -438,6 +438,7 @@ async fn dispatch_with_sender( lifecycle_cancel: Option, cancel_cleanup_timeout: Duration, ) -> Result { + let effect_aware_transfer = is_effect_aware_transfer(&method); let (respond_tx, respond_rx) = oneshot::channel(); let job = ToolJob { method, @@ -474,15 +475,21 @@ async fn dispatch_with_sender( mpsc::error::TrySendError::Closed(_) => DispatchError::QueueClosed, }); } - let mut response_timeout = timeout.saturating_add(Duration::from_secs(1)); - if lifecycle_cancellable { - // A lifecycle cancel can arrive at the original RPC deadline and - // intentionally keeps the worker alive while extension cleanup - // settles. Do not let this outer waiter abandon that reconciliation - // early, which could reopen a daemon session whose window is gone. - response_timeout = response_timeout.saturating_add(cancel_cleanup_timeout); - } - let waited = tokio::time::timeout(response_timeout, respond_rx).await; + // Lifecycle teardown and effect-aware transfers both keep the worker busy + // for bounded compensation after their original deadline. Keep the outer + // waiter alive for the same grace period so it cannot abandon reconciliation. + let response_grace = if lifecycle_cancellable || effect_aware_transfer { + cancel_cleanup_timeout + } else { + Duration::ZERO + }; + let waited = tokio::time::timeout( + timeout + .saturating_add(response_grace) + .saturating_add(Duration::from_secs(1)), + respond_rx, + ) + .await; match waited { Ok(Ok(Ok(v))) => Ok(v), Ok(Ok(Err(rpc))) => Err(DispatchError::Rpc(rpc)), @@ -636,44 +643,85 @@ async fn forward_one( .lifecycle_cancel .as_ref() .map(|_| &forward_lifecycle_cancel as &(dyn Fn() + Sync)); + let send_deadline_cancel = || { + client + .sink + .send(Frame::Request(RequestFrame { + id: format!("deadline-cancel-{rpc_id}"), + method: Method::Cancel, + params: Some(serde_json::json!({ "rpc_id": rpc_id })), + })) + .is_ok() + }; + let deadline_cancel: Option<&(dyn Fn() -> bool + Sync)> = + is_effect_aware_transfer(&job.method).then_some(&send_deadline_cancel); let waited = await_with_optional_cancel( job.timeout, job.cancel_cleanup_timeout, waiter, cancel_token.as_ref(), on_abort, + deadline_cancel, ) .await; let response = match waited { WaitOutcome::Response(resp) => resp, WaitOutcome::CancelledAfterResponse(resp) => { if job.method == Method::ToolSessionStop - && let ResponseBody::Ok(value) = resp.body + && let ResponseBody::Ok(value) = &resp.body { // Teardown crossed its final irreversible boundary before // the cancel landed. Commit the real extension result so the // daemon does not retain a session whose window is gone. - return Ok(value); + return Ok(value.clone()); } - // Cancellation wins the external verdict, but only after the - // extension's original RPC has settled. Preserve any non-cancel - // error so compensation failures remain explicit instead of - // being hidden behind a generic cancelled result. - if let ResponseBody::Err(err) = resp.body - && !matches!(err.code, ErrorCode::Cancelled | ErrorCode::UserAborted) - { - return Err(err); + // File transfer commits are irreversible. A late cancel cannot + // overwrite a confirmed success, and an unknown transfer effect + // must remain explicit so callers do not retry or release upload + // staging as though nothing happened. + if is_effect_aware_transfer(&job.method) { + match &resp.body { + ResponseBody::Ok(_) => resp, + ResponseBody::Err(err) + if transfer_effect(err) == Some("unknown") + || transfer_effect(err) == Some("committed") => + { + return Err(err.clone()); + } + ResponseBody::Err(err) + if !matches!(err.code, ErrorCode::Cancelled | ErrorCode::UserAborted) => + { + return Err(err.clone()); + } + ResponseBody::Err(_) => { + return Err(cancelled_error( + job.inflight.as_deref(), + "tool dispatch cancelled after extension cleanup", + )); + } + } + } else { + // For ordinary tools cancellation keeps the existing verdict, + // while non-cancel errors still expose compensation failures. + if let ResponseBody::Err(err) = resp.body + && !matches!(err.code, ErrorCode::Cancelled | ErrorCode::UserAborted) + { + return Err(err); + } + return Err(cancelled_error( + job.inflight.as_deref(), + "tool dispatch cancelled after extension cleanup", + )); } - return Err(cancelled_error( - job.inflight.as_deref(), - "tool dispatch cancelled after extension cleanup", - )); } WaitOutcome::TimedOutAfterResponse(resp) => match resp.body { - ResponseBody::Ok(value) if job.method == Method::ToolSessionStop => { + ResponseBody::Ok(value) + if job.method == Method::ToolSessionStop + || is_effect_aware_transfer(&job.method) => + { // The close crossed its irreversible boundary during the - // timeout cleanup grace. Reconcile daemon state instead of - // reopening a session whose window is already gone. + // timeout cleanup grace, or the transfer committed before its + // deadline cancel settled. Preserve the irreversible result. return Ok(value); } ResponseBody::Err(err) @@ -681,6 +729,9 @@ async fn forward_one( { return Err(err); } + ResponseBody::Err(err) if is_effect_aware_transfer(&job.method) => { + return Err(timed_out_transfer_error(&err)); + } _ => { return Err(RpcError { code: ErrorCode::Timeout, @@ -691,6 +742,14 @@ async fn forward_one( }, WaitOutcome::CleanupTimeout => { client.pending.lock().unwrap().cancel(&rpc_id); + if is_effect_aware_transfer(&job.method) { + return Err(unknown_transfer_error( + ErrorCode::Timeout, + "cancelled file transfer did not confirm its outcome before cleanup timed out", + "cleanup", + true, + )); + } return Err(RpcError { code: ErrorCode::Timeout, message: format!( @@ -700,8 +759,25 @@ async fn forward_one( data: Some(serde_json::json!({ "reason": "cancel_cleanup_timeout" })), }); } + WaitOutcome::TimeoutCleanupFailed => { + client.pending.lock().unwrap().cancel(&rpc_id); + return Err(unknown_transfer_error( + ErrorCode::Timeout, + "file transfer timed out and cleanup could not be confirmed", + "cleanup", + true, + )); + } WaitOutcome::WaiterClosed => { client.pending.lock().unwrap().cancel(&rpc_id); + if is_effect_aware_transfer(&job.method) { + return Err(unknown_transfer_error( + ErrorCode::ProtocolError, + "file transfer transport closed after dispatch; outcome is unknown", + "transport", + false, + )); + } return Err(RpcError { code: ErrorCode::ProtocolError, message: "transport closed mid-call".into(), @@ -710,6 +786,14 @@ async fn forward_one( } WaitOutcome::Timeout => { client.pending.lock().unwrap().cancel(&rpc_id); + if is_effect_aware_transfer(&job.method) { + return Err(unknown_transfer_error( + ErrorCode::Timeout, + "file transfer timed out after dispatch; outcome is unknown", + "transport", + false, + )); + } return Err(RpcError { code: ErrorCode::Timeout, message: format!("tool RPC timed out after {:?}", job.timeout), @@ -723,12 +807,65 @@ async fn forward_one( } } +fn is_effect_aware_transfer(method: &Method) -> bool { + matches!(method, Method::ToolUpload | Method::ToolDownload) +} + +fn transfer_effect(err: &RpcError) -> Option<&str> { + err.data.as_ref()?.get("effect_state")?.as_str() +} + +fn unknown_transfer_error( + code: ErrorCode, + message: impl Into, + phase: &str, + cleanup_failed: bool, +) -> RpcError { + let mut data = serde_json::json!({ + "reason": "transfer_outcome_unknown", + "effect_state": "unknown", + "phase": phase, + }); + if cleanup_failed { + data["cleanup_state"] = serde_json::json!("failed"); + } + RpcError { + code, + message: message.into(), + data: Some(data), + } +} + +fn timed_out_transfer_error(err: &RpcError) -> RpcError { + let data = err.data.as_ref(); + RpcError { + code: ErrorCode::Timeout, + message: "file transfer timed out after dispatch".into(), + data: Some(serde_json::json!({ + "reason": "transfer_timeout", + "effect_state": data + .and_then(|value| value.get("effect_state")) + .and_then(Value::as_str) + .unwrap_or("unknown"), + "phase": data + .and_then(|value| value.get("phase")) + .and_then(Value::as_str) + .unwrap_or("cleanup"), + "cleanup_state": data + .and_then(|value| value.get("cleanup_state")) + .and_then(Value::as_str) + .unwrap_or("complete"), + })), + } +} + #[derive(Debug)] enum WaitOutcome { Response(bsk_protocol::ResponseFrame), CancelledAfterResponse(bsk_protocol::ResponseFrame), TimedOutAfterResponse(bsk_protocol::ResponseFrame), CleanupTimeout, + TimeoutCleanupFailed, WaiterClosed, Timeout, } @@ -739,6 +876,7 @@ async fn await_with_optional_cancel( mut waiter: oneshot::Receiver, cancel: Option<&super::abort::AbortToken>, on_abort: Option<&(dyn Fn() + Sync)>, + on_deadline: Option<&(dyn Fn() -> bool + Sync)>, ) -> WaitOutcome { match cancel { Some(token) => { @@ -764,6 +902,16 @@ async fn await_with_optional_cancel( Err(_) => WaitOutcome::WaiterClosed, }, _ = &mut deadline => { + if let Some(send_cancel) = on_deadline { + return if send_cancel() { + match tokio::time::timeout(cleanup_timeout, &mut waiter).await { + Ok(Ok(resp)) => WaitOutcome::TimedOutAfterResponse(resp), + Ok(Err(_)) | Err(_) => WaitOutcome::TimeoutCleanupFailed, + } + } else { + WaitOutcome::TimeoutCleanupFailed + }; + } let Some(on_abort) = on_abort else { return WaitOutcome::Timeout; }; @@ -779,7 +927,16 @@ async fn await_with_optional_cancel( None => match tokio::time::timeout(timeout, &mut waiter).await { Ok(Ok(resp)) => WaitOutcome::Response(resp), Ok(Err(_)) => WaitOutcome::WaiterClosed, - Err(_) => WaitOutcome::Timeout, + Err(_) => match on_deadline { + Some(send_cancel) if send_cancel() => { + match tokio::time::timeout(cleanup_timeout, &mut waiter).await { + Ok(Ok(resp)) => WaitOutcome::TimedOutAfterResponse(resp), + Ok(Err(_)) | Err(_) => WaitOutcome::TimeoutCleanupFailed, + } + } + Some(_) => WaitOutcome::TimeoutCleanupFailed, + None => WaitOutcome::Timeout, + }, }, } } @@ -864,6 +1021,7 @@ mod await_with_optional_cancel_tests { rx, Some(&token), None, + None, ) .await; assert!( @@ -887,6 +1045,7 @@ mod await_with_optional_cancel_tests { rx, Some(&token), None, + None, ) .await; match outcome { @@ -907,6 +1066,7 @@ mod await_with_optional_cancel_tests { rx, None, None, + None, ) .await; assert!(matches!(outcome, WaitOutcome::Response(_))); @@ -924,6 +1084,7 @@ mod await_with_optional_cancel_tests { rx, Some(&token), None, + None, ) .await }); @@ -948,6 +1109,7 @@ mod await_with_optional_cancel_tests { rx, Some(&token), None, + None, ) .await; assert!(matches!(outcome, WaitOutcome::CleanupTimeout)); @@ -972,12 +1134,49 @@ mod await_with_optional_cancel_tests { rx, Some(&token), Some(&forward_abort), + None, ) .await; assert!(abort_forwarded.load(Ordering::SeqCst)); assert!(matches!(outcome, WaitOutcome::TimedOutAfterResponse(_))); } + + #[tokio::test] + async fn transfer_deadline_sends_cancel_and_waits_for_cleanup_response() { + let (tx, rx) = oneshot::channel(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(5)).await; + tx.send(dummy_response()).unwrap(); + }); + let send_cancel = || true; + let outcome = await_with_optional_cancel( + Duration::from_millis(1), + Duration::from_secs(1), + rx, + None, + None, + Some(&send_cancel), + ) + .await; + assert!(matches!(outcome, WaitOutcome::TimedOutAfterResponse(_))); + } + + #[tokio::test] + async fn transfer_deadline_is_unknown_when_cancel_cannot_be_sent() { + let (_tx, rx) = oneshot::channel(); + let send_cancel = || false; + let outcome = await_with_optional_cancel( + Duration::from_millis(1), + Duration::from_secs(1), + rx, + None, + None, + Some(&send_cancel), + ) + .await; + assert!(matches!(outcome, WaitOutcome::TimeoutCleanupFailed)); + } } #[cfg(test)] diff --git a/crates/bsk-protocol/schema/tool_download_params.json b/crates/bsk-protocol/schema/tool_download_params.json index b6487cc..3bfbba7 100644 --- a/crates/bsk-protocol/schema/tool_download_params.json +++ b/crates/bsk-protocol/schema/tool_download_params.json @@ -13,6 +13,15 @@ "null" ] }, + "max_byte_size": { + "description": "Daemon-injected authoritative transfer size limit. The extension uses it only for early cancellation; daemon import remains the final check.", + "type": [ + "integer", + "null" + ], + "format": "uint64", + "minimum": 0.0 + }, "ref": { "type": [ "string", diff --git a/crates/bsk-protocol/src/tools/file_transfer.rs b/crates/bsk-protocol/src/tools/file_transfer.rs index f2517ce..9677785 100644 --- a/crates/bsk-protocol/src/tools/file_transfer.rs +++ b/crates/bsk-protocol/src/tools/file_transfer.rs @@ -63,6 +63,10 @@ pub struct DownloadParams { /// Daemon-injected relative directory beneath Chrome's Downloads root. #[serde(default, skip_serializing_if = "Option::is_none")] pub browser_relative_dir: Option, + /// Daemon-injected authoritative transfer size limit. The extension uses + /// it only for early cancellation; daemon import remains the final check. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_byte_size: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] @@ -163,8 +167,10 @@ mod tests { tab_id: None, timeout_ms: None, browser_relative_dir: None, + max_byte_size: None, }) .unwrap(); assert!(value.get("browser_relative_dir").is_none()); + assert!(value.get("max_byte_size").is_none()); } } diff --git a/docs/architecture.md b/docs/architecture.md index d86748d..cfaf9e5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -146,10 +146,11 @@ flowchart LR ### File-transfer boundary - The invoking agent/harness decides whether a transfer is authorized and supplies the task-local source or destination path. -- The CLI is the only component that reads an upload source or writes the final download destination. The extension never receives either agent-facing path. -- The daemon issues opaque, session-scoped transfer IDs and stages bounded chunks in a private runtime directory. For upload it injects only its private staged paths. For download it mints a relative Chrome directory capability, validates the reported completed path against that capability, rejects symlinks and oversized/non-regular files, then imports the bytes into private staging. -- The extension only performs the browser-side mechanism, scoped to the requested session and exact Agent Window target. Upload installs a transaction-scoped DOM guard that records the one `` actually activated by the requested click while cancelling its native default action, then uses `DOM.setFileInputFiles`. File System Access picker entry points are intercepted only for that transaction and return a structured fallback. Download requires an exact-target `Page.downloadWillBegin` intent before a matching `chrome.downloads` item is routed into the daemon-minted relative directory. The extension does not classify content or make policy decisions. -- Download staging is released after the CLI commits the file. Upload staging remains until session teardown because the page may read an attached file only on a later form submission. All staging is released on session stop/browser disconnect and on daemon startup after a crash. Existing download destinations are not overwritten unless the CLI caller explicitly opts in. +- The CLI is the only component that reads an upload source or writes the final download destination. Before browser dispatch it owns rollback of partially staged uploads; after dispatch, ownership moves to the session because a transport timeout cannot prove that Chrome did not attach the file. Download output becomes visible through one atomic commit, and replacement is opt-in without a pre-delete window. The extension never receives either agent-facing path. +- The daemon is the authority for storage capabilities and limits. It issues opaque session-scoped transfer IDs, stages bounded chunks in a private runtime directory, and injects only private staged upload paths. For download it mints one relative Chrome directory capability. Only after validating the reported path, file type, symlink boundary, and authoritative byte limit does it take ownership of browser-file cleanup and import the bytes. +- The extension owns only the browser transaction. Both tools resolve one `ResolvedActionTarget` and use that same target for protocol setup and the triggering click. Upload arms Chrome's chooser interception before clicking, then accepts either an exact `Page.fileChooserOpened` input node or an independent probe anchored in the trigger node's own document; one verified input is committed with `DOM.setFileInputFiles`. A non-input picker is rejected immediately as unsupported. Download correlates exact-target CDP intent and `chrome.downloads` filename candidates in either arrival order, claims only one unique match, and never cancels an unclaimed candidate. +- Browser-side operations report `effect_state` (`none`, `committed`, or `unknown`), `phase`, and `cleanup_state`. Confirmed success wins over a late cancel; an unknown effect is preserved across timeout or transport loss and must not be retried blindly. A transfer deadline sends cancellation to the extension and keeps the session queue occupied for bounded compensation rather than abandoning an in-flight browser effect. +- Download staging is released after CLI commit. Upload staging remains until session teardown because the page may read an attached file only on a later form submission. Remaining staging is released on session stop/browser disconnect and on daemon startup after a crash. BrowserSkill does not inspect content or decide whether a transfer is appropriate. ## Repository layout diff --git a/skill/SKILL.md b/skill/SKILL.md index b13f631..8af5383 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -215,7 +215,7 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff The agent/harness decides whether a file transfer is appropriate and which local path belongs to the task. Treat upload as disclosure of that file to the current website, and download as accepting website-controlled bytes onto the local filesystem. Use only paths that are necessary for the user's bounded goal. -BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted capabilities reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. Upload captures the file input actually activated by the requested click and assigns only the staged file paths. Download requires an exact-target browser intent before it claims one Chrome download, routes it through a daemon-minted relative directory, and lets the daemon validate and import it. Upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive. +BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted capabilities reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. Upload intercepts the native chooser for one transaction, locates the input activated in the resolved target's document, and assigns only the staged file paths. Download uniquely correlates one exact-target browser intent with one Chrome download in either event order, routes it through a daemon-minted relative directory, and lets the daemon validate and import it. Upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive. Do not use `request-help` merely because a native file chooser or browser download is involved; try these commands first. For transfer failures, use the structured error instead of retrying blindly: @@ -223,6 +223,9 @@ Do not use `request-help` merely because a native file chooser or browser downlo - `reason=file_input_probe_failed` means BrowserSkill could not safely establish the browser-side upload transaction. Do not repeat the same action; use `request-help` when available. - `reason=set_file_input_failed` means BrowserSkill found the activated file input but Chrome rejected the staged path or assignment. Check the extension's file-URL access permission; otherwise use `request-help`. - `reason=download_capture_failed` means BrowserSkill could not attribute exactly one completed download to the requested target. Do not retry blindly or accept an unrelated browser download; use `request-help` when available. +- `effect_state=none` means BrowserSkill confirmed that no file-transfer effect was committed. Follow the accompanying reason; a corrected target or explicit human fallback may be attempted. +- `effect_state=unknown` means the browser may already have attached or created the file. Do not repeat the transfer. Observe the page if that can establish the result; otherwise stop and report the uncertainty. +- `effect_state=committed` means the browser-side effect occurred even if later completion or cleanup failed. Do not repeat it; continue only after verifying the resulting page/download state. If `request-help` returns `outcome="disabled"`, do not retry it. Stop gracefully and report the transfer mechanism that requires human intervention. From b801bc1f6209111861351f4807bfe4b9146435d0 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Fri, 21 Aug 2026 16:34:06 +0800 Subject: [PATCH 07/12] fix(bsk): fix ci --- .../src/tools/__tests__/file-transfer.test.ts | 6 +++- apps/extension/src/tools/download-capture.ts | 6 ++-- apps/extension/src/tools/download.ts | 6 +--- .../src/tools/file-input-transaction.ts | 29 ++++++++++++------- apps/extension/src/tools/upload.ts | 6 +--- 5 files changed, 30 insertions(+), 23 deletions(-) diff --git a/apps/extension/src/tools/__tests__/file-transfer.test.ts b/apps/extension/src/tools/__tests__/file-transfer.test.ts index 81dbe01..1e30091 100644 --- a/apps/extension/src/tools/__tests__/file-transfer.test.ts +++ b/apps/extension/src/tools/__tests__/file-transfer.test.ts @@ -440,7 +440,11 @@ describe("file transfer tools", () => { totalBytes: 4, bytesReceived: 0, } as chrome.downloads.DownloadItem; - const complete = { ...initial, state: "complete", fileSize: 4 } as chrome.downloads.DownloadItem; + const complete = { + ...initial, + state: "complete", + fileSize: 4, + } as chrome.downloads.DownloadItem; const downloads: DownloadsApi = { onCreated, onChanged, diff --git a/apps/extension/src/tools/download-capture.ts b/apps/extension/src/tools/download-capture.ts index 6d909b2..9a6ca9d 100644 --- a/apps/extension/src/tools/download-capture.ts +++ b/apps/extension/src/tools/download-capture.ts @@ -298,7 +298,8 @@ export async function captureBrowserDownload( const triggered = await options.trigger(); if (isRpcError(triggered)) { void completion.catch(() => undefined); - const effect: TransferEffectState = capturedId !== undefined ? "committed" : intent ? "unknown" : "none"; + const effect: TransferEffectState = + capturedId !== undefined ? "committed" : intent ? "unknown" : "none"; failureResult = { ...triggered, data: { ...triggered.data, effect_state: effect, phase: "trigger" }, @@ -310,7 +311,8 @@ export async function captureBrowserDownload( succeeded = true; return { click, item }; } catch (err) { - const effect: TransferEffectState = capturedId !== undefined ? "committed" : click ? "unknown" : "none"; + const effect: TransferEffectState = + capturedId !== undefined ? "committed" : click ? "unknown" : "none"; failureResult = captureError( err instanceof Error ? err.message : String(err), effect, diff --git a/apps/extension/src/tools/download.ts b/apps/extension/src/tools/download.ts index fb2c586..047e263 100644 --- a/apps/extension/src/tools/download.ts +++ b/apps/extension/src/tools/download.ts @@ -4,11 +4,7 @@ import type { SessionManager } from "@/session-manager/manager"; import type { DownloadParams, DownloadResult, RpcError } from "@/transport/types"; import { captureBrowserDownload, chromeDownloadsApi, type DownloadsApi } from "./download-capture"; -import { - clickResolvedTarget, - type InteractionDeps, - resolveActionTarget, -} from "./interaction"; +import { clickResolvedTarget, type InteractionDeps, resolveActionTarget } from "./interaction"; import { enforceAgentWindow, isRpcError, lookupSession, resolveTargetTab } from "./shared"; let downloadActive = false; diff --git a/apps/extension/src/tools/file-input-transaction.ts b/apps/extension/src/tools/file-input-transaction.ts index fb3901f..3c78a8d 100644 --- a/apps/extension/src/tools/file-input-transaction.ts +++ b/apps/extension/src/tools/file-input-transaction.ts @@ -46,7 +46,10 @@ export interface FileInputTransactionResult { } class BoundedWaitError extends Error { - constructor(readonly kind: "timeout" | "aborted", message: string) { + constructor( + readonly kind: "timeout" | "aborted", + message: string, + ) { super(message); } } @@ -99,7 +102,11 @@ function transferFailure( return transferError(code, reason, message, { effectState, phase }); } -function enrichFailure(error: RpcError, effectState: TransferEffectState, phase: UploadPhase): RpcError { +function enrichFailure( + error: RpcError, + effectState: TransferEffectState, + phase: UploadPhase, +): RpcError { return { ...error, data: { ...error.data, effect_state: effectState, phase }, @@ -187,10 +194,15 @@ export async function uploadThroughActivatedFileInput( try { const resolved = await waitBounded( - sendToCdpTarget<{ object?: { objectId?: string } }>(options.cdp, target, "DOM.resolveNode", { - backendNodeId: options.actionTarget.backendNodeId, - objectGroup, - }), + sendToCdpTarget<{ object?: { objectId?: string } }>( + options.cdp, + target, + "DOM.resolveNode", + { + backendNodeId: options.actionTarget.backendNodeId, + objectGroup, + }, + ), deadline, options.signal, "resolving upload trigger timed out", @@ -309,10 +321,7 @@ export async function uploadThroughActivatedFileInput( let backendNodeId: number | undefined; let multiple = summary.multiple; if (chooser) { - if ( - options.actionTarget.frameId && - chooser.frameId !== options.actionTarget.frameId - ) { + if (options.actionTarget.frameId && chooser.frameId !== options.actionTarget.frameId) { outcome = transferFailure( "unsupported", "file_input_not_activated", diff --git a/apps/extension/src/tools/upload.ts b/apps/extension/src/tools/upload.ts index 58370e4..05a43ea 100644 --- a/apps/extension/src/tools/upload.ts +++ b/apps/extension/src/tools/upload.ts @@ -5,11 +5,7 @@ import type { SessionManager } from "@/session-manager/manager"; import type { RpcError, UploadParams, UploadResult } from "@/transport/types"; import { uploadThroughActivatedFileInput } from "./file-input-transaction"; -import { - clickResolvedTarget, - type InteractionDeps, - resolveActionTarget, -} from "./interaction"; +import { clickResolvedTarget, type InteractionDeps, resolveActionTarget } from "./interaction"; import { type CdpRunner, enforceAgentWindow, From c74c119a82e752a0119f0eef61a797a6a324fd57 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Mon, 24 Aug 2026 16:26:04 +0800 Subject: [PATCH 08/12] fix(bsk): download bug fix --- .../src/tools/__tests__/file-transfer.test.ts | 149 ++++++++++++++++++ apps/extension/src/tools/download-capture.ts | 37 +++-- 2 files changed, 175 insertions(+), 11 deletions(-) diff --git a/apps/extension/src/tools/__tests__/file-transfer.test.ts b/apps/extension/src/tools/__tests__/file-transfer.test.ts index 1e30091..f3f365c 100644 --- a/apps/extension/src/tools/__tests__/file-transfer.test.ts +++ b/apps/extension/src/tools/__tests__/file-transfer.test.ts @@ -493,6 +493,155 @@ describe("file transfer tools", () => { expect(result).toMatchObject({ item: { id: 21, state: "complete" } }); }); + it("removes a completed download when the final size exceeds the limit", async () => { + const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>(); + const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>(); + const onDeterminingFilename = + fakeEvent< + ( + item: chrome.downloads.DownloadItem, + suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void, + ) => void | true + >(); + const initial = { + id: 22, + url: "https://example.test/oversized.bin", + finalUrl: "https://example.test/oversized.bin", + filename: "oversized.bin", + state: "in_progress", + fileSize: -1, + totalBytes: -1, + bytesReceived: 0, + } as chrome.downloads.DownloadItem; + const complete = { + ...initial, + filename: "/profile/Downloads/BrowserSkill/tr_22/oversized.bin", + state: "complete", + fileSize: 8, + totalBytes: 8, + bytesReceived: 8, + } as chrome.downloads.DownloadItem; + const downloads: DownloadsApi = { + onCreated, + onChanged, + onDeterminingFilename, + search: vi.fn(async () => [complete]), + cancel: vi.fn(async () => {}), + removeFile: vi.fn(async () => {}), + }; + let cdpEvent: Parameters>[0] | undefined; + const cdp: CdpRunner = { + send: vi.fn(async () => ({})) as CdpRunner["send"], + onEvent: (handler) => { + cdpEvent = handler; + return { dispose: vi.fn() }; + }, + }; + + const result = await captureBrowserDownload({ + cdp, + target: { tabId: 4 }, + downloads, + browserRelativeDir: "BrowserSkill/tr_22", + maxByteSize: 4, + timeoutMs: 1_000, + trigger: async () => { + cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", { + url: initial.url, + suggestedFilename: initial.filename, + }); + await new Promise((resolve) => { + onDeterminingFilename.emit(initial, () => resolve()); + }); + onCreated.emit(initial); + onChanged.emit({ id: initial.id, state: { current: "complete" } }); + return { tab_id: 4, x: 10, y: 10 }; + }, + }); + + expect(result).toMatchObject({ + code: "cdp_failed", + data: { reason: "download_capture_failed", effect_state: "committed" }, + }); + expect(downloads.removeFile).toHaveBeenCalledWith(initial.id); + expect(downloads.cancel).not.toHaveBeenCalled(); + }); + + it("reconciles a download that completes while cancellation is being requested", async () => { + const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>(); + const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>(); + const onDeterminingFilename = + fakeEvent< + ( + item: chrome.downloads.DownloadItem, + suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void, + ) => void | true + >(); + const initial = { + id: 23, + url: "https://example.test/racing.bin", + finalUrl: "https://example.test/racing.bin", + filename: "racing.bin", + state: "in_progress", + fileSize: -1, + totalBytes: -1, + bytesReceived: 0, + } as chrome.downloads.DownloadItem; + const complete = { + ...initial, + filename: "/profile/Downloads/BrowserSkill/tr_23/racing.bin", + state: "complete", + fileSize: 4, + totalBytes: 4, + bytesReceived: 4, + } as chrome.downloads.DownloadItem; + const downloads: DownloadsApi = { + onCreated, + onChanged, + onDeterminingFilename, + search: vi.fn().mockResolvedValueOnce([initial]).mockResolvedValueOnce([complete]), + cancel: vi.fn(async () => { + throw new Error("download already complete"); + }), + removeFile: vi.fn(async () => {}), + }; + let cdpEvent: Parameters>[0] | undefined; + const cdp: CdpRunner = { + send: vi.fn(async () => ({})) as CdpRunner["send"], + onEvent: (handler) => { + cdpEvent = handler; + return { dispose: vi.fn() }; + }, + }; + + const result = await captureBrowserDownload({ + cdp, + target: { tabId: 4 }, + downloads, + browserRelativeDir: "BrowserSkill/tr_23", + timeoutMs: 80, + trigger: async () => { + cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", { + url: initial.url, + suggestedFilename: initial.filename, + }); + await new Promise((resolve) => { + onDeterminingFilename.emit(initial, () => resolve()); + }); + onCreated.emit(initial); + return { tab_id: 4, x: 10, y: 10 }; + }, + }); + + expect(result).toMatchObject({ + code: "cdp_failed", + data: { reason: "download_capture_failed", effect_state: "committed" }, + }); + expect(downloads.cancel).toHaveBeenCalledWith(initial.id); + expect(downloads.removeFile).toHaveBeenCalledWith(initial.id); + expect(result).not.toMatchObject({ data: { cleanup_state: "failed" } }); + }); + it("rejects ambiguous attribution without cancelling either unclaimed download", async () => { const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>(); const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>(); diff --git a/apps/extension/src/tools/download-capture.ts b/apps/extension/src/tools/download-capture.ts index 9a6ca9d..6e75bff 100644 --- a/apps/extension/src/tools/download-capture.ts +++ b/apps/extension/src/tools/download-capture.ts @@ -108,13 +108,37 @@ function captureError( }); } +async function cleanupClaimedDownload(downloads: DownloadsApi, downloadId: number): Promise { + const lookup = async () => (await downloads.search({ id: downloadId }))[0]; + const item = await lookup(); + if (!item || item.state === "interrupted") return; + if (item.state === "complete") { + await downloads.removeFile(downloadId); + return; + } + + try { + await downloads.cancel(downloadId); + } catch (cancelError) { + // Completion can win the race after the lookup but before cancellation. + // Reconcile against Chrome's authoritative state before declaring cleanup + // failed so every terminal state has one explicit cleanup path. + const reconciled = await lookup(); + if (!reconciled || reconciled.state === "interrupted") return; + if (reconciled.state === "complete") { + await downloads.removeFile(downloadId); + return; + } + throw cancelError; + } +} + export async function captureBrowserDownload( options: DownloadCaptureOptions, ): Promise { let click: ClickResult | undefined; let intent: DownloadIntent | undefined; let capturedId: number | undefined; - let capturedItem: chrome.downloads.DownloadItem | undefined; let settled = false; let succeeded = false; let failureResult: RpcError | undefined; @@ -143,7 +167,6 @@ export async function captureBrowserDownload( return; } settled = true; - capturedItem = item; resolveCompletion(item); }; const suggestDefault = (candidate: DownloadCandidate) => { @@ -175,7 +198,6 @@ export async function captureBrowserDownload( candidate.suggested = true; clearTimeout(candidate.graceTimer); capturedId = candidate.item.id; - capturedItem = candidate.item; candidate.suggest({ filename: `${options.browserRelativeDir}/${safeBasename(intent.suggestedFilename)}`, conflictAction: "overwrite", @@ -225,7 +247,6 @@ export async function captureBrowserDownload( const createdListener = (item: chrome.downloads.DownloadItem) => { createdItems.set(item.id, item); if (capturedId !== item.id) return; - capturedItem = item; if (item.state === "interrupted") { fail(new Error(item.error ?? "download interrupted")); } else if (item.state === "complete") { @@ -286,7 +307,6 @@ export async function captureBrowserDownload( .search({ id: capturedId }) .then(([item]) => { if (!item || settled) return; - capturedItem = item; if (item.bytesReceived > (options.maxByteSize as number)) { fail(new Error(`download exceeds transfer limit ${options.maxByteSize}`)); } @@ -334,13 +354,8 @@ export async function captureBrowserDownload( if (candidate.item.id !== capturedId) suggestDefault(candidate); } if (!succeeded && capturedId !== undefined) { - const item = capturedItem ?? createdItems.get(capturedId); try { - if (item?.state === "complete") { - await options.downloads.removeFile(capturedId); - } else { - await options.downloads.cancel(capturedId); - } + await cleanupClaimedDownload(options.downloads, capturedId); } catch { if (failureResult?.data) failureResult.data.cleanup_state = "failed"; } From 8822bba9a52f8c8ce94ed31d0a83ee4185b7ece7 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Fri, 21 Aug 2026 17:32:13 +0800 Subject: [PATCH 09/12] feat(bsk): support drop to upload --- .../extension/e2e/fixtures/file-transfer.html | 81 +++++ .../__tests__/capture-suppress-bridge.test.ts | 12 +- .../src/lib/capture-suppress-bridge.ts | 10 +- .../src/tools/__tests__/file-transfer.test.ts | 284 +++++++++++++++- .../tools/__tests__/frame-geometry.test.ts | 2 + .../src/tools/file-drop-transaction.ts | 319 ++++++++++++++++++ .../src/tools/file-input-transaction.ts | 41 +-- apps/extension/src/tools/frame-geometry.ts | 28 +- apps/extension/src/tools/observation.ts | 6 +- .../src/tools/transfer-transaction.ts | 42 +++ apps/extension/src/tools/upload.ts | 48 ++- apps/extension/src/transport/types.ts | 6 + crates/bsk-cli/skill/SKILL.md | 9 +- crates/bsk-cli/src/cli/mod.rs | 2 +- crates/bsk-cli/src/cli/render_error.rs | 50 +++ crates/bsk-cli/src/cli/upload.rs | 28 +- crates/bsk-cli/tests/cli_parse.rs | 49 +++ .../schema/tool_upload_params.json | 10 + .../bsk-protocol/src/tools/file_transfer.rs | 37 ++ docs/architecture.md | 2 +- skill/SKILL.md | 9 +- 21 files changed, 997 insertions(+), 78 deletions(-) create mode 100644 apps/extension/e2e/fixtures/file-transfer.html create mode 100644 apps/extension/src/tools/file-drop-transaction.ts create mode 100644 apps/extension/src/tools/transfer-transaction.ts diff --git a/apps/extension/e2e/fixtures/file-transfer.html b/apps/extension/e2e/fixtures/file-transfer.html new file mode 100644 index 0000000..47e64f8 --- /dev/null +++ b/apps/extension/e2e/fixtures/file-transfer.html @@ -0,0 +1,81 @@ + + + + + BrowserSkill file-transfer fixture + + + +

Upload then download the same image

+

No image selected

+
+ + + Download uploaded image +
+
Drop an image here
+ + + + diff --git a/apps/extension/src/lib/__tests__/capture-suppress-bridge.test.ts b/apps/extension/src/lib/__tests__/capture-suppress-bridge.test.ts index 1baf524..7e5fe11 100644 --- a/apps/extension/src/lib/__tests__/capture-suppress-bridge.test.ts +++ b/apps/extension/src/lib/__tests__/capture-suppress-bridge.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it, vi } from "vitest"; import { CAPTURE_SUPPRESS, type CaptureSuppressMessage, - withOverlaysHiddenForCapture, + withExtensionOverlayHidden, } from "../capture-suppress-bridge"; function recordingSendToTab(events: string[]) { @@ -12,12 +12,12 @@ function recordingSendToTab(events: string[]) { }); } -describe("withOverlaysHiddenForCapture", () => { +describe("withExtensionOverlayHidden", () => { it("sends begin → fn → end and returns fn's result", async () => { const events: string[] = []; const sendToTab = recordingSendToTab(events); - const result = await withOverlaysHiddenForCapture( + const result = await withExtensionOverlayHidden( 7, async () => { events.push("capture"); @@ -37,7 +37,7 @@ describe("withOverlaysHiddenForCapture", () => { const sendToTab = recordingSendToTab(events); await expect( - withOverlaysHiddenForCapture( + withExtensionOverlayHidden( 7, async () => { events.push("capture"); @@ -55,7 +55,7 @@ describe("withOverlaysHiddenForCapture", () => { }); const fn = vi.fn(async () => "shot"); - const result = await withOverlaysHiddenForCapture(7, fn, sendToTab); + const result = await withExtensionOverlayHidden(7, fn, sendToTab); expect(result).toBe("shot"); expect(fn).toHaveBeenCalledTimes(1); @@ -69,7 +69,7 @@ describe("withOverlaysHiddenForCapture", () => { return { type: CAPTURE_SUPPRESS, ok: true }; }); - const result = await withOverlaysHiddenForCapture(7, async () => "shot", sendToTab); + const result = await withExtensionOverlayHidden(7, async () => "shot", sendToTab); expect(result).toBe("shot"); expect(sendToTab).toHaveBeenCalledTimes(2); diff --git a/apps/extension/src/lib/capture-suppress-bridge.ts b/apps/extension/src/lib/capture-suppress-bridge.ts index 03e05b3..07096f9 100644 --- a/apps/extension/src/lib/capture-suppress-bridge.ts +++ b/apps/extension/src/lib/capture-suppress-bridge.ts @@ -1,9 +1,9 @@ /** - * Wire protocol and background-side helper for hiding the extension's - * in-page overlay while a screenshot is captured, so captured frames only - * contain the page itself. + * Wire protocol and background-side helper for temporarily excluding the + * extension's in-page overlay. Screenshots use it to avoid captured chrome; + * coordinate-driven native input uses it so Chrome hit-tests the page. * - * The background wraps every capture in `withOverlaysHiddenForCapture`, + * The background wraps each bounded operation in `withExtensionOverlayHidden`, * which sends `begin` (the content script hides the overlay host and only * acks once the compositor has produced an overlay-free frame) and always * follows up with `end`, even when the capture throws. Tabs without the @@ -46,7 +46,7 @@ const defaultSendToTab: CaptureSuppressSendToTab = (tabId, message) => * (or any other `begin` failure) simply skips suppression — there is no * overlay to hide in that tab. */ -export async function withOverlaysHiddenForCapture( +export async function withExtensionOverlayHidden( tabId: number, fn: () => Promise, sendToTab: CaptureSuppressSendToTab = defaultSendToTab, diff --git a/apps/extension/src/tools/__tests__/file-transfer.test.ts b/apps/extension/src/tools/__tests__/file-transfer.test.ts index f3f365c..3ef1986 100644 --- a/apps/extension/src/tools/__tests__/file-transfer.test.ts +++ b/apps/extension/src/tools/__tests__/file-transfer.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "vitest"; import { SessionManager } from "@/session-manager/manager"; import { type DownloadsApi, handleDownload } from "../download"; import { captureBrowserDownload } from "../download-capture"; +import { uploadThroughFileDrop } from "../file-drop-transaction"; import { uploadThroughActivatedFileInput } from "../file-input-transaction"; import type { ResolvedActionTarget } from "../interaction"; import type { CdpRunner } from "../shared"; @@ -37,16 +38,20 @@ function fakeEvent unknown>() { }; } -function actionTarget(frameId?: string): ResolvedActionTarget { +function actionTarget(frameId?: string, sessionId?: string): ResolvedActionTarget { return { tab: { tabId: 4, windowId: 100, active: true }, backendNodeId: 123, - cdpTarget: { tabId: 4 }, + cdpTarget: { tabId: 4, ...(sessionId ? { sessionId } : {}) }, ...(frameId ? { frameId } : {}), usedRef: "e3", }; } +function withoutOverlay(operation: () => Promise): Promise { + return operation(); +} + function uploadCdp( options: { inputCount?: number; @@ -59,6 +64,7 @@ function uploadCdp( let cdpEvent: Parameters>[0] | undefined; const send = vi.fn(async (_tabId: number, method: string, params?: object) => { calls.push({ method, params }); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; if (method === "Page.setInterceptFileChooserDialog") return {}; if (method === "Page.getLayoutMetrics") return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; @@ -110,6 +116,50 @@ function uploadCdp( }; } +function dropCdp( + options: { + targetContainsPoint?: boolean; + dispatchError?: (type: string) => Error | Promise | undefined; + onCall?: (method: string) => void; + } = {}, +) { + const calls: Array<{ method: string; params?: object }> = []; + const detach = vi.fn(async () => {}); + const send = vi.fn(async (_tabId: number, method: string, params?: object) => { + calls.push({ method, params }); + options.onCall?.(method); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; + } + if (method === "DOM.getContentQuads") { + return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] }; + } + if (method === "DOM.getNodeForLocation") return { backendNodeId: 124 }; + if (method === "DOM.resolveNode") { + const id = (params as { backendNodeId?: number }).backendNodeId; + return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } }; + } + if (method === "Runtime.callFunctionOn") { + return { result: { value: options.targetContainsPoint ?? true } }; + } + if (method === "Input.dispatchDragEvent") { + const type = (params as { type?: string }).type ?? ""; + const error = options.dispatchError?.(type); + if (error instanceof Promise) return error; + if (error) throw error; + return {}; + } + if (method === "Runtime.releaseObjectGroup") return {}; + throw new Error(`unexpected drop CDP command ${method}`); + }); + const cdp: CdpRunner = { + send: send as unknown as CdpRunner["send"], + detach, + }; + return { calls, cdp, detach }; +} + describe("file transfer tools", () => { it("captures the file input activated by the requested click and injects only staged paths", async () => { const manager = sessions(); @@ -271,6 +321,236 @@ describe("file transfer tools", () => { expect(detach).toHaveBeenCalledWith(4); }); + it("delivers staged files through one explicit native drop without clicking", async () => { + const manager = sessions(); + const ctx = await manager.start("s1"); + ctx.refStore.set("e3", 123, { tabId: 4 }); + const lifecycle: string[] = []; + const { cdp, calls } = dropCdp({ onCall: (method) => lifecycle.push(method) }); + const overlayPhases: string[] = []; + + const result = await handleUpload( + manager, + { + session_id: "s1", + ref: "@e3", + mode: "drop", + files: [{ transfer_id: "tr_1", name: "one.png", staged_path: "/stage/one.png" }], + }, + { + cdp, + tabsApi: tabsApi(), + sendToTab: vi.fn(async (_tabId, message) => { + overlayPhases.push(message.phase); + lifecycle.push(`overlay:${message.phase}`); + }), + }, + ); + + expect(result).toMatchObject({ tab_id: 4, used_ref: "e3", file_names: ["one.png"] }); + expect(calls.filter((call) => call.method === "Input.dispatchDragEvent")).toEqual([ + { + method: "Input.dispatchDragEvent", + params: { + type: "dragEnter", + x: 110, + y: 70, + data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 }, + }, + }, + { + method: "Input.dispatchDragEvent", + params: { + type: "dragOver", + x: 110, + y: 70, + data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 }, + }, + }, + { + method: "Input.dispatchDragEvent", + params: { + type: "drop", + x: 110, + y: 70, + data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 }, + }, + }, + ]); + expect(calls.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false); + expect(overlayPhases).toEqual(["begin", "end"]); + expect(lifecycle.indexOf("DOM.getContentQuads")).toBeLessThan( + lifecycle.indexOf("overlay:begin"), + ); + expect(lifecycle.indexOf("overlay:begin")).toBeLessThan( + lifecycle.indexOf("DOM.getNodeForLocation"), + ); + expect(lifecycle.lastIndexOf("Input.dispatchDragEvent")).toBeLessThan( + lifecycle.indexOf("overlay:end"), + ); + expect(lifecycle.indexOf("overlay:end")).toBeLessThan( + lifecycle.indexOf("Runtime.releaseObjectGroup"), + ); + }); + + it("fails closed before dragging when another element owns the action point", async () => { + const { cdp, calls } = dropCdp({ targetContainsPoint: false }); + const result = await uploadThroughFileDrop({ + cdp, + actionTarget: actionTarget(), + files: ["/stage/one.png"], + timeoutMs: 100, + withOverlayHidden: withoutOverlay, + }); + + expect(result).toMatchObject({ + code: "permission_denied", + data: { + reason: "file_drop_target_unavailable", + mechanism: "drop", + effect_state: "none", + phase: "resolve_target", + }, + }); + expect(calls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false); + }); + + it("dispatches an OOPIF file drop in the child target's local coordinates", async () => { + const rootCalls: Array<{ method: string; params?: object }> = []; + const childCalls: Array<{ method: string; params?: object }> = []; + const cdp: CdpRunner = { + send: vi.fn(async (_tabId, method, params) => { + rootCalls.push({ method, params }); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "DOM.getBoxModel") { + return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } }; + } + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } }; + } + throw new Error(`unexpected root command ${method}`); + }) as CdpRunner["send"], + sendToTarget: vi.fn(async (_target, method, params) => { + childCalls.push({ method, params }); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "DOM.getContentQuads") { + return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] }; + } + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } }; + } + if (method === "DOM.getNodeForLocation") { + return { backendNodeId: 124, frameId: "child" }; + } + if (method === "DOM.resolveNode") { + const id = (params as { backendNodeId?: number }).backendNodeId; + return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } }; + } + if (method === "Runtime.callFunctionOn") return { result: { value: true } }; + if (method === "Input.dispatchDragEvent" || method === "Runtime.releaseObjectGroup") { + return {}; + } + throw new Error(`unexpected child command ${method}`); + }) as CdpRunner["sendToTarget"], + getFrameGraph: vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 10, + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })), + }; + + const result = await uploadThroughFileDrop({ + cdp, + actionTarget: actionTarget("child", "child-session"), + files: ["/stage/one.png"], + timeoutMs: 100, + withOverlayHidden: withoutOverlay, + }); + + expect(result).toMatchObject({ x: 110, y: 70 }); + expect( + childCalls + .filter((call) => call.method === "Input.dispatchDragEvent") + .map((call) => call.params), + ).toEqual([ + expect.objectContaining({ type: "dragEnter", x: 110, y: 70 }), + expect.objectContaining({ type: "dragOver", x: 110, y: 70 }), + expect.objectContaining({ type: "drop", x: 110, y: 70 }), + ]); + expect(rootCalls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false); + }); + + it("reports a missing native drag command as unsupported before commit", async () => { + const { cdp } = dropCdp({ + dispatchError: (type) => + type === "dragEnter" ? new Error("'Input.dispatchDragEvent' wasn't found") : undefined, + }); + const result = await uploadThroughFileDrop({ + cdp, + actionTarget: actionTarget(), + files: ["/stage/one.png"], + timeoutMs: 100, + withOverlayHidden: withoutOverlay, + }); + + expect(result).toMatchObject({ + code: "unsupported", + data: { + reason: "upload_mechanism_unsupported", + effect_state: "none", + phase: "drag_enter", + }, + }); + }); + + it("cancels an unresolved drag before returning a pre-commit timeout", async () => { + const { cdp, calls } = dropCdp({ + dispatchError: (type) => (type === "dragEnter" ? new Promise(() => {}) : undefined), + }); + const result = await uploadThroughFileDrop({ + cdp, + actionTarget: actionTarget(), + files: ["/stage/one.png"], + timeoutMs: 10, + withOverlayHidden: withoutOverlay, + }); + + expect(result).toMatchObject({ + code: "timeout", + data: { reason: "file_drop_failed", effect_state: "none", phase: "drag_enter" }, + }); + expect(calls).toContainEqual({ + method: "Input.dispatchDragEvent", + params: expect.objectContaining({ type: "dragCancel" }), + }); + }); + + it("marks an unresolved drop unknown and detaches browser state", async () => { + const { cdp, detach } = dropCdp({ + dispatchError: (type) => (type === "drop" ? new Promise(() => {}) : undefined), + }); + const result = await uploadThroughFileDrop({ + cdp, + actionTarget: actionTarget(), + files: ["/stage/one.png"], + timeoutMs: 10, + withOverlayHidden: withoutOverlay, + }); + + expect(result).toMatchObject({ + code: "timeout", + data: { reason: "file_drop_failed", effect_state: "unknown", phase: "drop" }, + }); + expect(detach).toHaveBeenCalledWith(4); + }); + it("routes one exact-target download through a browser-relative capability", async () => { const manager = sessions(); const ctx = await manager.start("s1"); diff --git a/apps/extension/src/tools/__tests__/frame-geometry.test.ts b/apps/extension/src/tools/__tests__/frame-geometry.test.ts index c7f82b4..9116f6a 100644 --- a/apps/extension/src/tools/__tests__/frame-geometry.test.ts +++ b/apps/extension/src/tools/__tests__/frame-geometry.test.ts @@ -169,6 +169,7 @@ describe("frame geometry projection", () => { expect(geometry).toMatchObject({ topBounds: { x: 100, y: 100, width: 200, height: 100 }, actionPoint: { x: 200, y: 150 }, + targetActionPoint: { x: 100, y: 50 }, }); }); @@ -261,6 +262,7 @@ describe("frame geometry projection", () => { expect(geometry).toMatchObject({ topBounds: { x: 100, y: 100, width: 200, height: 100 }, actionPoint: { x: 200, y: 150 }, + targetActionPoint: { x: 200, y: 150 }, }); }); diff --git a/apps/extension/src/tools/file-drop-transaction.ts b/apps/extension/src/tools/file-drop-transaction.ts new file mode 100644 index 0000000..f806e84 --- /dev/null +++ b/apps/extension/src/tools/file-drop-transaction.ts @@ -0,0 +1,319 @@ +// One native file-drop transaction bound to an explicit page target. CDP drag +// events are sent to the target that owns the node: root targets use top-level +// coordinates, while OOPIF targets require coordinates local to their session. + +import type { RpcError, TransferEffectState } from "@/transport/types"; +import { transferError } from "./errors"; +import { resolveNodeGeometry } from "./frame-geometry"; +import type { ResolvedActionTarget } from "./interaction"; +import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared"; +import { BoundedWaitError, waitBounded } from "./transfer-transaction"; + +const CLEANUP_TIMEOUT_MS = 1_000; + +type FileDropPhase = "resolve_target" | "drag_enter" | "drag_over" | "drop" | "cleanup"; + +interface RuntimeReply { + result?: { value?: unknown; objectId?: string }; + exceptionDetails?: { text?: string; exception?: { description?: string } }; +} + +interface DragData { + items: []; + files: string[]; + dragOperationsMask: number; +} + +export interface FileDropTransactionOptions { + cdp: CdpRunner; + actionTarget: ResolvedActionTarget; + files: string[]; + timeoutMs: number; + signal?: AbortSignal; + withOverlayHidden: (operation: () => Promise) => Promise; +} + +export interface FileDropTransactionResult { + x: number; + y: number; +} + +function failure( + code: RpcError["code"], + reason: "upload_mechanism_unsupported" | "file_drop_target_unavailable" | "file_drop_failed", + message: string, + effectState: TransferEffectState, + phase: FileDropPhase, +): RpcError { + const error = transferError(code, reason, message, { effectState, phase }); + error.data = { ...error.data, mechanism: "drop" }; + return error; +} + +function enrichFailure( + error: RpcError, + effectState: TransferEffectState, + phase: FileDropPhase, +): RpcError { + return { + ...error, + data: { + ...error.data, + mechanism: "drop", + effect_state: effectState, + phase, + }, + }; +} + +function messageOf(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function methodUnavailable(error: unknown): boolean { + return /(?:method|command).*(?:not found|wasn't found|unsupported)|wasn't found|-32601/i.test( + messageOf(error), + ); +} + +function runtimeError(reply: RuntimeReply, fallback: string): Error | null { + if (!reply.exceptionDetails) return null; + return new Error( + reply.exceptionDetails.exception?.description ?? reply.exceptionDetails.text ?? fallback, + ); +} + +async function pointBelongsToTarget( + options: FileDropTransactionOptions, + x: number, + y: number, + objectGroup: string, +): Promise { + const target = options.actionTarget.cdpTarget; + const hit = await sendToCdpTarget<{ backendNodeId?: number; frameId?: string }>( + options.cdp, + target, + "DOM.getNodeForLocation", + { x: Math.round(x), y: Math.round(y), includeUserAgentShadowDOM: true }, + ); + if (typeof hit.backendNodeId !== "number") return false; + if (options.actionTarget.frameId && hit.frameId !== options.actionTarget.frameId) return false; + + const [resolvedTarget, resolvedHit] = await Promise.all([ + sendToCdpTarget<{ object?: { objectId?: string } }>(options.cdp, target, "DOM.resolveNode", { + backendNodeId: options.actionTarget.backendNodeId, + objectGroup, + }), + sendToCdpTarget<{ object?: { objectId?: string } }>(options.cdp, target, "DOM.resolveNode", { + backendNodeId: hit.backendNodeId, + objectGroup, + }), + ]); + const targetObjectId = resolvedTarget.object?.objectId; + const hitObjectId = resolvedHit.object?.objectId; + if (!targetObjectId || !hitObjectId) return false; + + const reply = await sendToCdpTarget(options.cdp, target, "Runtime.callFunctionOn", { + objectId: targetObjectId, + functionDeclaration: `function(hit) { + for (let node = hit; node; ) { + if (node === this) return true; + if (this.contains?.(node)) return true; + const root = node.getRootNode?.(); + node = node.parentNode || (root && root !== node ? root.host : null); + } + return false; + }`, + arguments: [{ objectId: hitObjectId }], + returnByValue: true, + }); + const error = runtimeError(reply, "failed to verify file-drop target"); + if (error) throw error; + return reply.result?.value === true; +} + +export async function uploadThroughFileDrop( + options: FileDropTransactionOptions, +): Promise { + const deadline = Date.now() + options.timeoutMs; + const target = options.actionTarget.cdpTarget; + const objectGroup = `bsk-file-drop-${crypto.randomUUID()}`; + const dragData: DragData = { + items: [], + files: options.files, + dragOperationsMask: 1, + }; + let dragStarted = false; + let dropSent = false; + let dragPoint = { x: 0, y: 0 }; + let cleanupFailed = false; + let outcome: FileDropTransactionResult | RpcError = failure( + "protocol_error", + "file_drop_failed", + "file-drop transaction ended without an outcome", + "none", + "cleanup", + ); + + try { + const geometry = await waitBounded( + resolveNodeGeometry( + options.cdp, + options.actionTarget.tab.tabId, + { + target, + backendNodeId: options.actionTarget.backendNodeId, + ...(options.actionTarget.frameId ? { frameId: options.actionTarget.frameId } : {}), + }, + { scrollIntoView: true }, + ), + deadline, + options.signal, + "resolving file-drop target timed out", + ); + if (isRpcError(geometry)) { + outcome = enrichFailure(geometry, "none", "resolve_target"); + return outcome; + } + const point = geometry.targetActionPoint; + dragPoint = point; + + outcome = await options.withOverlayHidden(async () => { + try { + try { + const belongs = await waitBounded( + pointBelongsToTarget(options, point.x, point.y, objectGroup), + deadline, + options.signal, + "verifying file-drop target timed out", + ); + if (!belongs) { + return failure( + "permission_denied", + "file_drop_target_unavailable", + "file-drop target is not the topmost element at its action point", + "none", + "resolve_target", + ); + } + } catch (error) { + return failure( + error instanceof BoundedWaitError ? "timeout" : "cdp_failed", + "file_drop_target_unavailable", + messageOf(error), + "none", + "resolve_target", + ); + } + + for (const [type, phase] of [ + ["dragEnter", "drag_enter"], + ["dragOver", "drag_over"], + ] as const) { + try { + if (type === "dragEnter") dragStarted = true; + await waitBounded( + sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", { + type, + x: point.x, + y: point.y, + data: dragData, + }), + deadline, + options.signal, + `${type} timed out`, + ); + } catch (error) { + const unsupported = methodUnavailable(error); + if (type === "dragEnter" && unsupported) dragStarted = false; + return failure( + unsupported + ? "unsupported" + : error instanceof BoundedWaitError + ? "timeout" + : "cdp_failed", + unsupported ? "upload_mechanism_unsupported" : "file_drop_failed", + messageOf(error), + "none", + phase, + ); + } + } + + try { + dropSent = true; + await waitBounded( + sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", { + type: "drop", + x: point.x, + y: point.y, + data: dragData, + }), + deadline, + options.signal, + "file drop timed out", + ); + return { x: point.x, y: point.y }; + } catch (error) { + outcome = failure( + error instanceof BoundedWaitError ? "timeout" : "cdp_failed", + "file_drop_failed", + messageOf(error), + "unknown", + "drop", + ); + return outcome; + } + } finally { + if (dragStarted && !dropSent) { + try { + await waitBounded( + sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", { + type: "dragCancel", + x: dragPoint.x, + y: dragPoint.y, + data: dragData, + }), + Date.now() + CLEANUP_TIMEOUT_MS, + undefined, + "cancelling file drag timed out", + ); + } catch { + cleanupFailed = true; + } + } + } + }); + return outcome; + } catch (error) { + outcome = failure( + error instanceof BoundedWaitError ? "timeout" : "cdp_failed", + "file_drop_target_unavailable", + messageOf(error), + "none", + "resolve_target", + ); + return outcome; + } finally { + try { + await waitBounded( + sendToCdpTarget(options.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }), + Date.now() + CLEANUP_TIMEOUT_MS, + undefined, + "releasing file-drop object group timed out", + ); + } catch { + cleanupFailed = true; + } + + const effect = isRpcError(outcome) + ? (outcome.data?.effect_state as TransferEffectState | undefined) + : "committed"; + if (effect === "unknown" || cleanupFailed) { + await options.cdp.detach?.(target.tabId); + } + if (cleanupFailed && isRpcError(outcome)) { + outcome.data = { ...outcome.data, cleanup_state: "failed" }; + } + } +} diff --git a/apps/extension/src/tools/file-input-transaction.ts b/apps/extension/src/tools/file-input-transaction.ts index 3c78a8d..c37dce2 100644 --- a/apps/extension/src/tools/file-input-transaction.ts +++ b/apps/extension/src/tools/file-input-transaction.ts @@ -7,6 +7,7 @@ import type { ClickResult, RpcError, TransferEffectState } from "@/transport/typ import { transferError } from "./errors"; import type { ResolvedActionTarget } from "./interaction"; import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared"; +import { BoundedWaitError, remainingMs, waitBounded } from "./transfer-transaction"; const CLEANUP_TIMEOUT_MS = 1_000; const ACTIVATION_GRACE_MS = 1_000; @@ -45,46 +46,6 @@ export interface FileInputTransactionResult { multiple: boolean; } -class BoundedWaitError extends Error { - constructor( - readonly kind: "timeout" | "aborted", - message: string, - ) { - super(message); - } -} - -function remainingMs(deadline: number): number { - return Math.max(0, deadline - Date.now()); -} - -async function waitBounded( - promise: Promise, - deadline: number, - signal: AbortSignal | undefined, - timeoutMessage: string, -): Promise { - const remaining = remainingMs(deadline); - if (signal?.aborted) throw new BoundedWaitError("aborted", "upload transaction aborted"); - if (remaining === 0) throw new BoundedWaitError("timeout", timeoutMessage); - - let timer: ReturnType | undefined; - let onAbort: (() => void) | undefined; - const boundary = new Promise((_resolve, reject) => { - timer = setTimeout(() => reject(new BoundedWaitError("timeout", timeoutMessage)), remaining); - if (signal) { - onAbort = () => reject(new BoundedWaitError("aborted", "upload transaction aborted")); - signal.addEventListener("abort", onAbort, { once: true }); - } - }); - try { - return await Promise.race([promise, boundary]); - } finally { - if (timer) clearTimeout(timer); - if (signal && onAbort) signal.removeEventListener("abort", onAbort); - } -} - function runtimeError(reply: RuntimeReply, fallback: string): Error | null { if (!reply.exceptionDetails) return null; return new Error( diff --git a/apps/extension/src/tools/frame-geometry.ts b/apps/extension/src/tools/frame-geometry.ts index ba2be42..0c788c9 100644 --- a/apps/extension/src/tools/frame-geometry.ts +++ b/apps/extension/src/tools/frame-geometry.ts @@ -29,7 +29,10 @@ export interface NodeAddress { export interface ResolvedNodeGeometry { topVisibleRegions: Region; topBounds: ViewportRect; + /** Point in the top-level tab viewport, used by root-target input events. */ actionPoint: Point; + /** Point in the addressed CDP target's viewport, used by OOPIF-local input events. */ + targetActionPoint: Point; } function geometryError(message: string): RpcError { @@ -262,11 +265,30 @@ export async function resolveNodeGeometry( if (isRpcError(localRegion)) return localRegion; let topVisibleRegions: Region; + let targetActionPoint: Point | null = null; if (address.frameId && graph) { const projection = await resolveFrameProjection(cdp, graph, address.frameId); if (!projection) return geometryError(`could not resolve frame geometry for ${address.frameId}`); topVisibleRegions = projectRegionToViewport(localRegion, projection); + if (address.target.sessionId) { + const localViewport = projection.edges[0]?.sourceViewport ?? projection.topViewport; + const localVisibleRegions = localRegion + .map((polygon) => + clipPolygon( + polygon, + rectPolygon({ + x: 0, + y: 0, + w: localViewport.width, + h: localViewport.height, + }), + ), + ) + .filter((polygon) => polygon.length >= 3); + const localActionRegion = largestRegion(localVisibleRegions); + targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null; + } } else { const viewport = await targetViewport(cdp, address.target); if (!viewport) return geometryError("could not resolve top viewport geometry"); @@ -283,7 +305,11 @@ export async function resolveNodeGeometry( if (!topBounds || !actionPoint) { return { code: "permission_denied", message: "element not visible" }; } - return { topVisibleRegions, topBounds, actionPoint }; + if (address.target.sessionId && !targetActionPoint) { + return { code: "permission_denied", message: "element not visible in its target" }; + } + targetActionPoint ??= actionPoint; + return { topVisibleRegions, topBounds, actionPoint, targetActionPoint }; } catch (error) { return geometryError(error instanceof Error ? error.message : String(error)); } diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index f25d97b..98b6cb0 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -17,7 +17,7 @@ import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; import type { CdpTarget } from "@/browser-driver/frame-graph"; import { type CaptureSuppressSendToTab, - withOverlaysHiddenForCapture, + withExtensionOverlayHidden, } from "@/lib/capture-suppress-bridge"; import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { @@ -312,7 +312,7 @@ export async function handleScreenshot( tabId: target.tabId, ...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}), }; - const captured = await withOverlaysHiddenForCapture( + const captured = await withExtensionOverlayHidden( target.tabId, () => captureElementScreenshot( @@ -344,7 +344,7 @@ export async function handleScreenshot( ); } - const captured = await withOverlaysHiddenForCapture( + const captured = await withExtensionOverlayHidden( target.tabId, () => captureFullTabPng(deps, ctx, target, signal), deps.sendToTab, diff --git a/apps/extension/src/tools/transfer-transaction.ts b/apps/extension/src/tools/transfer-transaction.ts new file mode 100644 index 0000000..6debaeb --- /dev/null +++ b/apps/extension/src/tools/transfer-transaction.ts @@ -0,0 +1,42 @@ +// Shared time boundary for browser-side file-transfer transactions. Concrete +// upload/download mechanisms own their state machines and effect semantics. + +export class BoundedWaitError extends Error { + constructor( + readonly kind: "timeout" | "aborted", + message: string, + ) { + super(message); + } +} + +export function remainingMs(deadline: number): number { + return Math.max(0, deadline - Date.now()); +} + +export async function waitBounded( + promise: Promise, + deadline: number, + signal: AbortSignal | undefined, + timeoutMessage: string, +): Promise { + const remaining = remainingMs(deadline); + if (signal?.aborted) throw new BoundedWaitError("aborted", "file transfer aborted"); + if (remaining === 0) throw new BoundedWaitError("timeout", timeoutMessage); + + let timer: ReturnType | undefined; + let onAbort: (() => void) | undefined; + const boundary = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new BoundedWaitError("timeout", timeoutMessage)), remaining); + if (signal) { + onAbort = () => reject(new BoundedWaitError("aborted", "file transfer aborted")); + signal.addEventListener("abort", onAbort, { once: true }); + } + }); + try { + return await Promise.race([promise, boundary]); + } finally { + if (timer) clearTimeout(timer); + if (signal && onAbort) signal.removeEventListener("abort", onAbort); + } +} diff --git a/apps/extension/src/tools/upload.ts b/apps/extension/src/tools/upload.ts index 05a43ea..9203dd2 100644 --- a/apps/extension/src/tools/upload.ts +++ b/apps/extension/src/tools/upload.ts @@ -2,8 +2,13 @@ // target, then delegate the browser protocol transaction to the file-input // transaction module. +import { + type CaptureSuppressSendToTab, + withExtensionOverlayHidden, +} from "@/lib/capture-suppress-bridge"; import type { SessionManager } from "@/session-manager/manager"; import type { RpcError, UploadParams, UploadResult } from "@/transport/types"; +import { uploadThroughFileDrop } from "./file-drop-transaction"; import { uploadThroughActivatedFileInput } from "./file-input-transaction"; import { clickResolvedTarget, type InteractionDeps, resolveActionTarget } from "./interaction"; import { @@ -18,6 +23,7 @@ const DEFAULT_TIMEOUT_MS = 120_000; export interface UploadDeps extends InteractionDeps { cdp: CdpRunner; + sendToTab?: CaptureSuppressSendToTab; } export async function handleUpload( @@ -41,19 +47,39 @@ export async function handleUpload( const address = await resolveActionTarget(deps.cdp, ctx, target, params, "upload"); if (isRpcError(address)) return address; const timeoutMs = params.timeout_ms ?? DEFAULT_TIMEOUT_MS; - const transaction = await uploadThroughActivatedFileInput({ - cdp: deps.cdp, - actionTarget: address, - files: params.files.map((file) => file.staged_path as string), - timeoutMs, - signal: deps.signal, - trigger: () => clickResolvedTarget(ctx, address, {}, deps), - }); - if (isRpcError(transaction)) return transaction; + const files = params.files.map((file) => file.staged_path as string); + const mode = params.mode ?? "input"; + let usedRef = address.usedRef; + let usedSelector = address.usedSelector; + if (mode === "drop") { + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const transaction = await uploadThroughFileDrop({ + cdp: deps.cdp, + actionTarget: address, + files, + timeoutMs, + signal: deps.signal, + withOverlayHidden: (operation) => + withExtensionOverlayHidden(target.tabId, operation, deps.sendToTab), + }); + if (isRpcError(transaction)) return transaction; + } else { + const transaction = await uploadThroughActivatedFileInput({ + cdp: deps.cdp, + actionTarget: address, + files, + timeoutMs, + signal: deps.signal, + trigger: () => clickResolvedTarget(ctx, address, {}, deps), + }); + if (isRpcError(transaction)) return transaction; + usedRef = transaction.click.used_ref; + usedSelector = transaction.click.used_selector; + } return { tab_id: target.tabId, - used_ref: transaction.click.used_ref, - used_selector: transaction.click.used_selector, + used_ref: usedRef, + used_selector: usedSelector, file_names: params.files.map((file) => file.name), }; } diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index 7143cb1..f2a01ca 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -36,6 +36,9 @@ export type RpcErrorReason = | "file_input_probe_failed" | "file_input_not_activated" | "set_file_input_failed" + | "upload_mechanism_unsupported" + | "file_drop_target_unavailable" + | "file_drop_failed" | "download_capture_failed" | "transfer_outcome_unknown" | "transfer_timeout" @@ -518,12 +521,15 @@ export interface UploadFile { staged_path?: string; } +export type UploadMode = "input" | "drop"; + export interface UploadParams { session_id: string; ref?: string; selector?: string; tab_id?: number; files: UploadFile[]; + mode?: UploadMode; timeout_ms?: number; } diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index 8af5383..bb8dce3 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -210,18 +210,23 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff | Command | Summary | |---------|---------| -| `bsk upload --file ` | Click one upload trigger and attach an agent-readable local file (`--file` is repeatable) | +| `bsk upload --file ` | Attach a local file through a file-input trigger; use `--mode drop` only for an explicit drop zone (`--file` is repeatable) | | `bsk download --out ` | Click one download trigger and copy the single completed file to an exact local path (`--overwrite` is opt-in) | The agent/harness decides whether a file transfer is appropriate and which local path belongs to the task. Treat upload as disclosure of that file to the current website, and download as accepting website-controlled bytes onto the local filesystem. Use only paths that are necessary for the user's bounded goal. -BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted capabilities reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. Upload intercepts the native chooser for one transaction, locates the input activated in the resolved target's document, and assigns only the staged file paths. Download uniquely correlates one exact-target browser intent with one Chrome download in either event order, routes it through a daemon-minted relative directory, and lets the daemon validate and import it. Upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive. +BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted capabilities reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. The default upload mode intercepts one chooser transaction, locates the file input activated in the resolved target's document, and assigns only the staged paths. `--mode drop` does not click: it delivers the same staged files through one native browser drop bound to the explicitly resolved drop zone. These mechanisms never fall back to each other automatically. Download uniquely correlates one exact-target browser intent with one Chrome download in either event order, routes it through a daemon-minted relative directory, and lets the daemon validate and import it. Upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive. + +A successful drop command means Chrome dispatched the native file drop to the resolved zone; it does not prove that the site's application logic accepted the attachment. Observe the page once after the command, and do not repeat the drop when the attachment is already present. Do not use `request-help` merely because a native file chooser or browser download is involved; try these commands first. For transfer failures, use the structured error instead of retrying blindly: - `reason=file_input_not_activated` means the requested click did not activate exactly one ``; the page may use a non-input picker such as `window.showOpenFilePicker()`. Do not retry blindly. If human help is available, call `request-help` and tell the user the exact original local path to choose. The staged daemon path is internal and must not be shown to the user. - `reason=file_input_probe_failed` means BrowserSkill could not safely establish the browser-side upload transaction. Do not repeat the same action; use `request-help` when available. - `reason=set_file_input_failed` means BrowserSkill found the activated file input but Chrome rejected the staged path or assignment. Check the extension's file-URL access permission; otherwise use `request-help`. +- `reason=upload_mechanism_unsupported` means the browser cannot perform a native file drop. Do not retry the same drop; use a standard file-input trigger or `request-help`. +- `reason=file_drop_target_unavailable` means the specified point is no longer owned by the resolved drop zone. When `effect_state=none`, re-observe and select the visible drop zone itself; do not guess another target. +- `reason=file_drop_failed` means the native drop command failed. Do not retry when `effect_state=unknown`; first observe whether the page already received the file. - `reason=download_capture_failed` means BrowserSkill could not attribute exactly one completed download to the requested target. Do not retry blindly or accept an unrelated browser download; use `request-help` when available. - `effect_state=none` means BrowserSkill confirmed that no file-transfer effect was committed. Follow the accompanying reason; a corrected target or explicit human fallback may be attempted. - `effect_state=unknown` means the browser may already have attached or created the file. Do not repeat the transfer. Observe the page if that can establish the result; otherwise stop and report the uncertainty. diff --git a/crates/bsk-cli/src/cli/mod.rs b/crates/bsk-cli/src/cli/mod.rs index 4776ebe..8140045 100644 --- a/crates/bsk-cli/src/cli/mod.rs +++ b/crates/bsk-cli/src/cli/mod.rs @@ -182,7 +182,7 @@ pub enum Command { /// Set ``; the page may use a non-input picker such as `window.showOpenFilePicker()`. Do not retry blindly. If human help is available, call `request-help` and tell the user the exact original local path to choose. The staged daemon path is internal and must not be shown to the user. - `reason=file_input_probe_failed` means BrowserSkill could not safely establish the browser-side upload transaction. Do not repeat the same action; use `request-help` when available. - `reason=set_file_input_failed` means BrowserSkill found the activated file input but Chrome rejected the staged path or assignment. Check the extension's file-URL access permission; otherwise use `request-help`. +- `reason=upload_mechanism_unsupported` means the browser cannot perform a native file drop. Do not retry the same drop; use a standard file-input trigger or `request-help`. +- `reason=file_drop_target_unavailable` means the specified point is no longer owned by the resolved drop zone. When `effect_state=none`, re-observe and select the visible drop zone itself; do not guess another target. +- `reason=file_drop_failed` means the native drop command failed. Do not retry when `effect_state=unknown`; first observe whether the page already received the file. - `reason=download_capture_failed` means BrowserSkill could not attribute exactly one completed download to the requested target. Do not retry blindly or accept an unrelated browser download; use `request-help` when available. - `effect_state=none` means BrowserSkill confirmed that no file-transfer effect was committed. Follow the accompanying reason; a corrected target or explicit human fallback may be attempted. - `effect_state=unknown` means the browser may already have attached or created the file. Do not repeat the transfer. Observe the page if that can establish the result; otherwise stop and report the uncertainty. From b1b1ae1d48e507783e1da42fe0b45fb3f70be8fd Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Fri, 21 Aug 2026 17:38:57 +0800 Subject: [PATCH 10/12] feat(bsk): update skill to support drop --- crates/bsk-cli/skill/SKILL.md | 15 +++++++++++---- crates/bsk-cli/src/cli/render_error.rs | 9 ++++++--- skill/SKILL.md | 15 +++++++++++---- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index bb8dce3..8fdf138 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -210,22 +210,29 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff | Command | Summary | |---------|---------| -| `bsk upload --file ` | Attach a local file through a file-input trigger; use `--mode drop` only for an explicit drop zone (`--file` is repeatable) | +| `bsk upload --file ` | Attach a local file through a file-input trigger; use `--mode drop` for a reliably identified attachment-receiving area (`--file` is repeatable) | | `bsk download --out ` | Click one download trigger and copy the single completed file to an exact local path (`--overwrite` is opt-in) | The agent/harness decides whether a file transfer is appropriate and which local path belongs to the task. Treat upload as disclosure of that file to the current website, and download as accepting website-controlled bytes onto the local filesystem. Use only paths that are necessary for the user's bounded goal. -BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted capabilities reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. The default upload mode intercepts one chooser transaction, locates the file input activated in the resolved target's document, and assigns only the staged paths. `--mode drop` does not click: it delivers the same staged files through one native browser drop bound to the explicitly resolved drop zone. These mechanisms never fall back to each other automatically. Download uniquely correlates one exact-target browser intent with one Chrome download in either event order, routes it through a daemon-minted relative directory, and lets the daemon validate and import it. Upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive. +BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted capabilities reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. The default upload mode intercepts one chooser transaction, locates the file input activated in the resolved target's document, and assigns only the staged paths. `--mode drop` does not click: it delivers the same staged files through one native browser drop bound to the explicitly resolved attachment target. These mechanisms never fall back to each other automatically. Download uniquely correlates one exact-target browser intent with one Chrome download in either event order, routes it through a daemon-minted relative directory, and lets the daemon validate and import it. Upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive. + +Choose the upload mechanism explicitly: + +- Use the default input mode for an upload button, file-input label, or “upload from computer” action. +- Use `--mode drop` only when the attachment can be reliably attributed to the target business editor: an explicit drop zone, chat composer, email editor, or form attachment area. Do not target page whitespace, a generic container, or an area whose attachment ownership is ambiguous. +- If input mode returns `reason=file_input_not_activated` with `effect_state=none`, re-observe. When a reliable attachment-receiving area is present, attempt drop once against that area; otherwise use `request-help` when available. +- Never switch mechanisms or repeat an upload when `effect_state=unknown` or `committed`. A successful drop command means Chrome dispatched the native file drop to the resolved zone; it does not prove that the site's application logic accepted the attachment. Observe the page once after the command, and do not repeat the drop when the attachment is already present. Do not use `request-help` merely because a native file chooser or browser download is involved; try these commands first. For transfer failures, use the structured error instead of retrying blindly: -- `reason=file_input_not_activated` means the requested click did not activate exactly one ``; the page may use a non-input picker such as `window.showOpenFilePicker()`. Do not retry blindly. If human help is available, call `request-help` and tell the user the exact original local path to choose. The staged daemon path is internal and must not be shown to the user. +- `reason=file_input_not_activated` means the requested click did not activate exactly one ``; the page may use a non-input picker such as `window.showOpenFilePicker()`. Follow the explicit mechanism-selection rule above. If no reliable attachment-receiving area exists, call `request-help` when available and tell the user the exact original local path to choose. The staged daemon path is internal and must not be shown to the user. - `reason=file_input_probe_failed` means BrowserSkill could not safely establish the browser-side upload transaction. Do not repeat the same action; use `request-help` when available. - `reason=set_file_input_failed` means BrowserSkill found the activated file input but Chrome rejected the staged path or assignment. Check the extension's file-URL access permission; otherwise use `request-help`. - `reason=upload_mechanism_unsupported` means the browser cannot perform a native file drop. Do not retry the same drop; use a standard file-input trigger or `request-help`. -- `reason=file_drop_target_unavailable` means the specified point is no longer owned by the resolved drop zone. When `effect_state=none`, re-observe and select the visible drop zone itself; do not guess another target. +- `reason=file_drop_target_unavailable` means the specified point is no longer owned by the resolved attachment target. When `effect_state=none`, re-observe and select the attachment-receiving editor or drop zone itself; do not guess another target. - `reason=file_drop_failed` means the native drop command failed. Do not retry when `effect_state=unknown`; first observe whether the page already received the file. - `reason=download_capture_failed` means BrowserSkill could not attribute exactly one completed download to the requested target. Do not retry blindly or accept an unrelated browser download; use `request-help` when available. - `effect_state=none` means BrowserSkill confirmed that no file-transfer effect was committed. Follow the accompanying reason; a corrected target or explicit human fallback may be attempted. diff --git a/crates/bsk-cli/src/cli/render_error.rs b/crates/bsk-cli/src/cli/render_error.rs index e402b8d..01f80dd 100644 --- a/crates/bsk-cli/src/cli/render_error.rs +++ b/crates/bsk-cli/src/cli/render_error.rs @@ -306,7 +306,7 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend (ErrorCode::Unsupported, reason::FILE_INPUT_NOT_ACTIVATED) => RenderInfo { summary: "the upload trigger did not activate a file input", hint: Some( - "the page may use a non-input picker such as showOpenFilePicker(); do not retry blindly — use `bsk request-help` with the exact original local path, or stop if human help is disabled", + "the page may use a non-input picker such as showOpenFilePicker(); when effect_state is none, re-observe and use `--mode drop` once only for a reliably identified attachment-receiving area — otherwise use `bsk request-help` with the exact original local path", ), exit_code: base.exit_code, }, @@ -339,7 +339,7 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend ) => RenderInfo { summary: "the file-drop target could not be safely resolved", hint: Some( - "rerun observe and select the visible drop zone itself; no file was delivered when effect_state is none", + "rerun observe and select the attachment-receiving editor or drop zone itself; do not guess another target — no file was delivered when effect_state is none", ), exit_code: base.exit_code, }, @@ -504,7 +504,10 @@ mod tests { info.summary, "the upload trigger did not activate a file input" ); - assert!(info.hint.unwrap().contains("exact original local path")); + let hint = info.hint.unwrap(); + assert!(hint.contains("--mode drop")); + assert!(hint.contains("reliably identified attachment-receiving area")); + assert!(hint.contains("exact original local path")); let control = serde_json::json!({ "reason": reason::FILE_INPUT_PROBE_FAILED }); let info = info_for_error(ErrorCode::Timeout, Some(&control)); diff --git a/skill/SKILL.md b/skill/SKILL.md index bb8dce3..8fdf138 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -210,22 +210,29 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff | Command | Summary | |---------|---------| -| `bsk upload --file ` | Attach a local file through a file-input trigger; use `--mode drop` only for an explicit drop zone (`--file` is repeatable) | +| `bsk upload --file ` | Attach a local file through a file-input trigger; use `--mode drop` for a reliably identified attachment-receiving area (`--file` is repeatable) | | `bsk download --out ` | Click one download trigger and copy the single completed file to an exact local path (`--overwrite` is opt-in) | The agent/harness decides whether a file transfer is appropriate and which local path belongs to the task. Treat upload as disclosure of that file to the current website, and download as accepting website-controlled bytes onto the local filesystem. Use only paths that are necessary for the user's bounded goal. -BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted capabilities reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. The default upload mode intercepts one chooser transaction, locates the file input activated in the resolved target's document, and assigns only the staged paths. `--mode drop` does not click: it delivers the same staged files through one native browser drop bound to the explicitly resolved drop zone. These mechanisms never fall back to each other automatically. Download uniquely correlates one exact-target browser intent with one Chrome download in either event order, routes it through a daemon-minted relative directory, and lets the daemon validate and import it. Upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive. +BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted capabilities reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. The default upload mode intercepts one chooser transaction, locates the file input activated in the resolved target's document, and assigns only the staged paths. `--mode drop` does not click: it delivers the same staged files through one native browser drop bound to the explicitly resolved attachment target. These mechanisms never fall back to each other automatically. Download uniquely correlates one exact-target browser intent with one Chrome download in either event order, routes it through a daemon-minted relative directory, and lets the daemon validate and import it. Upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive. + +Choose the upload mechanism explicitly: + +- Use the default input mode for an upload button, file-input label, or “upload from computer” action. +- Use `--mode drop` only when the attachment can be reliably attributed to the target business editor: an explicit drop zone, chat composer, email editor, or form attachment area. Do not target page whitespace, a generic container, or an area whose attachment ownership is ambiguous. +- If input mode returns `reason=file_input_not_activated` with `effect_state=none`, re-observe. When a reliable attachment-receiving area is present, attempt drop once against that area; otherwise use `request-help` when available. +- Never switch mechanisms or repeat an upload when `effect_state=unknown` or `committed`. A successful drop command means Chrome dispatched the native file drop to the resolved zone; it does not prove that the site's application logic accepted the attachment. Observe the page once after the command, and do not repeat the drop when the attachment is already present. Do not use `request-help` merely because a native file chooser or browser download is involved; try these commands first. For transfer failures, use the structured error instead of retrying blindly: -- `reason=file_input_not_activated` means the requested click did not activate exactly one ``; the page may use a non-input picker such as `window.showOpenFilePicker()`. Do not retry blindly. If human help is available, call `request-help` and tell the user the exact original local path to choose. The staged daemon path is internal and must not be shown to the user. +- `reason=file_input_not_activated` means the requested click did not activate exactly one ``; the page may use a non-input picker such as `window.showOpenFilePicker()`. Follow the explicit mechanism-selection rule above. If no reliable attachment-receiving area exists, call `request-help` when available and tell the user the exact original local path to choose. The staged daemon path is internal and must not be shown to the user. - `reason=file_input_probe_failed` means BrowserSkill could not safely establish the browser-side upload transaction. Do not repeat the same action; use `request-help` when available. - `reason=set_file_input_failed` means BrowserSkill found the activated file input but Chrome rejected the staged path or assignment. Check the extension's file-URL access permission; otherwise use `request-help`. - `reason=upload_mechanism_unsupported` means the browser cannot perform a native file drop. Do not retry the same drop; use a standard file-input trigger or `request-help`. -- `reason=file_drop_target_unavailable` means the specified point is no longer owned by the resolved drop zone. When `effect_state=none`, re-observe and select the visible drop zone itself; do not guess another target. +- `reason=file_drop_target_unavailable` means the specified point is no longer owned by the resolved attachment target. When `effect_state=none`, re-observe and select the attachment-receiving editor or drop zone itself; do not guess another target. - `reason=file_drop_failed` means the native drop command failed. Do not retry when `effect_state=unknown`; first observe whether the page already received the file. - `reason=download_capture_failed` means BrowserSkill could not attribute exactly one completed download to the requested target. Do not retry blindly or accept an unrelated browser download; use `request-help` when available. - `effect_state=none` means BrowserSkill confirmed that no file-transfer effect was committed. Follow the accompanying reason; a corrected target or explicit human fallback may be attempted. From 6cc0d384897800788922b258c8fde585eefa815e Mon Sep 17 00:00:00 2001 From: drakezhang Date: Wed, 2 Sep 2026 17:43:58 +0800 Subject: [PATCH 11/12] docs(skill): compress CLI SKILL.md with file transfer guidance Restore the lean style from PR #167 while integrating essential file transfer documentation for upload/download. Reduces the skill from 362 lines / 26k chars to 180 lines / 9.2k chars. A/B eval (24 browser runs, Composer 2.5 Fast, core suite) confirms no pass-rate regression: candidate 12/12 vs baseline 10/12 (baseline failures were a network blip and a behavioral variance, not skill quality). Co-authored-by: Cursor --- crates/bsk-cli/skill/SKILL.md | 438 ++++++++++------------------------ skill/SKILL.md | 438 ++++++++++------------------------ 2 files changed, 256 insertions(+), 620 deletions(-) diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index 8fdf138..357625c 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -1,362 +1,180 @@ --- name: browser-skill description: | - Use when the user asks to perform browser automation tasks against their - logged-in browser: visit and read pages, fill forms, scrape data, click - through a flow, regression-test a PR's UI, validate a deployed page. - Requires the bsk CLI installed and the browser-skill extension loaded. + Use when the user asks to automate their logged-in Chromium browser: visit + and read pages, fill forms, scrape data, click through flows, regression-test + a PR's UI, validate a deployed page, or operate a tab they identify. Requires + the bsk CLI and browser extension. --- # browser-skill -Drive the user's **real Chromium browser** (with their logins and cookies) through the `bsk` CLI. The extension opens an isolated **Agent Window** for automation; the user's normal windows stay protected unless you explicitly borrow a tab. +Drive the user's real Chromium browser through `bsk`. Automation runs in an isolated **Agent +Window** with the user's existing logins and cookies. User-window tabs remain protected unless they +are explicitly borrowed. -## When to use +Do not use this skill for tasks with no browser, for extension installation, or when the user only +wants instructions. Never extract credentials, cookies, tokens, or other secrets from pages. -- Open pages, read titles/text, scrape structured data from sites the user can already access -- Fill forms, click through multi-step flows, smoke-test a UI change -- Understand pages with `bsk snapshot` first; use `bsk get-html` or `bsk screenshot` only when the snapshot is insufficient -- Operate on a specific user tab they point you at (after `bsk tab borrow`) +## Required lifecycle -## When NOT to use +Every browser task owns a bounded session: -- Tasks with **no browser** involved (files, APIs, databases only) -- Installing or configuring the extension (point the user to setup docs instead) -- **Credential harvesting** — never run `bsk evaluate` on banking, SSO, or password-manager pages to extract tokens, cookies, or secrets -- Long-lived control of a user's personal login window — borrow only for the immediate step, then `bsk tab return` or end the session -- Replacing the user's manual browsing when they only wanted an explanation - -## Prerequisites - -1. `bsk` on `PATH` (Rust CLI from browser-skill) -2. browser-skill **extension** loaded in Chromium and connected (popup shows green) -3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails - -## Mandatory workflow - -Every automation task **must** follow this lifecycle. Do **not** rely on idle timeouts (default session idle is 5 minutes). - -``` -1. bsk session start → capture the 4-letter session id printed on stdout -2. … every tool command … → always pass --session -3. bsk session stop → REQUIRED when done (even on error paths) +```text +1. bsk session start # retain the printed 4-letter session id +2. bsk ... --session # pass it to every session-scoped command +3. bsk session stop # always run on success and error paths ``` -Optional: `bsk session start --browser ` when multiple browsers are connected (`bsk browsers` / error output lists them). Add `--no-focus` to open the Agent Window in the background without stealing focus from the user's current window. +Do not rely on the idle timeout for cleanup. Stop the session as soon as the goal is met unless the +user explicitly asks to keep it open. Stopping also returns borrowed tabs. -Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**. +Any `bsk` command auto-starts the background services it needs; never manage the daemon by hand. +When multiple browsers are connected, use `bsk browsers` and start with +`bsk session start --browser `. Add `--no-focus` to that same start command when the +Agent Window does not need to interrupt the user's current work; it is not a flag on other commands. +Run `bsk doctor` when startup or transport problems persist after one retry. -## Stop when the goal is met +## Work toward one observable goal -Every task is a **bounded goal**, not open-ended browsing. The goal may come from the user's request, a recorded `trace.json`, or both. +- Derive a concrete success condition from the user's request or a supplied trace. +- Take the shortest purposeful path: observe, act, then make at most one observation to confirm an + ambiguous result. +- Once success is visible, do not click, refresh, navigate, switch tabs, or perform extra checks. +- If a human-only step appears or two attempts make no progress, request help instead of + brute-forcing. -1. **Define success first** — one concrete, observable condition derived from the user's words, `purpose`, or the last meaningful step in a trace (e.g. "form submitted", "item added to cart", "playback started"). -2. **Take the shortest path** — snapshot → act → at most one check. Do not wander, re-try unrelated actions, or stack exploratory steps. -3. **Stop as soon as success is reached** — run `bsk session stop ` immediately unless the user explicitly asked to keep the session open (e.g. "don't close yet", "keep browsing"). -4. **No post-success work** — once the goal is met, do not click, refresh, navigate, re-search, switch tabs, or "double-check" that it worked. Further verification is a new task. -5. **When blocked, pause — do not brute-force** — if the page requires human input (login, captcha, OTP, payment confirmation) or an action fails twice with no progress, call `bsk request-help` instead of retrying blindly. See **Ask the human for help** below. -6. **When unsure** — at most one extra `bsk snapshot`. If success looks met, stop. If not, ask the user; do not keep clicking. +With a trace, follow its semantic target information and values in order, but treat its refs as +record-local hints. Stop when its purpose or last meaningful effect is satisfied. A trace guides the +task; it does not expand the user's goal or authorize additional actions. -**With a trace:** replay steps in order using `target` role/name/tag and raw `value`/`selection` fields. After the last step (or when its `effect.navigated_to` / success hint is satisfied), apply rules 3–4 immediately. The trace guides execution; it does not extend control beyond the goal. +## Observe, act, observe -**Without a trace:** the user's request *is* the success condition. Satisfying it ends the task — same stop rules apply. +Use this default loop: -## Core interaction loop - -Write operations only affect tabs in the **Agent Window** (or tabs you **borrowed** into it). - -``` +```text bsk navigate --session -bsk observe --session → primary semantic VOM view; reveals hover/focus surfaces -bsk snapshot --session → static aria tree fallback when VOM is insufficient -bsk hover @e3 --session → reveal hover-triggered menus before re-observing/clicking -bsk click @e4 --session → or bsk fill, bsk select, bsk press -bsk observe --session → again after navigation / DOM change +bsk observe --session +bsk click|hover|fill|select|press ... --session +bsk observe --session # after navigation or a meaningful DOM change ``` -**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page. +Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also +make them stale. Observe again before the next interaction. -Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`). +An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. +The listed items are labels, not usable refs: hover the trigger, observe again, then act on the +revealed item's own ref. Do not click the trigger itself unless the user wants the trigger's action. +`[has-submenu]` and `[expanded]` mark the same kind of trigger without listing what it hides. -When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover --session `, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action. +`bsk observe` does not hover the page on its own. Reach for `--probe-hover` when a control you have +good reason to expect is absent **and** no marker points at a trigger — that combination is what a +CSS-only hover menu looks like from here. It hovers a bounded set of likely triggers, so it costs a +few seconds and touches the live page; once you know which element hides the menu, `bsk hover ` +is cheaper and more precise. -## Observation priority +Escalate page reading only as needed: -Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question: +1. `bsk observe` for normal semantic understanding, text, controls, and refs. +2. `bsk observe --probe-hover` once when an expected control is missing and no marker points at a + trigger. +3. `bsk snapshot` when a stricter static accessibility tree is more useful. +4. `bsk get-html` for exact markup or hidden metadata that semantic views cannot provide. +5. `bsk screenshot` for layout, styling, canvas, images, or requested visual evidence. -1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery -2. `bsk snapshot` — strict static accessibility tree fallback -3. `bsk get-html` — when hidden DOM, metadata, or markup details are required -4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab. +Do not start with raw HTML or screenshots merely to discover ordinary controls. When interaction is +needed, obtain a fresh observation before acting on screenshot or HTML findings. -Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page. +## Respect the Agent Window boundary -## Sandbox rules +Normal page writes affect only Agent Window tabs. To operate a user tab, first list it with +`bsk tab list --scope user --session `, then `bsk tab borrow `. Return it immediately +after the relevant step with `bsk tab return `; never invent a tab id or keep a personal tab +borrowed across unrelated work. -| Rule | Detail | -|------|--------| -| Agent Window | `bsk tab create`, `bsk navigate`, `bsk click`, etc. work on agent tabs by default | -| User tabs | Read-only until borrowed: `bsk tab list --session --scope user` then `bsk tab borrow --session ` | -| Return borrowed tabs | Call `bsk tab return --session ` when finished; unreturned tabs are **auto-returned** on `bsk session stop` | -| Writes off-agent | Commands that mutate the page fail if the tab is not in the Agent Window — borrow or create a tab first | +## Ask the human when needed -## Global flags +Use `bsk request-help` for login, captcha, OTP, payment confirmation, consent, or another step the +user must complete. Give a precise prompt and pass fresh `--target` refs/selectors when concrete +controls can be highlighted. Use completion criteria only when the page has a clear stable success +signal. -| Flag | Purpose | -|------|---------| -| `--json` | Machine-readable JSON on stdout (errors too) | -| `--quiet` | Suppress informational stderr | -| `-v` / `-vv` | More verbose logging | +The result `outcome` is one of `continued`, `completed`, `cancelled`, `timed_out`, or `disabled` +(`navigated` is deprecated — never treat navigation as a completion signal). Resume only after +`continued` or `completed`. Treat `cancelled` as rejection, and `timed_out` or `disabled` as a +blocker rather than a reason to retry. After control returns, run a fresh `bsk observe` before +reasoning about the page or using refs. -Auto-update is **on by default** — the background daemon upgrades `bsk` itself when a new release is available (postponed while a session is active). Set `BSK_AUTO_UPDATE=off` to disable it and upgrade manually with `bsk update`. +## Command inventory -Command-specific flags (timeouts, `--tab-id`, `--wait-until`, …): **`bsk --help`** +This list of names is complete. Never invent a command outside it; read +`bsk --help` for flags instead of guessing them. -## CLI command reference (one line each) - -Details and flags: **`bsk --help`** - -### Diagnostics - -| Command | Summary | -|---------|---------| -| `bsk status` | Connection health, connected browsers, active sessions | -| `bsk doctor` | Deep diagnostics and repair hints | -| `bsk browsers` | List connected browser instances (ids, labels, versions) | - -### Session - -| Command | Summary | -|---------|---------| -| `bsk session start` | Open Agent Window (`--width`/`--height` for initial size); prints **4-letter session id** | -| `bsk session start --no-focus` | Open Agent Window in the background without stealing focus | -| `bsk session stop ` | End session, close Agent Window, auto-return borrowed tabs | -| `bsk session stop --all` | Stop every active session | -| `bsk session list` | List active sessions | - -### Window (require `--session `) - -| Command | Summary | -|---------|---------| -| `bsk window resize` | Resize the Agent Window (`--width`, `--height`; 100..=7680 CSS px) | - -### Device emulation — `bsk emulate` (requires `--session `) - -Emulate a mobile device environment on the agent tab via CDP — viewport, User-Agent, and touch — to debug a page's mobile layout and behaviour: - -```bash -bsk emulate --session --device iphone-14 -bsk emulate --session --width 390 --height 844 --dpr 3 --mobile --ua "Mozilla/5.0 (iPhone…" --touch -bsk emulate --session --off +```text +session start|stop|list browsers status doctor update logs +navigate navigate-back navigate-forward reload wait-for-navigation wait-ms +observe snapshot get-html screenshot console network +click hover fill select press evaluate +tab list|create|close|select|borrow|return window resize emulate +upload download request-help record start|stop ``` -- Presets (`--device`): `iphone-14`, `iphone-14-pro-max`, `iphone-se`, `pixel-7`, `galaxy-s23`, `ipad-mini`, `galaxy-tab-s8`. Manual flags (`--width`/`--height`/`--dpr`/`--mobile`/`--ua`/`--accept-language`/`--touch`/`--max-touch-points`) also work without a preset, or override individual preset fields; `--no-mobile`/`--no-touch` turn a preset's mobile viewport / touch emulation back off. -- Repeated runs merge field by field onto the tab's current emulation state — only the flags you pass change. E.g. after `--device iphone-14`, `bsk emulate --session --width 390 --height 844` keeps the preset's dpr (`3`) and mobile viewport. (The extension remembers the state per tab; after an extension reload the next run applies only the fields it carries.) -- `--off` clears every override (viewport, UA, touch) and the remembered state, restoring the tab's real environment. -- Scope: overrides apply to **one tab only** (CDP per-target) and are **not inherited by new tabs** — re-run `bsk emulate` after opening or switching to another tab (default target is the session's active tab; `--tab-id` overrides). -- Emulation covers viewport/UA/touch only; it does not throttle the network, fake geolocation, or synthesise real touch-event streams. +Required flags that are easy to get wrong: -### Tabs (require `--session `) - -| Command | Summary | -|---------|---------| -| `bsk tab list` | List tabs (`--scope user\|agent\|all`, default `all`) | -| `bsk tab create` | New tab in Agent Window (`--url`, `--no-active`, `--index`) | -| `bsk tab close ` | Close an agent tab | -| `bsk tab select ` | Focus an agent tab | -| `bsk tab borrow ` | Move a user tab into the Agent Window | -| `bsk tab return ` | Return a borrowed tab to its original window | - -### Observation (require `--session` unless noted) - -| Command | Summary | -|---------|---------| -| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs | -| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces | -| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) | -| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) | - -### Console & network debugging (read-only; require `--session`) - -| Command | Summary | -|---------|---------| -| `bsk console` | Buffered page console messages, JS exceptions, and browser log entries (`--include-stack` for stack traces) | -| `bsk network` | Buffered network responses (status, method, URL, MIME/resource type) and failures (`net::ERR_*` reason) | - -Both capture from the moment the tab is attached and read a bounded per-tab buffer: `--since ` pages from a cursor (`next_since` in the result), `--limit` (default 50, max 200), `--max-text-chars` (default 1000, max 4096), `--tab-id` to target a non-active tab. Both are strictly read-only — they never intercept or modify traffic, and request/response headers, bodies, and timings are not captured. - -### Navigation - -| Command | Summary | -|---------|---------| -| `bsk navigate ` | Go to URL in agent tab (`--wait-until`, `--timeout`) | -| `bsk navigate-back` | History back one step | -| `bsk navigate-forward` | History forward one step | -| `bsk reload` | Reload current tab (`--hard` bypass cache) | - -(`bsk navigate back` / `bsk navigate forward` are equivalent subcommands.) - -### Interaction - -| Command | Summary | -|---------|---------| -| `bsk click ` | Click element (`--button`, `--click-count`, `--modifiers`) | -| `bsk hover ` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) | -| `bsk fill --value ` | Clear and type into input | -| `bsk select --value ` | Set ``; the page may use a non-input picker such as `window.showOpenFilePicker()`. Follow the explicit mechanism-selection rule above. If no reliable attachment-receiving area exists, call `request-help` when available and tell the user the exact original local path to choose. The staged daemon path is internal and must not be shown to the user. -- `reason=file_input_probe_failed` means BrowserSkill could not safely establish the browser-side upload transaction. Do not repeat the same action; use `request-help` when available. -- `reason=set_file_input_failed` means BrowserSkill found the activated file input but Chrome rejected the staged path or assignment. Check the extension's file-URL access permission; otherwise use `request-help`. -- `reason=upload_mechanism_unsupported` means the browser cannot perform a native file drop. Do not retry the same drop; use a standard file-input trigger or `request-help`. -- `reason=file_drop_target_unavailable` means the specified point is no longer owned by the resolved attachment target. When `effect_state=none`, re-observe and select the attachment-receiving editor or drop zone itself; do not guess another target. -- `reason=file_drop_failed` means the native drop command failed. Do not retry when `effect_state=unknown`; first observe whether the page already received the file. -- `reason=download_capture_failed` means BrowserSkill could not attribute exactly one completed download to the requested target. Do not retry blindly or accept an unrelated browser download; use `request-help` when available. -- `effect_state=none` means BrowserSkill confirmed that no file-transfer effect was committed. Follow the accompanying reason; a corrected target or explicit human fallback may be attempted. -- `effect_state=unknown` means the browser may already have attached or created the file. Do not repeat the transfer. Observe the page if that can establish the result; otherwise stop and report the uncertainty. -- `effect_state=committed` means the browser-side effect occurred even if later completion or cleanup failed. Do not repeat it; continue only after verifying the resulting page/download state. - -If `request-help` returns `outcome="disabled"`, do not retry it. Stop gracefully and report the transfer mechanism that requires human intervention. - -### Scripting & timing - -| Command | Summary | -|---------|---------| -| `bsk evaluate ` | Run JS in agent tab (see red lines); JS throw → stderr, **exit 0** | -| `bsk wait-for-navigation` | Block until load/DOM idle/etc. (`--wait-until`, `--timeout`) | -| `bsk wait-ms ` | Sleep (`500ms`, `2s`, `1m`; **no** `--session`) | - -### Ask the human for help — `bsk request-help` - -When a step needs a human (captcha, login, OTP) or you want the user to -confirm an important action, pause and ask: - - bsk request-help --session --prompt "Solve the captcha, then click Done only after the site accepts it" \ - --title "Captcha required" --target @e7 --target "#submit" --timeout 5m - -- `--prompt` (required): what the user should do. -- `--title` (optional): custom title for the overlay panel. When omitted, - the extension shows its default localized title. -- `--target` (repeatable): a snapshot ref (`@e7`) or CSS selector - (`#submit`) to scroll to and flash-highlight. **Strongly recommended** — - whenever the prompt refers to a concrete element (a button to click, a - field to fill, a checkbox to toggle), pass its `@eN` ref / selector so the - user is guided straight to the right spot instead of hunting for it. For - interaction scenarios, always include the relevant target(s); reserve a - prompt with no `--target` for cases where there is genuinely no specific - element to point at (e.g. "wait for the page to finish loading"). -- `--timeout` (default `5m`): how long to wait. -- `--completion-criteria` (optional): JSON success detector. Use it only - when there is a concrete post-help success signal, e.g. - `{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`. - -The target tab is brought to the foreground; the page stays interactive -while the agent control mask is hidden. The call blocks until the user -explicitly acts, the timeout expires, cancellation arrives, or explicit -completion criteria match. Page reloads, SPA route changes, and captcha -refreshes do not return control by themselves. The result `outcome` is one of: - -- `continued` — the user finished and clicked Done / return control (treat as confirm). -- `cancelled` — the user clicked Cancel (treat as reject/abort). -- `timed_out` — nobody acted within the timeout. -- `completed` — the explicit `--completion-criteria` matched while the user had control. -- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal. - -`note` carries any text the user typed back. `resolved_targets` reports -which refs/selectors matched a live element. - -`request-help` does not refresh the page model after the user returns -control. After a `continued` or `completed` result, issue a separate -observation tool call (usually `bsk snapshot --session `) before using -new refs or reasoning about the post-help page state. -#### Disabling request-help (unattended mode) -Set `BSK_REQUEST_HELP=off` on unattended servers: `bsk request-help` then -returns immediately with `outcome="disabled"` (no overlay, no waiting, -exit 0). Any other value keeps it enabled. If you get `disabled`, do not -retry — complete the task autonomously or stop gracefully. - -### Recording — `bsk record` - -Capture the user's own actions in the Agent Window to a **trace bundle**, for later LLM-driven automation. New CLI builds request **trace v3** (page observations + action chain); older extensions may still return **trace v2** (actions only), which the CLI exports as a single `trace.json` without `states/`. - -```bash -bsk record start --browser \ - [--url https://…] [--purpose "publish a wiki doc"] \ - [--max-page-tokens 3000] [--redact-values] \ - [--output trace] -# `--url` is optional; default https://example.com/ when omitted (must be http(s)). -# Blocks until the user clicks Finish in the recording panel, then writes: -# trace/trace.json — action chain (+ state index when v3) -# trace/states/ — v3 only: one `sN.txt` observe snapshot per settled page state - -bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable +```text +bsk fill --value bsk select --value +bsk screenshot --out bsk emulate --device +bsk upload --file bsk download --out ``` -- **v3 bundle (preferred):** `--output` is a directory (default `./trace`) containing `trace.json` and `states/`. `trace.json` lists `states[]` (page observation ids) and `steps[]` (each step binds `state` = observe snapshot *before* the action and `result.state` = snapshot *after* settle). Page bodies live in `states/sN.txt` using the same VOM format as `bsk observe`. -- **v2 fallback:** when the connected extension is older, export may contain only legacy `trace.json` with `pages[]` action context and no `states/` observation files. Update the extension for full v3 bundles. -- Each `states[]` entry is one **settled page observation** (captured after recording start, navigation landing, or action settle — not on a timer). -- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead. -- `--purpose` is optional context metadata; it does **not** change what gets captured. -- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`. -- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text. +`select` matches an option's `value` attribute, not its visible label. Device preset ids are +lowercase and hyphenated, such as `iphone-14`. -## Error handling +- `console` and `network` provide bounded, read-only debugging evidence. +- `emulate` applies viewport, user-agent, and touch overrides to one tab; new tabs do not inherit + them. Use `--off` to restore the real environment. +- `evaluate` is a last resort when observe plus normal interactions cannot complete the task. With + `--json`, inspect `.ok`: a JavaScript exception may still have CLI exit code 0 because the RPC + succeeded. Never evaluate credential surfaces to read storage, cookies, or auth data. +- `record` captures a user's actions for later replay. Read `bsk record start --help` before use, + and never record banking, SSO, password-manager, or other sensitive pages. -### Exit codes (`echo $?` after `bsk …`) +## File transfer -| Code | Meaning | What to do | -|------|---------|------------| -| `0` | Success (including `evaluate` where JS threw but RPC succeeded) | Continue | -| `1` | User error — bad args, unknown session, tab not in Agent Window, stale ref | Fix args; `bsk session list`; re-snapshot | -| `2` | Protocol / transport — service unreachable, IPC failure | `bsk doctor`; check extension connected; retry the command | -| `3` | Browser / CDP execution failed | Retry; simplify selector; check tab still open | -| `4` | Timeout | Increase `--timeout`; try `--wait-until domcontentloaded` | -| `5` | Version skew (CLI vs extension) | Upgrade/reinstall matching versions | +`upload` and `download` stage files through the daemon; the agent never touches browser-internal +paths. Treat upload as disclosure to the website, download as accepting website-controlled bytes. -Human errors print `error:` + `hint:` on stderr; `--json` includes `code`, `message`, `hint`, `exit_code`. +Upload has two independent mechanisms — choose explicitly, never rely on automatic fallback: -### When to run diagnostics +- **Default (input mode):** for upload buttons, file-input labels, or "upload from computer" + actions. The command clicks the target and intercepts the native file chooser. +- **`--mode drop`:** for reliably identified attachment-receiving areas — an explicit drop zone, + chat composer, email editor, or form attachment area. Do not target page whitespace, generic + containers, or areas whose attachment ownership is ambiguous. -| Situation | Command | -|-----------|---------| -| Before first task in a session | `bsk status` — extension connected? | -| Any failure you cannot fix in one retry | `bsk doctor` | -| Multiple browsers / wrong target | `bsk browsers` then `bsk session start --browser ` | +Decision sequence when uploading: -Always **`bsk session stop `** in a `finally`-style path so the Agent Window closes and borrowed tabs return. +1. Try input mode (the default). +2. If it returns `reason=file_input_not_activated` with `effect_state=none`, re-observe. When a + reliable attachment target exists, try `--mode drop` once against that target. +3. Otherwise fall back to `request-help`. +4. **Never** switch mechanisms or repeat when `effect_state` is `unknown` or `committed` — the + browser may already have applied the file. -## Red lines +A successful drop means Chrome dispatched the native file-drop event; it does not prove the site +accepted the attachment. Observe the page once after the command. -1. **No token theft** — do not `bsk evaluate` on sensitive sites to read `localStorage`, cookies, or auth headers for exfiltration. -2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks. -3. **No skip stop** — always `bsk session stop `; never assume idle timeout will clean up. -4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open. -5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics or conditional surfaces help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual. -6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces. +Download default-refuses to overwrite; pass `--overwrite` when replacing an existing file is +intended. Read `bsk upload --help` and `bsk download --help` for all flags and error details. ---- +## Recover without wandering -**More detail for any command:** `bsk --help` +- Stale ref: observe again and retry the intended action once. +- Unknown tab or session: list current tabs/sessions; never guess identifiers. +- Timeout: inspect current page state before deciding whether one longer purposeful wait is useful. +- Unsupported command: continue with available capabilities; suggest updating only when the missing + command is necessary. +- Unrecoverable failure: report the blocker and stop the session in a finally-style path. + +The CLI's current help and error hints are authoritative for flags, parameters, and recovery +details. diff --git a/skill/SKILL.md b/skill/SKILL.md index 8fdf138..357625c 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -1,362 +1,180 @@ --- name: browser-skill description: | - Use when the user asks to perform browser automation tasks against their - logged-in browser: visit and read pages, fill forms, scrape data, click - through a flow, regression-test a PR's UI, validate a deployed page. - Requires the bsk CLI installed and the browser-skill extension loaded. + Use when the user asks to automate their logged-in Chromium browser: visit + and read pages, fill forms, scrape data, click through flows, regression-test + a PR's UI, validate a deployed page, or operate a tab they identify. Requires + the bsk CLI and browser extension. --- # browser-skill -Drive the user's **real Chromium browser** (with their logins and cookies) through the `bsk` CLI. The extension opens an isolated **Agent Window** for automation; the user's normal windows stay protected unless you explicitly borrow a tab. +Drive the user's real Chromium browser through `bsk`. Automation runs in an isolated **Agent +Window** with the user's existing logins and cookies. User-window tabs remain protected unless they +are explicitly borrowed. -## When to use +Do not use this skill for tasks with no browser, for extension installation, or when the user only +wants instructions. Never extract credentials, cookies, tokens, or other secrets from pages. -- Open pages, read titles/text, scrape structured data from sites the user can already access -- Fill forms, click through multi-step flows, smoke-test a UI change -- Understand pages with `bsk snapshot` first; use `bsk get-html` or `bsk screenshot` only when the snapshot is insufficient -- Operate on a specific user tab they point you at (after `bsk tab borrow`) +## Required lifecycle -## When NOT to use +Every browser task owns a bounded session: -- Tasks with **no browser** involved (files, APIs, databases only) -- Installing or configuring the extension (point the user to setup docs instead) -- **Credential harvesting** — never run `bsk evaluate` on banking, SSO, or password-manager pages to extract tokens, cookies, or secrets -- Long-lived control of a user's personal login window — borrow only for the immediate step, then `bsk tab return` or end the session -- Replacing the user's manual browsing when they only wanted an explanation - -## Prerequisites - -1. `bsk` on `PATH` (Rust CLI from browser-skill) -2. browser-skill **extension** loaded in Chromium and connected (popup shows green) -3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails - -## Mandatory workflow - -Every automation task **must** follow this lifecycle. Do **not** rely on idle timeouts (default session idle is 5 minutes). - -``` -1. bsk session start → capture the 4-letter session id printed on stdout -2. … every tool command … → always pass --session -3. bsk session stop → REQUIRED when done (even on error paths) +```text +1. bsk session start # retain the printed 4-letter session id +2. bsk ... --session # pass it to every session-scoped command +3. bsk session stop # always run on success and error paths ``` -Optional: `bsk session start --browser ` when multiple browsers are connected (`bsk browsers` / error output lists them). Add `--no-focus` to open the Agent Window in the background without stealing focus from the user's current window. +Do not rely on the idle timeout for cleanup. Stop the session as soon as the goal is met unless the +user explicitly asks to keep it open. Stopping also returns borrowed tabs. -Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**. +Any `bsk` command auto-starts the background services it needs; never manage the daemon by hand. +When multiple browsers are connected, use `bsk browsers` and start with +`bsk session start --browser `. Add `--no-focus` to that same start command when the +Agent Window does not need to interrupt the user's current work; it is not a flag on other commands. +Run `bsk doctor` when startup or transport problems persist after one retry. -## Stop when the goal is met +## Work toward one observable goal -Every task is a **bounded goal**, not open-ended browsing. The goal may come from the user's request, a recorded `trace.json`, or both. +- Derive a concrete success condition from the user's request or a supplied trace. +- Take the shortest purposeful path: observe, act, then make at most one observation to confirm an + ambiguous result. +- Once success is visible, do not click, refresh, navigate, switch tabs, or perform extra checks. +- If a human-only step appears or two attempts make no progress, request help instead of + brute-forcing. -1. **Define success first** — one concrete, observable condition derived from the user's words, `purpose`, or the last meaningful step in a trace (e.g. "form submitted", "item added to cart", "playback started"). -2. **Take the shortest path** — snapshot → act → at most one check. Do not wander, re-try unrelated actions, or stack exploratory steps. -3. **Stop as soon as success is reached** — run `bsk session stop ` immediately unless the user explicitly asked to keep the session open (e.g. "don't close yet", "keep browsing"). -4. **No post-success work** — once the goal is met, do not click, refresh, navigate, re-search, switch tabs, or "double-check" that it worked. Further verification is a new task. -5. **When blocked, pause — do not brute-force** — if the page requires human input (login, captcha, OTP, payment confirmation) or an action fails twice with no progress, call `bsk request-help` instead of retrying blindly. See **Ask the human for help** below. -6. **When unsure** — at most one extra `bsk snapshot`. If success looks met, stop. If not, ask the user; do not keep clicking. +With a trace, follow its semantic target information and values in order, but treat its refs as +record-local hints. Stop when its purpose or last meaningful effect is satisfied. A trace guides the +task; it does not expand the user's goal or authorize additional actions. -**With a trace:** replay steps in order using `target` role/name/tag and raw `value`/`selection` fields. After the last step (or when its `effect.navigated_to` / success hint is satisfied), apply rules 3–4 immediately. The trace guides execution; it does not extend control beyond the goal. +## Observe, act, observe -**Without a trace:** the user's request *is* the success condition. Satisfying it ends the task — same stop rules apply. +Use this default loop: -## Core interaction loop - -Write operations only affect tabs in the **Agent Window** (or tabs you **borrowed** into it). - -``` +```text bsk navigate --session -bsk observe --session → primary semantic VOM view; reveals hover/focus surfaces -bsk snapshot --session → static aria tree fallback when VOM is insufficient -bsk hover @e3 --session → reveal hover-triggered menus before re-observing/clicking -bsk click @e4 --session → or bsk fill, bsk select, bsk press -bsk observe --session → again after navigation / DOM change +bsk observe --session +bsk click|hover|fill|select|press ... --session +bsk observe --session # after navigation or a meaningful DOM change ``` -**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page. +Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also +make them stale. Observe again before the next interaction. -Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`). +An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. +The listed items are labels, not usable refs: hover the trigger, observe again, then act on the +revealed item's own ref. Do not click the trigger itself unless the user wants the trigger's action. +`[has-submenu]` and `[expanded]` mark the same kind of trigger without listing what it hides. -When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover --session `, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action. +`bsk observe` does not hover the page on its own. Reach for `--probe-hover` when a control you have +good reason to expect is absent **and** no marker points at a trigger — that combination is what a +CSS-only hover menu looks like from here. It hovers a bounded set of likely triggers, so it costs a +few seconds and touches the live page; once you know which element hides the menu, `bsk hover ` +is cheaper and more precise. -## Observation priority +Escalate page reading only as needed: -Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question: +1. `bsk observe` for normal semantic understanding, text, controls, and refs. +2. `bsk observe --probe-hover` once when an expected control is missing and no marker points at a + trigger. +3. `bsk snapshot` when a stricter static accessibility tree is more useful. +4. `bsk get-html` for exact markup or hidden metadata that semantic views cannot provide. +5. `bsk screenshot` for layout, styling, canvas, images, or requested visual evidence. -1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery -2. `bsk snapshot` — strict static accessibility tree fallback -3. `bsk get-html` — when hidden DOM, metadata, or markup details are required -4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab. +Do not start with raw HTML or screenshots merely to discover ordinary controls. When interaction is +needed, obtain a fresh observation before acting on screenshot or HTML findings. -Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page. +## Respect the Agent Window boundary -## Sandbox rules +Normal page writes affect only Agent Window tabs. To operate a user tab, first list it with +`bsk tab list --scope user --session `, then `bsk tab borrow `. Return it immediately +after the relevant step with `bsk tab return `; never invent a tab id or keep a personal tab +borrowed across unrelated work. -| Rule | Detail | -|------|--------| -| Agent Window | `bsk tab create`, `bsk navigate`, `bsk click`, etc. work on agent tabs by default | -| User tabs | Read-only until borrowed: `bsk tab list --session --scope user` then `bsk tab borrow --session ` | -| Return borrowed tabs | Call `bsk tab return --session ` when finished; unreturned tabs are **auto-returned** on `bsk session stop` | -| Writes off-agent | Commands that mutate the page fail if the tab is not in the Agent Window — borrow or create a tab first | +## Ask the human when needed -## Global flags +Use `bsk request-help` for login, captcha, OTP, payment confirmation, consent, or another step the +user must complete. Give a precise prompt and pass fresh `--target` refs/selectors when concrete +controls can be highlighted. Use completion criteria only when the page has a clear stable success +signal. -| Flag | Purpose | -|------|---------| -| `--json` | Machine-readable JSON on stdout (errors too) | -| `--quiet` | Suppress informational stderr | -| `-v` / `-vv` | More verbose logging | +The result `outcome` is one of `continued`, `completed`, `cancelled`, `timed_out`, or `disabled` +(`navigated` is deprecated — never treat navigation as a completion signal). Resume only after +`continued` or `completed`. Treat `cancelled` as rejection, and `timed_out` or `disabled` as a +blocker rather than a reason to retry. After control returns, run a fresh `bsk observe` before +reasoning about the page or using refs. -Auto-update is **on by default** — the background daemon upgrades `bsk` itself when a new release is available (postponed while a session is active). Set `BSK_AUTO_UPDATE=off` to disable it and upgrade manually with `bsk update`. +## Command inventory -Command-specific flags (timeouts, `--tab-id`, `--wait-until`, …): **`bsk --help`** +This list of names is complete. Never invent a command outside it; read +`bsk --help` for flags instead of guessing them. -## CLI command reference (one line each) - -Details and flags: **`bsk --help`** - -### Diagnostics - -| Command | Summary | -|---------|---------| -| `bsk status` | Connection health, connected browsers, active sessions | -| `bsk doctor` | Deep diagnostics and repair hints | -| `bsk browsers` | List connected browser instances (ids, labels, versions) | - -### Session - -| Command | Summary | -|---------|---------| -| `bsk session start` | Open Agent Window (`--width`/`--height` for initial size); prints **4-letter session id** | -| `bsk session start --no-focus` | Open Agent Window in the background without stealing focus | -| `bsk session stop ` | End session, close Agent Window, auto-return borrowed tabs | -| `bsk session stop --all` | Stop every active session | -| `bsk session list` | List active sessions | - -### Window (require `--session `) - -| Command | Summary | -|---------|---------| -| `bsk window resize` | Resize the Agent Window (`--width`, `--height`; 100..=7680 CSS px) | - -### Device emulation — `bsk emulate` (requires `--session `) - -Emulate a mobile device environment on the agent tab via CDP — viewport, User-Agent, and touch — to debug a page's mobile layout and behaviour: - -```bash -bsk emulate --session --device iphone-14 -bsk emulate --session --width 390 --height 844 --dpr 3 --mobile --ua "Mozilla/5.0 (iPhone…" --touch -bsk emulate --session --off +```text +session start|stop|list browsers status doctor update logs +navigate navigate-back navigate-forward reload wait-for-navigation wait-ms +observe snapshot get-html screenshot console network +click hover fill select press evaluate +tab list|create|close|select|borrow|return window resize emulate +upload download request-help record start|stop ``` -- Presets (`--device`): `iphone-14`, `iphone-14-pro-max`, `iphone-se`, `pixel-7`, `galaxy-s23`, `ipad-mini`, `galaxy-tab-s8`. Manual flags (`--width`/`--height`/`--dpr`/`--mobile`/`--ua`/`--accept-language`/`--touch`/`--max-touch-points`) also work without a preset, or override individual preset fields; `--no-mobile`/`--no-touch` turn a preset's mobile viewport / touch emulation back off. -- Repeated runs merge field by field onto the tab's current emulation state — only the flags you pass change. E.g. after `--device iphone-14`, `bsk emulate --session --width 390 --height 844` keeps the preset's dpr (`3`) and mobile viewport. (The extension remembers the state per tab; after an extension reload the next run applies only the fields it carries.) -- `--off` clears every override (viewport, UA, touch) and the remembered state, restoring the tab's real environment. -- Scope: overrides apply to **one tab only** (CDP per-target) and are **not inherited by new tabs** — re-run `bsk emulate` after opening or switching to another tab (default target is the session's active tab; `--tab-id` overrides). -- Emulation covers viewport/UA/touch only; it does not throttle the network, fake geolocation, or synthesise real touch-event streams. +Required flags that are easy to get wrong: -### Tabs (require `--session `) - -| Command | Summary | -|---------|---------| -| `bsk tab list` | List tabs (`--scope user\|agent\|all`, default `all`) | -| `bsk tab create` | New tab in Agent Window (`--url`, `--no-active`, `--index`) | -| `bsk tab close ` | Close an agent tab | -| `bsk tab select ` | Focus an agent tab | -| `bsk tab borrow ` | Move a user tab into the Agent Window | -| `bsk tab return ` | Return a borrowed tab to its original window | - -### Observation (require `--session` unless noted) - -| Command | Summary | -|---------|---------| -| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs | -| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces | -| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) | -| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) | - -### Console & network debugging (read-only; require `--session`) - -| Command | Summary | -|---------|---------| -| `bsk console` | Buffered page console messages, JS exceptions, and browser log entries (`--include-stack` for stack traces) | -| `bsk network` | Buffered network responses (status, method, URL, MIME/resource type) and failures (`net::ERR_*` reason) | - -Both capture from the moment the tab is attached and read a bounded per-tab buffer: `--since ` pages from a cursor (`next_since` in the result), `--limit` (default 50, max 200), `--max-text-chars` (default 1000, max 4096), `--tab-id` to target a non-active tab. Both are strictly read-only — they never intercept or modify traffic, and request/response headers, bodies, and timings are not captured. - -### Navigation - -| Command | Summary | -|---------|---------| -| `bsk navigate ` | Go to URL in agent tab (`--wait-until`, `--timeout`) | -| `bsk navigate-back` | History back one step | -| `bsk navigate-forward` | History forward one step | -| `bsk reload` | Reload current tab (`--hard` bypass cache) | - -(`bsk navigate back` / `bsk navigate forward` are equivalent subcommands.) - -### Interaction - -| Command | Summary | -|---------|---------| -| `bsk click ` | Click element (`--button`, `--click-count`, `--modifiers`) | -| `bsk hover ` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) | -| `bsk fill --value ` | Clear and type into input | -| `bsk select --value ` | Set ``; the page may use a non-input picker such as `window.showOpenFilePicker()`. Follow the explicit mechanism-selection rule above. If no reliable attachment-receiving area exists, call `request-help` when available and tell the user the exact original local path to choose. The staged daemon path is internal and must not be shown to the user. -- `reason=file_input_probe_failed` means BrowserSkill could not safely establish the browser-side upload transaction. Do not repeat the same action; use `request-help` when available. -- `reason=set_file_input_failed` means BrowserSkill found the activated file input but Chrome rejected the staged path or assignment. Check the extension's file-URL access permission; otherwise use `request-help`. -- `reason=upload_mechanism_unsupported` means the browser cannot perform a native file drop. Do not retry the same drop; use a standard file-input trigger or `request-help`. -- `reason=file_drop_target_unavailable` means the specified point is no longer owned by the resolved attachment target. When `effect_state=none`, re-observe and select the attachment-receiving editor or drop zone itself; do not guess another target. -- `reason=file_drop_failed` means the native drop command failed. Do not retry when `effect_state=unknown`; first observe whether the page already received the file. -- `reason=download_capture_failed` means BrowserSkill could not attribute exactly one completed download to the requested target. Do not retry blindly or accept an unrelated browser download; use `request-help` when available. -- `effect_state=none` means BrowserSkill confirmed that no file-transfer effect was committed. Follow the accompanying reason; a corrected target or explicit human fallback may be attempted. -- `effect_state=unknown` means the browser may already have attached or created the file. Do not repeat the transfer. Observe the page if that can establish the result; otherwise stop and report the uncertainty. -- `effect_state=committed` means the browser-side effect occurred even if later completion or cleanup failed. Do not repeat it; continue only after verifying the resulting page/download state. - -If `request-help` returns `outcome="disabled"`, do not retry it. Stop gracefully and report the transfer mechanism that requires human intervention. - -### Scripting & timing - -| Command | Summary | -|---------|---------| -| `bsk evaluate ` | Run JS in agent tab (see red lines); JS throw → stderr, **exit 0** | -| `bsk wait-for-navigation` | Block until load/DOM idle/etc. (`--wait-until`, `--timeout`) | -| `bsk wait-ms ` | Sleep (`500ms`, `2s`, `1m`; **no** `--session`) | - -### Ask the human for help — `bsk request-help` - -When a step needs a human (captcha, login, OTP) or you want the user to -confirm an important action, pause and ask: - - bsk request-help --session --prompt "Solve the captcha, then click Done only after the site accepts it" \ - --title "Captcha required" --target @e7 --target "#submit" --timeout 5m - -- `--prompt` (required): what the user should do. -- `--title` (optional): custom title for the overlay panel. When omitted, - the extension shows its default localized title. -- `--target` (repeatable): a snapshot ref (`@e7`) or CSS selector - (`#submit`) to scroll to and flash-highlight. **Strongly recommended** — - whenever the prompt refers to a concrete element (a button to click, a - field to fill, a checkbox to toggle), pass its `@eN` ref / selector so the - user is guided straight to the right spot instead of hunting for it. For - interaction scenarios, always include the relevant target(s); reserve a - prompt with no `--target` for cases where there is genuinely no specific - element to point at (e.g. "wait for the page to finish loading"). -- `--timeout` (default `5m`): how long to wait. -- `--completion-criteria` (optional): JSON success detector. Use it only - when there is a concrete post-help success signal, e.g. - `{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`. - -The target tab is brought to the foreground; the page stays interactive -while the agent control mask is hidden. The call blocks until the user -explicitly acts, the timeout expires, cancellation arrives, or explicit -completion criteria match. Page reloads, SPA route changes, and captcha -refreshes do not return control by themselves. The result `outcome` is one of: - -- `continued` — the user finished and clicked Done / return control (treat as confirm). -- `cancelled` — the user clicked Cancel (treat as reject/abort). -- `timed_out` — nobody acted within the timeout. -- `completed` — the explicit `--completion-criteria` matched while the user had control. -- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal. - -`note` carries any text the user typed back. `resolved_targets` reports -which refs/selectors matched a live element. - -`request-help` does not refresh the page model after the user returns -control. After a `continued` or `completed` result, issue a separate -observation tool call (usually `bsk snapshot --session `) before using -new refs or reasoning about the post-help page state. -#### Disabling request-help (unattended mode) -Set `BSK_REQUEST_HELP=off` on unattended servers: `bsk request-help` then -returns immediately with `outcome="disabled"` (no overlay, no waiting, -exit 0). Any other value keeps it enabled. If you get `disabled`, do not -retry — complete the task autonomously or stop gracefully. - -### Recording — `bsk record` - -Capture the user's own actions in the Agent Window to a **trace bundle**, for later LLM-driven automation. New CLI builds request **trace v3** (page observations + action chain); older extensions may still return **trace v2** (actions only), which the CLI exports as a single `trace.json` without `states/`. - -```bash -bsk record start --browser \ - [--url https://…] [--purpose "publish a wiki doc"] \ - [--max-page-tokens 3000] [--redact-values] \ - [--output trace] -# `--url` is optional; default https://example.com/ when omitted (must be http(s)). -# Blocks until the user clicks Finish in the recording panel, then writes: -# trace/trace.json — action chain (+ state index when v3) -# trace/states/ — v3 only: one `sN.txt` observe snapshot per settled page state - -bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable +```text +bsk fill --value bsk select --value +bsk screenshot --out bsk emulate --device +bsk upload --file bsk download --out ``` -- **v3 bundle (preferred):** `--output` is a directory (default `./trace`) containing `trace.json` and `states/`. `trace.json` lists `states[]` (page observation ids) and `steps[]` (each step binds `state` = observe snapshot *before* the action and `result.state` = snapshot *after* settle). Page bodies live in `states/sN.txt` using the same VOM format as `bsk observe`. -- **v2 fallback:** when the connected extension is older, export may contain only legacy `trace.json` with `pages[]` action context and no `states/` observation files. Update the extension for full v3 bundles. -- Each `states[]` entry is one **settled page observation** (captured after recording start, navigation landing, or action settle — not on a timer). -- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead. -- `--purpose` is optional context metadata; it does **not** change what gets captured. -- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`. -- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text. +`select` matches an option's `value` attribute, not its visible label. Device preset ids are +lowercase and hyphenated, such as `iphone-14`. -## Error handling +- `console` and `network` provide bounded, read-only debugging evidence. +- `emulate` applies viewport, user-agent, and touch overrides to one tab; new tabs do not inherit + them. Use `--off` to restore the real environment. +- `evaluate` is a last resort when observe plus normal interactions cannot complete the task. With + `--json`, inspect `.ok`: a JavaScript exception may still have CLI exit code 0 because the RPC + succeeded. Never evaluate credential surfaces to read storage, cookies, or auth data. +- `record` captures a user's actions for later replay. Read `bsk record start --help` before use, + and never record banking, SSO, password-manager, or other sensitive pages. -### Exit codes (`echo $?` after `bsk …`) +## File transfer -| Code | Meaning | What to do | -|------|---------|------------| -| `0` | Success (including `evaluate` where JS threw but RPC succeeded) | Continue | -| `1` | User error — bad args, unknown session, tab not in Agent Window, stale ref | Fix args; `bsk session list`; re-snapshot | -| `2` | Protocol / transport — service unreachable, IPC failure | `bsk doctor`; check extension connected; retry the command | -| `3` | Browser / CDP execution failed | Retry; simplify selector; check tab still open | -| `4` | Timeout | Increase `--timeout`; try `--wait-until domcontentloaded` | -| `5` | Version skew (CLI vs extension) | Upgrade/reinstall matching versions | +`upload` and `download` stage files through the daemon; the agent never touches browser-internal +paths. Treat upload as disclosure to the website, download as accepting website-controlled bytes. -Human errors print `error:` + `hint:` on stderr; `--json` includes `code`, `message`, `hint`, `exit_code`. +Upload has two independent mechanisms — choose explicitly, never rely on automatic fallback: -### When to run diagnostics +- **Default (input mode):** for upload buttons, file-input labels, or "upload from computer" + actions. The command clicks the target and intercepts the native file chooser. +- **`--mode drop`:** for reliably identified attachment-receiving areas — an explicit drop zone, + chat composer, email editor, or form attachment area. Do not target page whitespace, generic + containers, or areas whose attachment ownership is ambiguous. -| Situation | Command | -|-----------|---------| -| Before first task in a session | `bsk status` — extension connected? | -| Any failure you cannot fix in one retry | `bsk doctor` | -| Multiple browsers / wrong target | `bsk browsers` then `bsk session start --browser ` | +Decision sequence when uploading: -Always **`bsk session stop `** in a `finally`-style path so the Agent Window closes and borrowed tabs return. +1. Try input mode (the default). +2. If it returns `reason=file_input_not_activated` with `effect_state=none`, re-observe. When a + reliable attachment target exists, try `--mode drop` once against that target. +3. Otherwise fall back to `request-help`. +4. **Never** switch mechanisms or repeat when `effect_state` is `unknown` or `committed` — the + browser may already have applied the file. -## Red lines +A successful drop means Chrome dispatched the native file-drop event; it does not prove the site +accepted the attachment. Observe the page once after the command. -1. **No token theft** — do not `bsk evaluate` on sensitive sites to read `localStorage`, cookies, or auth headers for exfiltration. -2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks. -3. **No skip stop** — always `bsk session stop `; never assume idle timeout will clean up. -4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open. -5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics or conditional surfaces help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual. -6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces. +Download default-refuses to overwrite; pass `--overwrite` when replacing an existing file is +intended. Read `bsk upload --help` and `bsk download --help` for all flags and error details. ---- +## Recover without wandering -**More detail for any command:** `bsk --help` +- Stale ref: observe again and retry the intended action once. +- Unknown tab or session: list current tabs/sessions; never guess identifiers. +- Timeout: inspect current page state before deciding whether one longer purposeful wait is useful. +- Unsupported command: continue with available capabilities; suggest updating only when the missing + command is necessary. +- Unrecoverable failure: report the blocker and stop the session in a finally-style path. + +The CLI's current help and error hints are authoritative for flags, parameters, and recovery +details. From 43067b6bfe653a8f689868891c8713363e77514e Mon Sep 17 00:00:00 2001 From: drakezhang Date: Wed, 2 Sep 2026 17:52:57 +0800 Subject: [PATCH 12/12] style: remove extra blank lines in file-transfer test Fix biome formatter error on CI. Co-authored-by: Cursor --- apps/extension/src/tools/__tests__/file-transfer.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/extension/src/tools/__tests__/file-transfer.test.ts b/apps/extension/src/tools/__tests__/file-transfer.test.ts index ee39d4a..3ef1986 100644 --- a/apps/extension/src/tools/__tests__/file-transfer.test.ts +++ b/apps/extension/src/tools/__tests__/file-transfer.test.ts @@ -296,8 +296,6 @@ describe("file transfer tools", () => { }); }); - - it("marks a timed-out file assignment unknown and detaches browser state", async () => { const fixture = uploadCdp(); const originalSend = fixture.cdp.send;