From 8822bba9a52f8c8ce94ed31d0a83ee4185b7ece7 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Fri, 21 Aug 2026 17:32:13 +0800 Subject: [PATCH] feat(bsk): support drop to upload --- .../extension/e2e/fixtures/file-transfer.html | 81 +++++ .../__tests__/capture-suppress-bridge.test.ts | 12 +- .../src/lib/capture-suppress-bridge.ts | 10 +- .../src/tools/__tests__/file-transfer.test.ts | 284 +++++++++++++++- .../tools/__tests__/frame-geometry.test.ts | 2 + .../src/tools/file-drop-transaction.ts | 319 ++++++++++++++++++ .../src/tools/file-input-transaction.ts | 41 +-- apps/extension/src/tools/frame-geometry.ts | 28 +- apps/extension/src/tools/observation.ts | 6 +- .../src/tools/transfer-transaction.ts | 42 +++ apps/extension/src/tools/upload.ts | 48 ++- apps/extension/src/transport/types.ts | 6 + crates/bsk-cli/skill/SKILL.md | 9 +- crates/bsk-cli/src/cli/mod.rs | 2 +- crates/bsk-cli/src/cli/render_error.rs | 50 +++ crates/bsk-cli/src/cli/upload.rs | 28 +- crates/bsk-cli/tests/cli_parse.rs | 49 +++ .../schema/tool_upload_params.json | 10 + .../bsk-protocol/src/tools/file_transfer.rs | 37 ++ docs/architecture.md | 2 +- skill/SKILL.md | 9 +- 21 files changed, 997 insertions(+), 78 deletions(-) create mode 100644 apps/extension/e2e/fixtures/file-transfer.html create mode 100644 apps/extension/src/tools/file-drop-transaction.ts create mode 100644 apps/extension/src/tools/transfer-transaction.ts diff --git a/apps/extension/e2e/fixtures/file-transfer.html b/apps/extension/e2e/fixtures/file-transfer.html new file mode 100644 index 0000000..47e64f8 --- /dev/null +++ b/apps/extension/e2e/fixtures/file-transfer.html @@ -0,0 +1,81 @@ + + + + + BrowserSkill file-transfer fixture + + + +

Upload then download the same image

+

No image selected

+
+ + + Download uploaded image +
+
Drop an image here
+ + + + diff --git a/apps/extension/src/lib/__tests__/capture-suppress-bridge.test.ts b/apps/extension/src/lib/__tests__/capture-suppress-bridge.test.ts index 1baf524..7e5fe11 100644 --- a/apps/extension/src/lib/__tests__/capture-suppress-bridge.test.ts +++ b/apps/extension/src/lib/__tests__/capture-suppress-bridge.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it, vi } from "vitest"; import { CAPTURE_SUPPRESS, type CaptureSuppressMessage, - withOverlaysHiddenForCapture, + withExtensionOverlayHidden, } from "../capture-suppress-bridge"; function recordingSendToTab(events: string[]) { @@ -12,12 +12,12 @@ function recordingSendToTab(events: string[]) { }); } -describe("withOverlaysHiddenForCapture", () => { +describe("withExtensionOverlayHidden", () => { it("sends begin → fn → end and returns fn's result", async () => { const events: string[] = []; const sendToTab = recordingSendToTab(events); - const result = await withOverlaysHiddenForCapture( + const result = await withExtensionOverlayHidden( 7, async () => { events.push("capture"); @@ -37,7 +37,7 @@ describe("withOverlaysHiddenForCapture", () => { const sendToTab = recordingSendToTab(events); await expect( - withOverlaysHiddenForCapture( + withExtensionOverlayHidden( 7, async () => { events.push("capture"); @@ -55,7 +55,7 @@ describe("withOverlaysHiddenForCapture", () => { }); const fn = vi.fn(async () => "shot"); - const result = await withOverlaysHiddenForCapture(7, fn, sendToTab); + const result = await withExtensionOverlayHidden(7, fn, sendToTab); expect(result).toBe("shot"); expect(fn).toHaveBeenCalledTimes(1); @@ -69,7 +69,7 @@ describe("withOverlaysHiddenForCapture", () => { return { type: CAPTURE_SUPPRESS, ok: true }; }); - const result = await withOverlaysHiddenForCapture(7, async () => "shot", sendToTab); + const result = await withExtensionOverlayHidden(7, async () => "shot", sendToTab); expect(result).toBe("shot"); expect(sendToTab).toHaveBeenCalledTimes(2); diff --git a/apps/extension/src/lib/capture-suppress-bridge.ts b/apps/extension/src/lib/capture-suppress-bridge.ts index 03e05b3..07096f9 100644 --- a/apps/extension/src/lib/capture-suppress-bridge.ts +++ b/apps/extension/src/lib/capture-suppress-bridge.ts @@ -1,9 +1,9 @@ /** - * Wire protocol and background-side helper for hiding the extension's - * in-page overlay while a screenshot is captured, so captured frames only - * contain the page itself. + * Wire protocol and background-side helper for temporarily excluding the + * extension's in-page overlay. Screenshots use it to avoid captured chrome; + * coordinate-driven native input uses it so Chrome hit-tests the page. * - * The background wraps every capture in `withOverlaysHiddenForCapture`, + * The background wraps each bounded operation in `withExtensionOverlayHidden`, * which sends `begin` (the content script hides the overlay host and only * acks once the compositor has produced an overlay-free frame) and always * follows up with `end`, even when the capture throws. Tabs without the @@ -46,7 +46,7 @@ const defaultSendToTab: CaptureSuppressSendToTab = (tabId, message) => * (or any other `begin` failure) simply skips suppression — there is no * overlay to hide in that tab. */ -export async function withOverlaysHiddenForCapture( +export async function withExtensionOverlayHidden( tabId: number, fn: () => Promise, sendToTab: CaptureSuppressSendToTab = defaultSendToTab, diff --git a/apps/extension/src/tools/__tests__/file-transfer.test.ts b/apps/extension/src/tools/__tests__/file-transfer.test.ts index f3f365c..3ef1986 100644 --- a/apps/extension/src/tools/__tests__/file-transfer.test.ts +++ b/apps/extension/src/tools/__tests__/file-transfer.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "vitest"; import { SessionManager } from "@/session-manager/manager"; import { type DownloadsApi, handleDownload } from "../download"; import { captureBrowserDownload } from "../download-capture"; +import { uploadThroughFileDrop } from "../file-drop-transaction"; import { uploadThroughActivatedFileInput } from "../file-input-transaction"; import type { ResolvedActionTarget } from "../interaction"; import type { CdpRunner } from "../shared"; @@ -37,16 +38,20 @@ function fakeEvent unknown>() { }; } -function actionTarget(frameId?: string): ResolvedActionTarget { +function actionTarget(frameId?: string, sessionId?: string): ResolvedActionTarget { return { tab: { tabId: 4, windowId: 100, active: true }, backendNodeId: 123, - cdpTarget: { tabId: 4 }, + cdpTarget: { tabId: 4, ...(sessionId ? { sessionId } : {}) }, ...(frameId ? { frameId } : {}), usedRef: "e3", }; } +function withoutOverlay(operation: () => Promise): Promise { + return operation(); +} + function uploadCdp( options: { inputCount?: number; @@ -59,6 +64,7 @@ function uploadCdp( let cdpEvent: Parameters>[0] | undefined; const send = vi.fn(async (_tabId: number, method: string, params?: object) => { calls.push({ method, params }); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; if (method === "Page.setInterceptFileChooserDialog") return {}; if (method === "Page.getLayoutMetrics") return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; @@ -110,6 +116,50 @@ function uploadCdp( }; } +function dropCdp( + options: { + targetContainsPoint?: boolean; + dispatchError?: (type: string) => Error | Promise | undefined; + onCall?: (method: string) => void; + } = {}, +) { + const calls: Array<{ method: string; params?: object }> = []; + const detach = vi.fn(async () => {}); + const send = vi.fn(async (_tabId: number, method: string, params?: object) => { + calls.push({ method, params }); + options.onCall?.(method); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; + } + if (method === "DOM.getContentQuads") { + return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] }; + } + if (method === "DOM.getNodeForLocation") return { backendNodeId: 124 }; + if (method === "DOM.resolveNode") { + const id = (params as { backendNodeId?: number }).backendNodeId; + return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } }; + } + if (method === "Runtime.callFunctionOn") { + return { result: { value: options.targetContainsPoint ?? true } }; + } + if (method === "Input.dispatchDragEvent") { + const type = (params as { type?: string }).type ?? ""; + const error = options.dispatchError?.(type); + if (error instanceof Promise) return error; + if (error) throw error; + return {}; + } + if (method === "Runtime.releaseObjectGroup") return {}; + throw new Error(`unexpected drop CDP command ${method}`); + }); + const cdp: CdpRunner = { + send: send as unknown as CdpRunner["send"], + detach, + }; + return { calls, cdp, detach }; +} + describe("file transfer tools", () => { it("captures the file input activated by the requested click and injects only staged paths", async () => { const manager = sessions(); @@ -271,6 +321,236 @@ describe("file transfer tools", () => { expect(detach).toHaveBeenCalledWith(4); }); + it("delivers staged files through one explicit native drop without clicking", async () => { + const manager = sessions(); + const ctx = await manager.start("s1"); + ctx.refStore.set("e3", 123, { tabId: 4 }); + const lifecycle: string[] = []; + const { cdp, calls } = dropCdp({ onCall: (method) => lifecycle.push(method) }); + const overlayPhases: string[] = []; + + const result = await handleUpload( + manager, + { + session_id: "s1", + ref: "@e3", + mode: "drop", + files: [{ transfer_id: "tr_1", name: "one.png", staged_path: "/stage/one.png" }], + }, + { + cdp, + tabsApi: tabsApi(), + sendToTab: vi.fn(async (_tabId, message) => { + overlayPhases.push(message.phase); + lifecycle.push(`overlay:${message.phase}`); + }), + }, + ); + + expect(result).toMatchObject({ tab_id: 4, used_ref: "e3", file_names: ["one.png"] }); + expect(calls.filter((call) => call.method === "Input.dispatchDragEvent")).toEqual([ + { + method: "Input.dispatchDragEvent", + params: { + type: "dragEnter", + x: 110, + y: 70, + data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 }, + }, + }, + { + method: "Input.dispatchDragEvent", + params: { + type: "dragOver", + x: 110, + y: 70, + data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 }, + }, + }, + { + method: "Input.dispatchDragEvent", + params: { + type: "drop", + x: 110, + y: 70, + data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 }, + }, + }, + ]); + expect(calls.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false); + expect(overlayPhases).toEqual(["begin", "end"]); + expect(lifecycle.indexOf("DOM.getContentQuads")).toBeLessThan( + lifecycle.indexOf("overlay:begin"), + ); + expect(lifecycle.indexOf("overlay:begin")).toBeLessThan( + lifecycle.indexOf("DOM.getNodeForLocation"), + ); + expect(lifecycle.lastIndexOf("Input.dispatchDragEvent")).toBeLessThan( + lifecycle.indexOf("overlay:end"), + ); + expect(lifecycle.indexOf("overlay:end")).toBeLessThan( + lifecycle.indexOf("Runtime.releaseObjectGroup"), + ); + }); + + it("fails closed before dragging when another element owns the action point", async () => { + const { cdp, calls } = dropCdp({ targetContainsPoint: false }); + const result = await uploadThroughFileDrop({ + cdp, + actionTarget: actionTarget(), + files: ["/stage/one.png"], + timeoutMs: 100, + withOverlayHidden: withoutOverlay, + }); + + expect(result).toMatchObject({ + code: "permission_denied", + data: { + reason: "file_drop_target_unavailable", + mechanism: "drop", + effect_state: "none", + phase: "resolve_target", + }, + }); + expect(calls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false); + }); + + it("dispatches an OOPIF file drop in the child target's local coordinates", async () => { + const rootCalls: Array<{ method: string; params?: object }> = []; + const childCalls: Array<{ method: string; params?: object }> = []; + const cdp: CdpRunner = { + send: vi.fn(async (_tabId, method, params) => { + rootCalls.push({ method, params }); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "DOM.getBoxModel") { + return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } }; + } + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } }; + } + throw new Error(`unexpected root command ${method}`); + }) as CdpRunner["send"], + sendToTarget: vi.fn(async (_target, method, params) => { + childCalls.push({ method, params }); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "DOM.getContentQuads") { + return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] }; + } + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } }; + } + if (method === "DOM.getNodeForLocation") { + return { backendNodeId: 124, frameId: "child" }; + } + if (method === "DOM.resolveNode") { + const id = (params as { backendNodeId?: number }).backendNodeId; + return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } }; + } + if (method === "Runtime.callFunctionOn") return { result: { value: true } }; + if (method === "Input.dispatchDragEvent" || method === "Runtime.releaseObjectGroup") { + return {}; + } + throw new Error(`unexpected child command ${method}`); + }) as CdpRunner["sendToTarget"], + getFrameGraph: vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 10, + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })), + }; + + const result = await uploadThroughFileDrop({ + cdp, + actionTarget: actionTarget("child", "child-session"), + files: ["/stage/one.png"], + timeoutMs: 100, + withOverlayHidden: withoutOverlay, + }); + + expect(result).toMatchObject({ x: 110, y: 70 }); + expect( + childCalls + .filter((call) => call.method === "Input.dispatchDragEvent") + .map((call) => call.params), + ).toEqual([ + expect.objectContaining({ type: "dragEnter", x: 110, y: 70 }), + expect.objectContaining({ type: "dragOver", x: 110, y: 70 }), + expect.objectContaining({ type: "drop", x: 110, y: 70 }), + ]); + expect(rootCalls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false); + }); + + it("reports a missing native drag command as unsupported before commit", async () => { + const { cdp } = dropCdp({ + dispatchError: (type) => + type === "dragEnter" ? new Error("'Input.dispatchDragEvent' wasn't found") : undefined, + }); + const result = await uploadThroughFileDrop({ + cdp, + actionTarget: actionTarget(), + files: ["/stage/one.png"], + timeoutMs: 100, + withOverlayHidden: withoutOverlay, + }); + + expect(result).toMatchObject({ + code: "unsupported", + data: { + reason: "upload_mechanism_unsupported", + effect_state: "none", + phase: "drag_enter", + }, + }); + }); + + it("cancels an unresolved drag before returning a pre-commit timeout", async () => { + const { cdp, calls } = dropCdp({ + dispatchError: (type) => (type === "dragEnter" ? new Promise(() => {}) : undefined), + }); + const result = await uploadThroughFileDrop({ + cdp, + actionTarget: actionTarget(), + files: ["/stage/one.png"], + timeoutMs: 10, + withOverlayHidden: withoutOverlay, + }); + + expect(result).toMatchObject({ + code: "timeout", + data: { reason: "file_drop_failed", effect_state: "none", phase: "drag_enter" }, + }); + expect(calls).toContainEqual({ + method: "Input.dispatchDragEvent", + params: expect.objectContaining({ type: "dragCancel" }), + }); + }); + + it("marks an unresolved drop unknown and detaches browser state", async () => { + const { cdp, detach } = dropCdp({ + dispatchError: (type) => (type === "drop" ? new Promise(() => {}) : undefined), + }); + const result = await uploadThroughFileDrop({ + cdp, + actionTarget: actionTarget(), + files: ["/stage/one.png"], + timeoutMs: 10, + withOverlayHidden: withoutOverlay, + }); + + expect(result).toMatchObject({ + code: "timeout", + data: { reason: "file_drop_failed", effect_state: "unknown", phase: "drop" }, + }); + expect(detach).toHaveBeenCalledWith(4); + }); + it("routes one exact-target download through a browser-relative capability", async () => { const manager = sessions(); const ctx = await manager.start("s1"); diff --git a/apps/extension/src/tools/__tests__/frame-geometry.test.ts b/apps/extension/src/tools/__tests__/frame-geometry.test.ts index c7f82b4..9116f6a 100644 --- a/apps/extension/src/tools/__tests__/frame-geometry.test.ts +++ b/apps/extension/src/tools/__tests__/frame-geometry.test.ts @@ -169,6 +169,7 @@ describe("frame geometry projection", () => { expect(geometry).toMatchObject({ topBounds: { x: 100, y: 100, width: 200, height: 100 }, actionPoint: { x: 200, y: 150 }, + targetActionPoint: { x: 100, y: 50 }, }); }); @@ -261,6 +262,7 @@ describe("frame geometry projection", () => { expect(geometry).toMatchObject({ topBounds: { x: 100, y: 100, width: 200, height: 100 }, actionPoint: { x: 200, y: 150 }, + targetActionPoint: { x: 200, y: 150 }, }); }); diff --git a/apps/extension/src/tools/file-drop-transaction.ts b/apps/extension/src/tools/file-drop-transaction.ts new file mode 100644 index 0000000..f806e84 --- /dev/null +++ b/apps/extension/src/tools/file-drop-transaction.ts @@ -0,0 +1,319 @@ +// One native file-drop transaction bound to an explicit page target. CDP drag +// events are sent to the target that owns the node: root targets use top-level +// coordinates, while OOPIF targets require coordinates local to their session. + +import type { RpcError, TransferEffectState } from "@/transport/types"; +import { transferError } from "./errors"; +import { resolveNodeGeometry } from "./frame-geometry"; +import type { ResolvedActionTarget } from "./interaction"; +import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared"; +import { BoundedWaitError, waitBounded } from "./transfer-transaction"; + +const CLEANUP_TIMEOUT_MS = 1_000; + +type FileDropPhase = "resolve_target" | "drag_enter" | "drag_over" | "drop" | "cleanup"; + +interface RuntimeReply { + result?: { value?: unknown; objectId?: string }; + exceptionDetails?: { text?: string; exception?: { description?: string } }; +} + +interface DragData { + items: []; + files: string[]; + dragOperationsMask: number; +} + +export interface FileDropTransactionOptions { + cdp: CdpRunner; + actionTarget: ResolvedActionTarget; + files: string[]; + timeoutMs: number; + signal?: AbortSignal; + withOverlayHidden: (operation: () => Promise) => Promise; +} + +export interface FileDropTransactionResult { + x: number; + y: number; +} + +function failure( + code: RpcError["code"], + reason: "upload_mechanism_unsupported" | "file_drop_target_unavailable" | "file_drop_failed", + message: string, + effectState: TransferEffectState, + phase: FileDropPhase, +): RpcError { + const error = transferError(code, reason, message, { effectState, phase }); + error.data = { ...error.data, mechanism: "drop" }; + return error; +} + +function enrichFailure( + error: RpcError, + effectState: TransferEffectState, + phase: FileDropPhase, +): RpcError { + return { + ...error, + data: { + ...error.data, + mechanism: "drop", + effect_state: effectState, + phase, + }, + }; +} + +function messageOf(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function methodUnavailable(error: unknown): boolean { + return /(?:method|command).*(?:not found|wasn't found|unsupported)|wasn't found|-32601/i.test( + messageOf(error), + ); +} + +function runtimeError(reply: RuntimeReply, fallback: string): Error | null { + if (!reply.exceptionDetails) return null; + return new Error( + reply.exceptionDetails.exception?.description ?? reply.exceptionDetails.text ?? fallback, + ); +} + +async function pointBelongsToTarget( + options: FileDropTransactionOptions, + x: number, + y: number, + objectGroup: string, +): Promise { + const target = options.actionTarget.cdpTarget; + const hit = await sendToCdpTarget<{ backendNodeId?: number; frameId?: string }>( + options.cdp, + target, + "DOM.getNodeForLocation", + { x: Math.round(x), y: Math.round(y), includeUserAgentShadowDOM: true }, + ); + if (typeof hit.backendNodeId !== "number") return false; + if (options.actionTarget.frameId && hit.frameId !== options.actionTarget.frameId) return false; + + const [resolvedTarget, resolvedHit] = await Promise.all([ + sendToCdpTarget<{ object?: { objectId?: string } }>(options.cdp, target, "DOM.resolveNode", { + backendNodeId: options.actionTarget.backendNodeId, + objectGroup, + }), + sendToCdpTarget<{ object?: { objectId?: string } }>(options.cdp, target, "DOM.resolveNode", { + backendNodeId: hit.backendNodeId, + objectGroup, + }), + ]); + const targetObjectId = resolvedTarget.object?.objectId; + const hitObjectId = resolvedHit.object?.objectId; + if (!targetObjectId || !hitObjectId) return false; + + const reply = await sendToCdpTarget(options.cdp, target, "Runtime.callFunctionOn", { + objectId: targetObjectId, + functionDeclaration: `function(hit) { + for (let node = hit; node; ) { + if (node === this) return true; + if (this.contains?.(node)) return true; + const root = node.getRootNode?.(); + node = node.parentNode || (root && root !== node ? root.host : null); + } + return false; + }`, + arguments: [{ objectId: hitObjectId }], + returnByValue: true, + }); + const error = runtimeError(reply, "failed to verify file-drop target"); + if (error) throw error; + return reply.result?.value === true; +} + +export async function uploadThroughFileDrop( + options: FileDropTransactionOptions, +): Promise { + const deadline = Date.now() + options.timeoutMs; + const target = options.actionTarget.cdpTarget; + const objectGroup = `bsk-file-drop-${crypto.randomUUID()}`; + const dragData: DragData = { + items: [], + files: options.files, + dragOperationsMask: 1, + }; + let dragStarted = false; + let dropSent = false; + let dragPoint = { x: 0, y: 0 }; + let cleanupFailed = false; + let outcome: FileDropTransactionResult | RpcError = failure( + "protocol_error", + "file_drop_failed", + "file-drop transaction ended without an outcome", + "none", + "cleanup", + ); + + try { + const geometry = await waitBounded( + resolveNodeGeometry( + options.cdp, + options.actionTarget.tab.tabId, + { + target, + backendNodeId: options.actionTarget.backendNodeId, + ...(options.actionTarget.frameId ? { frameId: options.actionTarget.frameId } : {}), + }, + { scrollIntoView: true }, + ), + deadline, + options.signal, + "resolving file-drop target timed out", + ); + if (isRpcError(geometry)) { + outcome = enrichFailure(geometry, "none", "resolve_target"); + return outcome; + } + const point = geometry.targetActionPoint; + dragPoint = point; + + outcome = await options.withOverlayHidden(async () => { + try { + try { + const belongs = await waitBounded( + pointBelongsToTarget(options, point.x, point.y, objectGroup), + deadline, + options.signal, + "verifying file-drop target timed out", + ); + if (!belongs) { + return failure( + "permission_denied", + "file_drop_target_unavailable", + "file-drop target is not the topmost element at its action point", + "none", + "resolve_target", + ); + } + } catch (error) { + return failure( + error instanceof BoundedWaitError ? "timeout" : "cdp_failed", + "file_drop_target_unavailable", + messageOf(error), + "none", + "resolve_target", + ); + } + + for (const [type, phase] of [ + ["dragEnter", "drag_enter"], + ["dragOver", "drag_over"], + ] as const) { + try { + if (type === "dragEnter") dragStarted = true; + await waitBounded( + sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", { + type, + x: point.x, + y: point.y, + data: dragData, + }), + deadline, + options.signal, + `${type} timed out`, + ); + } catch (error) { + const unsupported = methodUnavailable(error); + if (type === "dragEnter" && unsupported) dragStarted = false; + return failure( + unsupported + ? "unsupported" + : error instanceof BoundedWaitError + ? "timeout" + : "cdp_failed", + unsupported ? "upload_mechanism_unsupported" : "file_drop_failed", + messageOf(error), + "none", + phase, + ); + } + } + + try { + dropSent = true; + await waitBounded( + sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", { + type: "drop", + x: point.x, + y: point.y, + data: dragData, + }), + deadline, + options.signal, + "file drop timed out", + ); + return { x: point.x, y: point.y }; + } catch (error) { + outcome = failure( + error instanceof BoundedWaitError ? "timeout" : "cdp_failed", + "file_drop_failed", + messageOf(error), + "unknown", + "drop", + ); + return outcome; + } + } finally { + if (dragStarted && !dropSent) { + try { + await waitBounded( + sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", { + type: "dragCancel", + x: dragPoint.x, + y: dragPoint.y, + data: dragData, + }), + Date.now() + CLEANUP_TIMEOUT_MS, + undefined, + "cancelling file drag timed out", + ); + } catch { + cleanupFailed = true; + } + } + } + }); + return outcome; + } catch (error) { + outcome = failure( + error instanceof BoundedWaitError ? "timeout" : "cdp_failed", + "file_drop_target_unavailable", + messageOf(error), + "none", + "resolve_target", + ); + return outcome; + } finally { + try { + await waitBounded( + sendToCdpTarget(options.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }), + Date.now() + CLEANUP_TIMEOUT_MS, + undefined, + "releasing file-drop object group timed out", + ); + } catch { + cleanupFailed = true; + } + + const effect = isRpcError(outcome) + ? (outcome.data?.effect_state as TransferEffectState | undefined) + : "committed"; + if (effect === "unknown" || cleanupFailed) { + await options.cdp.detach?.(target.tabId); + } + if (cleanupFailed && isRpcError(outcome)) { + outcome.data = { ...outcome.data, cleanup_state: "failed" }; + } + } +} diff --git a/apps/extension/src/tools/file-input-transaction.ts b/apps/extension/src/tools/file-input-transaction.ts index 3c78a8d..c37dce2 100644 --- a/apps/extension/src/tools/file-input-transaction.ts +++ b/apps/extension/src/tools/file-input-transaction.ts @@ -7,6 +7,7 @@ import type { ClickResult, RpcError, TransferEffectState } from "@/transport/typ import { transferError } from "./errors"; import type { ResolvedActionTarget } from "./interaction"; import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared"; +import { BoundedWaitError, remainingMs, waitBounded } from "./transfer-transaction"; const CLEANUP_TIMEOUT_MS = 1_000; const ACTIVATION_GRACE_MS = 1_000; @@ -45,46 +46,6 @@ export interface FileInputTransactionResult { multiple: boolean; } -class BoundedWaitError extends Error { - constructor( - readonly kind: "timeout" | "aborted", - message: string, - ) { - super(message); - } -} - -function remainingMs(deadline: number): number { - return Math.max(0, deadline - Date.now()); -} - -async function waitBounded( - promise: Promise, - deadline: number, - signal: AbortSignal | undefined, - timeoutMessage: string, -): Promise { - const remaining = remainingMs(deadline); - if (signal?.aborted) throw new BoundedWaitError("aborted", "upload transaction aborted"); - if (remaining === 0) throw new BoundedWaitError("timeout", timeoutMessage); - - let timer: ReturnType | undefined; - let onAbort: (() => void) | undefined; - const boundary = new Promise((_resolve, reject) => { - timer = setTimeout(() => reject(new BoundedWaitError("timeout", timeoutMessage)), remaining); - if (signal) { - onAbort = () => reject(new BoundedWaitError("aborted", "upload transaction aborted")); - signal.addEventListener("abort", onAbort, { once: true }); - } - }); - try { - return await Promise.race([promise, boundary]); - } finally { - if (timer) clearTimeout(timer); - if (signal && onAbort) signal.removeEventListener("abort", onAbort); - } -} - function runtimeError(reply: RuntimeReply, fallback: string): Error | null { if (!reply.exceptionDetails) return null; return new Error( diff --git a/apps/extension/src/tools/frame-geometry.ts b/apps/extension/src/tools/frame-geometry.ts index ba2be42..0c788c9 100644 --- a/apps/extension/src/tools/frame-geometry.ts +++ b/apps/extension/src/tools/frame-geometry.ts @@ -29,7 +29,10 @@ export interface NodeAddress { export interface ResolvedNodeGeometry { topVisibleRegions: Region; topBounds: ViewportRect; + /** Point in the top-level tab viewport, used by root-target input events. */ actionPoint: Point; + /** Point in the addressed CDP target's viewport, used by OOPIF-local input events. */ + targetActionPoint: Point; } function geometryError(message: string): RpcError { @@ -262,11 +265,30 @@ export async function resolveNodeGeometry( if (isRpcError(localRegion)) return localRegion; let topVisibleRegions: Region; + let targetActionPoint: Point | null = null; if (address.frameId && graph) { const projection = await resolveFrameProjection(cdp, graph, address.frameId); if (!projection) return geometryError(`could not resolve frame geometry for ${address.frameId}`); topVisibleRegions = projectRegionToViewport(localRegion, projection); + if (address.target.sessionId) { + const localViewport = projection.edges[0]?.sourceViewport ?? projection.topViewport; + const localVisibleRegions = localRegion + .map((polygon) => + clipPolygon( + polygon, + rectPolygon({ + x: 0, + y: 0, + w: localViewport.width, + h: localViewport.height, + }), + ), + ) + .filter((polygon) => polygon.length >= 3); + const localActionRegion = largestRegion(localVisibleRegions); + targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null; + } } else { const viewport = await targetViewport(cdp, address.target); if (!viewport) return geometryError("could not resolve top viewport geometry"); @@ -283,7 +305,11 @@ export async function resolveNodeGeometry( if (!topBounds || !actionPoint) { return { code: "permission_denied", message: "element not visible" }; } - return { topVisibleRegions, topBounds, actionPoint }; + if (address.target.sessionId && !targetActionPoint) { + return { code: "permission_denied", message: "element not visible in its target" }; + } + targetActionPoint ??= actionPoint; + return { topVisibleRegions, topBounds, actionPoint, targetActionPoint }; } catch (error) { return geometryError(error instanceof Error ? error.message : String(error)); } diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index f25d97b..98b6cb0 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -17,7 +17,7 @@ import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; import type { CdpTarget } from "@/browser-driver/frame-graph"; import { type CaptureSuppressSendToTab, - withOverlaysHiddenForCapture, + withExtensionOverlayHidden, } from "@/lib/capture-suppress-bridge"; import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { @@ -312,7 +312,7 @@ export async function handleScreenshot( tabId: target.tabId, ...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}), }; - const captured = await withOverlaysHiddenForCapture( + const captured = await withExtensionOverlayHidden( target.tabId, () => captureElementScreenshot( @@ -344,7 +344,7 @@ export async function handleScreenshot( ); } - const captured = await withOverlaysHiddenForCapture( + const captured = await withExtensionOverlayHidden( target.tabId, () => captureFullTabPng(deps, ctx, target, signal), deps.sendToTab, diff --git a/apps/extension/src/tools/transfer-transaction.ts b/apps/extension/src/tools/transfer-transaction.ts new file mode 100644 index 0000000..6debaeb --- /dev/null +++ b/apps/extension/src/tools/transfer-transaction.ts @@ -0,0 +1,42 @@ +// Shared time boundary for browser-side file-transfer transactions. Concrete +// upload/download mechanisms own their state machines and effect semantics. + +export class BoundedWaitError extends Error { + constructor( + readonly kind: "timeout" | "aborted", + message: string, + ) { + super(message); + } +} + +export function remainingMs(deadline: number): number { + return Math.max(0, deadline - Date.now()); +} + +export async function waitBounded( + promise: Promise, + deadline: number, + signal: AbortSignal | undefined, + timeoutMessage: string, +): Promise { + const remaining = remainingMs(deadline); + if (signal?.aborted) throw new BoundedWaitError("aborted", "file transfer aborted"); + if (remaining === 0) throw new BoundedWaitError("timeout", timeoutMessage); + + let timer: ReturnType | undefined; + let onAbort: (() => void) | undefined; + const boundary = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new BoundedWaitError("timeout", timeoutMessage)), remaining); + if (signal) { + onAbort = () => reject(new BoundedWaitError("aborted", "file transfer aborted")); + signal.addEventListener("abort", onAbort, { once: true }); + } + }); + try { + return await Promise.race([promise, boundary]); + } finally { + if (timer) clearTimeout(timer); + if (signal && onAbort) signal.removeEventListener("abort", onAbort); + } +} diff --git a/apps/extension/src/tools/upload.ts b/apps/extension/src/tools/upload.ts index 05a43ea..9203dd2 100644 --- a/apps/extension/src/tools/upload.ts +++ b/apps/extension/src/tools/upload.ts @@ -2,8 +2,13 @@ // target, then delegate the browser protocol transaction to the file-input // transaction module. +import { + type CaptureSuppressSendToTab, + withExtensionOverlayHidden, +} from "@/lib/capture-suppress-bridge"; import type { SessionManager } from "@/session-manager/manager"; import type { RpcError, UploadParams, UploadResult } from "@/transport/types"; +import { uploadThroughFileDrop } from "./file-drop-transaction"; import { uploadThroughActivatedFileInput } from "./file-input-transaction"; import { clickResolvedTarget, type InteractionDeps, resolveActionTarget } from "./interaction"; import { @@ -18,6 +23,7 @@ const DEFAULT_TIMEOUT_MS = 120_000; export interface UploadDeps extends InteractionDeps { cdp: CdpRunner; + sendToTab?: CaptureSuppressSendToTab; } export async function handleUpload( @@ -41,19 +47,39 @@ export async function handleUpload( const address = await resolveActionTarget(deps.cdp, ctx, target, params, "upload"); if (isRpcError(address)) return address; const timeoutMs = params.timeout_ms ?? DEFAULT_TIMEOUT_MS; - const transaction = await uploadThroughActivatedFileInput({ - cdp: deps.cdp, - actionTarget: address, - files: params.files.map((file) => file.staged_path as string), - timeoutMs, - signal: deps.signal, - trigger: () => clickResolvedTarget(ctx, address, {}, deps), - }); - if (isRpcError(transaction)) return transaction; + const files = params.files.map((file) => file.staged_path as string); + const mode = params.mode ?? "input"; + let usedRef = address.usedRef; + let usedSelector = address.usedSelector; + if (mode === "drop") { + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const transaction = await uploadThroughFileDrop({ + cdp: deps.cdp, + actionTarget: address, + files, + timeoutMs, + signal: deps.signal, + withOverlayHidden: (operation) => + withExtensionOverlayHidden(target.tabId, operation, deps.sendToTab), + }); + if (isRpcError(transaction)) return transaction; + } else { + const transaction = await uploadThroughActivatedFileInput({ + cdp: deps.cdp, + actionTarget: address, + files, + timeoutMs, + signal: deps.signal, + trigger: () => clickResolvedTarget(ctx, address, {}, deps), + }); + if (isRpcError(transaction)) return transaction; + usedRef = transaction.click.used_ref; + usedSelector = transaction.click.used_selector; + } return { tab_id: target.tabId, - used_ref: transaction.click.used_ref, - used_selector: transaction.click.used_selector, + used_ref: usedRef, + used_selector: usedSelector, file_names: params.files.map((file) => file.name), }; } diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index 7143cb1..f2a01ca 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -36,6 +36,9 @@ export type RpcErrorReason = | "file_input_probe_failed" | "file_input_not_activated" | "set_file_input_failed" + | "upload_mechanism_unsupported" + | "file_drop_target_unavailable" + | "file_drop_failed" | "download_capture_failed" | "transfer_outcome_unknown" | "transfer_timeout" @@ -518,12 +521,15 @@ export interface UploadFile { staged_path?: string; } +export type UploadMode = "input" | "drop"; + export interface UploadParams { session_id: string; ref?: string; selector?: string; tab_id?: number; files: UploadFile[]; + mode?: UploadMode; timeout_ms?: number; } diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index 8af5383..bb8dce3 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -210,18 +210,23 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff | Command | Summary | |---------|---------| -| `bsk upload --file ` | Click one upload trigger and attach an agent-readable local file (`--file` is repeatable) | +| `bsk upload --file ` | Attach a local file through a file-input trigger; use `--mode drop` only for an explicit drop zone (`--file` is repeatable) | | `bsk download --out ` | Click one download trigger and copy the single completed file to an exact local path (`--overwrite` is opt-in) | The agent/harness decides whether a file transfer is appropriate and which local path belongs to the task. Treat upload as disclosure of that file to the current website, and download as accepting website-controlled bytes onto the local filesystem. Use only paths that are necessary for the user's bounded goal. -BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted capabilities reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. Upload intercepts the native chooser for one transaction, locates the input activated in the resolved target's document, and assigns only the staged file paths. Download uniquely correlates one exact-target browser intent with one Chrome download in either event order, routes it through a daemon-minted relative directory, and lets the daemon validate and import it. Upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive. +BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted capabilities reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. The default upload mode intercepts one chooser transaction, locates the file input activated in the resolved target's document, and assigns only the staged paths. `--mode drop` does not click: it delivers the same staged files through one native browser drop bound to the explicitly resolved drop zone. These mechanisms never fall back to each other automatically. Download uniquely correlates one exact-target browser intent with one Chrome download in either event order, routes it through a daemon-minted relative directory, and lets the daemon validate and import it. Upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive. + +A successful drop command means Chrome dispatched the native file drop to the resolved zone; it does not prove that the site's application logic accepted the attachment. Observe the page once after the command, and do not repeat the drop when the attachment is already present. Do not use `request-help` merely because a native file chooser or browser download is involved; try these commands first. For transfer failures, use the structured error instead of retrying blindly: - `reason=file_input_not_activated` means the requested click did not activate exactly one ``; the page may use a non-input picker such as `window.showOpenFilePicker()`. Do not retry blindly. If human help is available, call `request-help` and tell the user the exact original local path to choose. The staged daemon path is internal and must not be shown to the user. - `reason=file_input_probe_failed` means BrowserSkill could not safely establish the browser-side upload transaction. Do not repeat the same action; use `request-help` when available. - `reason=set_file_input_failed` means BrowserSkill found the activated file input but Chrome rejected the staged path or assignment. Check the extension's file-URL access permission; otherwise use `request-help`. +- `reason=upload_mechanism_unsupported` means the browser cannot perform a native file drop. Do not retry the same drop; use a standard file-input trigger or `request-help`. +- `reason=file_drop_target_unavailable` means the specified point is no longer owned by the resolved drop zone. When `effect_state=none`, re-observe and select the visible drop zone itself; do not guess another target. +- `reason=file_drop_failed` means the native drop command failed. Do not retry when `effect_state=unknown`; first observe whether the page already received the file. - `reason=download_capture_failed` means BrowserSkill could not attribute exactly one completed download to the requested target. Do not retry blindly or accept an unrelated browser download; use `request-help` when available. - `effect_state=none` means BrowserSkill confirmed that no file-transfer effect was committed. Follow the accompanying reason; a corrected target or explicit human fallback may be attempted. - `effect_state=unknown` means the browser may already have attached or created the file. Do not repeat the transfer. Observe the page if that can establish the result; otherwise stop and report the uncertainty. diff --git a/crates/bsk-cli/src/cli/mod.rs b/crates/bsk-cli/src/cli/mod.rs index 4776ebe..8140045 100644 --- a/crates/bsk-cli/src/cli/mod.rs +++ b/crates/bsk-cli/src/cli/mod.rs @@ -182,7 +182,7 @@ pub enum Command { /// Set ``; the page may use a non-input picker such as `window.showOpenFilePicker()`. Do not retry blindly. If human help is available, call `request-help` and tell the user the exact original local path to choose. The staged daemon path is internal and must not be shown to the user. - `reason=file_input_probe_failed` means BrowserSkill could not safely establish the browser-side upload transaction. Do not repeat the same action; use `request-help` when available. - `reason=set_file_input_failed` means BrowserSkill found the activated file input but Chrome rejected the staged path or assignment. Check the extension's file-URL access permission; otherwise use `request-help`. +- `reason=upload_mechanism_unsupported` means the browser cannot perform a native file drop. Do not retry the same drop; use a standard file-input trigger or `request-help`. +- `reason=file_drop_target_unavailable` means the specified point is no longer owned by the resolved drop zone. When `effect_state=none`, re-observe and select the visible drop zone itself; do not guess another target. +- `reason=file_drop_failed` means the native drop command failed. Do not retry when `effect_state=unknown`; first observe whether the page already received the file. - `reason=download_capture_failed` means BrowserSkill could not attribute exactly one completed download to the requested target. Do not retry blindly or accept an unrelated browser download; use `request-help` when available. - `effect_state=none` means BrowserSkill confirmed that no file-transfer effect was committed. Follow the accompanying reason; a corrected target or explicit human fallback may be attempted. - `effect_state=unknown` means the browser may already have attached or created the file. Do not repeat the transfer. Observe the page if that can establish the result; otherwise stop and report the uncertainty.