diff --git a/apps/extension/e2e/fixtures/file-transfer.html b/apps/extension/e2e/fixtures/file-transfer.html new file mode 100644 index 0000000..47e64f8 --- /dev/null +++ b/apps/extension/e2e/fixtures/file-transfer.html @@ -0,0 +1,81 @@ + + + + + BrowserSkill file-transfer fixture + + + +

Upload then download the same image

+

No image selected

+
+ + + Download uploaded image +
+
Drop an image here
+ + + + diff --git a/apps/extension/src/lib/__tests__/capture-suppress-bridge.test.ts b/apps/extension/src/lib/__tests__/capture-suppress-bridge.test.ts index 1baf524..7e5fe11 100644 --- a/apps/extension/src/lib/__tests__/capture-suppress-bridge.test.ts +++ b/apps/extension/src/lib/__tests__/capture-suppress-bridge.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it, vi } from "vitest"; import { CAPTURE_SUPPRESS, type CaptureSuppressMessage, - withOverlaysHiddenForCapture, + withExtensionOverlayHidden, } from "../capture-suppress-bridge"; function recordingSendToTab(events: string[]) { @@ -12,12 +12,12 @@ function recordingSendToTab(events: string[]) { }); } -describe("withOverlaysHiddenForCapture", () => { +describe("withExtensionOverlayHidden", () => { it("sends begin → fn → end and returns fn's result", async () => { const events: string[] = []; const sendToTab = recordingSendToTab(events); - const result = await withOverlaysHiddenForCapture( + const result = await withExtensionOverlayHidden( 7, async () => { events.push("capture"); @@ -37,7 +37,7 @@ describe("withOverlaysHiddenForCapture", () => { const sendToTab = recordingSendToTab(events); await expect( - withOverlaysHiddenForCapture( + withExtensionOverlayHidden( 7, async () => { events.push("capture"); @@ -55,7 +55,7 @@ describe("withOverlaysHiddenForCapture", () => { }); const fn = vi.fn(async () => "shot"); - const result = await withOverlaysHiddenForCapture(7, fn, sendToTab); + const result = await withExtensionOverlayHidden(7, fn, sendToTab); expect(result).toBe("shot"); expect(fn).toHaveBeenCalledTimes(1); @@ -69,7 +69,7 @@ describe("withOverlaysHiddenForCapture", () => { return { type: CAPTURE_SUPPRESS, ok: true }; }); - const result = await withOverlaysHiddenForCapture(7, async () => "shot", sendToTab); + const result = await withExtensionOverlayHidden(7, async () => "shot", sendToTab); expect(result).toBe("shot"); expect(sendToTab).toHaveBeenCalledTimes(2); diff --git a/apps/extension/src/lib/capture-suppress-bridge.ts b/apps/extension/src/lib/capture-suppress-bridge.ts index 03e05b3..07096f9 100644 --- a/apps/extension/src/lib/capture-suppress-bridge.ts +++ b/apps/extension/src/lib/capture-suppress-bridge.ts @@ -1,9 +1,9 @@ /** - * Wire protocol and background-side helper for hiding the extension's - * in-page overlay while a screenshot is captured, so captured frames only - * contain the page itself. + * Wire protocol and background-side helper for temporarily excluding the + * extension's in-page overlay. Screenshots use it to avoid captured chrome; + * coordinate-driven native input uses it so Chrome hit-tests the page. * - * The background wraps every capture in `withOverlaysHiddenForCapture`, + * The background wraps each bounded operation in `withExtensionOverlayHidden`, * which sends `begin` (the content script hides the overlay host and only * acks once the compositor has produced an overlay-free frame) and always * follows up with `end`, even when the capture throws. Tabs without the @@ -46,7 +46,7 @@ const defaultSendToTab: CaptureSuppressSendToTab = (tabId, message) => * (or any other `begin` failure) simply skips suppression — there is no * overlay to hide in that tab. */ -export async function withOverlaysHiddenForCapture( +export async function withExtensionOverlayHidden( tabId: number, fn: () => Promise, sendToTab: CaptureSuppressSendToTab = defaultSendToTab, diff --git a/apps/extension/src/tools/__tests__/file-transfer.test.ts b/apps/extension/src/tools/__tests__/file-transfer.test.ts index d1d2751..3ef1986 100644 --- a/apps/extension/src/tools/__tests__/file-transfer.test.ts +++ b/apps/extension/src/tools/__tests__/file-transfer.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "vitest"; import { SessionManager } from "@/session-manager/manager"; import { type DownloadsApi, handleDownload } from "../download"; import { captureBrowserDownload } from "../download-capture"; +import { uploadThroughFileDrop } from "../file-drop-transaction"; import { uploadThroughActivatedFileInput } from "../file-input-transaction"; import type { ResolvedActionTarget } from "../interaction"; import type { CdpRunner } from "../shared"; @@ -37,16 +38,20 @@ function fakeEvent unknown>() { }; } -function actionTarget(frameId?: string): ResolvedActionTarget { +function actionTarget(frameId?: string, sessionId?: string): ResolvedActionTarget { return { tab: { tabId: 4, windowId: 100, active: true }, backendNodeId: 123, - cdpTarget: { tabId: 4 }, + cdpTarget: { tabId: 4, ...(sessionId ? { sessionId } : {}) }, ...(frameId ? { frameId } : {}), usedRef: "e3", }; } +function withoutOverlay(operation: () => Promise): Promise { + return operation(); +} + function uploadCdp( options: { inputCount?: number; @@ -59,6 +64,7 @@ function uploadCdp( let cdpEvent: Parameters>[0] | undefined; const send = vi.fn(async (_tabId: number, method: string, params?: object) => { calls.push({ method, params }); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; if (method === "Page.setInterceptFileChooserDialog") return {}; if (method === "Page.getLayoutMetrics") return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; @@ -110,6 +116,50 @@ function uploadCdp( }; } +function dropCdp( + options: { + targetContainsPoint?: boolean; + dispatchError?: (type: string) => Error | Promise | undefined; + onCall?: (method: string) => void; + } = {}, +) { + const calls: Array<{ method: string; params?: object }> = []; + const detach = vi.fn(async () => {}); + const send = vi.fn(async (_tabId: number, method: string, params?: object) => { + calls.push({ method, params }); + options.onCall?.(method); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; + } + if (method === "DOM.getContentQuads") { + return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] }; + } + if (method === "DOM.getNodeForLocation") return { backendNodeId: 124 }; + if (method === "DOM.resolveNode") { + const id = (params as { backendNodeId?: number }).backendNodeId; + return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } }; + } + if (method === "Runtime.callFunctionOn") { + return { result: { value: options.targetContainsPoint ?? true } }; + } + if (method === "Input.dispatchDragEvent") { + const type = (params as { type?: string }).type ?? ""; + const error = options.dispatchError?.(type); + if (error instanceof Promise) return error; + if (error) throw error; + return {}; + } + if (method === "Runtime.releaseObjectGroup") return {}; + throw new Error(`unexpected drop CDP command ${method}`); + }); + const cdp: CdpRunner = { + send: send as unknown as CdpRunner["send"], + detach, + }; + return { calls, cdp, detach }; +} + describe("file transfer tools", () => { it("captures the file input activated by the requested click and injects only staged paths", async () => { const manager = sessions(); @@ -246,104 +296,6 @@ describe("file transfer tools", () => { }); }); - it("does not dispatch browser work when upload is already cancelled", async () => { - const controller = new AbortController(); - controller.abort(); - const { cdp, calls } = uploadCdp(); - - const result = await uploadThroughActivatedFileInput({ - cdp, - actionTarget: actionTarget(), - files: ["/private/stage/one"], - timeoutMs: 100, - signal: controller.signal, - trigger: vi.fn(), - }); - - expect(result).toMatchObject({ - code: "cancelled", - data: { effect_state: "none", phase: "arm_interception" }, - }); - expect(calls).not.toContainEqual({ - method: "Page.setInterceptFileChooserDialog", - params: { enabled: true }, - }); - expect(calls.some((call) => call.method === "DOM.setFileInputFiles")).toBe(false); - }); - - it("stops before file assignment when cancellation interrupts input resolution", async () => { - const controller = new AbortController(); - const fixture = uploadCdp(); - const originalSend = fixture.cdp.send; - let markProbeStarted!: () => void; - const probeStarted = new Promise((resolve) => { - markProbeStarted = resolve; - }); - fixture.cdp.send = vi.fn((tabId: number, method: string, params?: object) => { - const declaration = (params as { functionDeclaration?: string } | undefined) - ?.functionDeclaration; - if ( - method === "Runtime.callFunctionOn" && - declaration?.includes("count: state.inputs.length") - ) { - markProbeStarted(); - return new Promise(() => {}); - } - return originalSend(tabId, method, params); - }) as CdpRunner["send"]; - - const transaction = uploadThroughActivatedFileInput({ - cdp: fixture.cdp, - actionTarget: actionTarget(), - files: ["/private/stage/one"], - timeoutMs: 1_000, - signal: controller.signal, - trigger: async () => ({ tab_id: 4, x: 10, y: 10 }), - }); - await probeStarted; - controller.abort(); - const result = await transaction; - - expect(result).toMatchObject({ - code: "cancelled", - data: { effect_state: "none", phase: "resolve_input" }, - }); - expect(fixture.calls.some((call) => call.method === "DOM.setFileInputFiles")).toBe(false); - }); - - it("disarms chooser interception when its enable response times out", async () => { - const calls: Array<{ method: string; params?: object }> = []; - const cdp: CdpRunner = { - send: vi.fn(async (_tabId: number, method: string, params?: object) => { - calls.push({ method, params }); - if ( - method === "Page.setInterceptFileChooserDialog" && - (params as { enabled?: boolean }).enabled - ) { - return new Promise(() => {}); - } - return {}; - }) as CdpRunner["send"], - }; - - const result = await uploadThroughActivatedFileInput({ - cdp, - actionTarget: actionTarget(), - files: ["/private/stage/one"], - timeoutMs: 5, - trigger: vi.fn(), - }); - - expect(result).toMatchObject({ - code: "timeout", - data: { effect_state: "none", phase: "arm_interception" }, - }); - expect(calls).toContainEqual({ - method: "Page.setInterceptFileChooserDialog", - params: { enabled: false, cancel: true }, - }); - }); - it("marks a timed-out file assignment unknown and detaches browser state", async () => { const fixture = uploadCdp(); const originalSend = fixture.cdp.send; @@ -369,6 +321,236 @@ describe("file transfer tools", () => { expect(detach).toHaveBeenCalledWith(4); }); + it("delivers staged files through one explicit native drop without clicking", async () => { + const manager = sessions(); + const ctx = await manager.start("s1"); + ctx.refStore.set("e3", 123, { tabId: 4 }); + const lifecycle: string[] = []; + const { cdp, calls } = dropCdp({ onCall: (method) => lifecycle.push(method) }); + const overlayPhases: string[] = []; + + const result = await handleUpload( + manager, + { + session_id: "s1", + ref: "@e3", + mode: "drop", + files: [{ transfer_id: "tr_1", name: "one.png", staged_path: "/stage/one.png" }], + }, + { + cdp, + tabsApi: tabsApi(), + sendToTab: vi.fn(async (_tabId, message) => { + overlayPhases.push(message.phase); + lifecycle.push(`overlay:${message.phase}`); + }), + }, + ); + + expect(result).toMatchObject({ tab_id: 4, used_ref: "e3", file_names: ["one.png"] }); + expect(calls.filter((call) => call.method === "Input.dispatchDragEvent")).toEqual([ + { + method: "Input.dispatchDragEvent", + params: { + type: "dragEnter", + x: 110, + y: 70, + data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 }, + }, + }, + { + method: "Input.dispatchDragEvent", + params: { + type: "dragOver", + x: 110, + y: 70, + data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 }, + }, + }, + { + method: "Input.dispatchDragEvent", + params: { + type: "drop", + x: 110, + y: 70, + data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 }, + }, + }, + ]); + expect(calls.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false); + expect(overlayPhases).toEqual(["begin", "end"]); + expect(lifecycle.indexOf("DOM.getContentQuads")).toBeLessThan( + lifecycle.indexOf("overlay:begin"), + ); + expect(lifecycle.indexOf("overlay:begin")).toBeLessThan( + lifecycle.indexOf("DOM.getNodeForLocation"), + ); + expect(lifecycle.lastIndexOf("Input.dispatchDragEvent")).toBeLessThan( + lifecycle.indexOf("overlay:end"), + ); + expect(lifecycle.indexOf("overlay:end")).toBeLessThan( + lifecycle.indexOf("Runtime.releaseObjectGroup"), + ); + }); + + it("fails closed before dragging when another element owns the action point", async () => { + const { cdp, calls } = dropCdp({ targetContainsPoint: false }); + const result = await uploadThroughFileDrop({ + cdp, + actionTarget: actionTarget(), + files: ["/stage/one.png"], + timeoutMs: 100, + withOverlayHidden: withoutOverlay, + }); + + expect(result).toMatchObject({ + code: "permission_denied", + data: { + reason: "file_drop_target_unavailable", + mechanism: "drop", + effect_state: "none", + phase: "resolve_target", + }, + }); + expect(calls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false); + }); + + it("dispatches an OOPIF file drop in the child target's local coordinates", async () => { + const rootCalls: Array<{ method: string; params?: object }> = []; + const childCalls: Array<{ method: string; params?: object }> = []; + const cdp: CdpRunner = { + send: vi.fn(async (_tabId, method, params) => { + rootCalls.push({ method, params }); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "DOM.getBoxModel") { + return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } }; + } + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } }; + } + throw new Error(`unexpected root command ${method}`); + }) as CdpRunner["send"], + sendToTarget: vi.fn(async (_target, method, params) => { + childCalls.push({ method, params }); + if (method === "DOM.scrollIntoViewIfNeeded") return {}; + if (method === "DOM.getContentQuads") { + return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] }; + } + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } }; + } + if (method === "DOM.getNodeForLocation") { + return { backendNodeId: 124, frameId: "child" }; + } + if (method === "DOM.resolveNode") { + const id = (params as { backendNodeId?: number }).backendNodeId; + return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } }; + } + if (method === "Runtime.callFunctionOn") return { result: { value: true } }; + if (method === "Input.dispatchDragEvent" || method === "Runtime.releaseObjectGroup") { + return {}; + } + throw new Error(`unexpected child command ${method}`); + }) as CdpRunner["sendToTarget"], + getFrameGraph: vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 10, + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })), + }; + + const result = await uploadThroughFileDrop({ + cdp, + actionTarget: actionTarget("child", "child-session"), + files: ["/stage/one.png"], + timeoutMs: 100, + withOverlayHidden: withoutOverlay, + }); + + expect(result).toMatchObject({ x: 110, y: 70 }); + expect( + childCalls + .filter((call) => call.method === "Input.dispatchDragEvent") + .map((call) => call.params), + ).toEqual([ + expect.objectContaining({ type: "dragEnter", x: 110, y: 70 }), + expect.objectContaining({ type: "dragOver", x: 110, y: 70 }), + expect.objectContaining({ type: "drop", x: 110, y: 70 }), + ]); + expect(rootCalls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false); + }); + + it("reports a missing native drag command as unsupported before commit", async () => { + const { cdp } = dropCdp({ + dispatchError: (type) => + type === "dragEnter" ? new Error("'Input.dispatchDragEvent' wasn't found") : undefined, + }); + const result = await uploadThroughFileDrop({ + cdp, + actionTarget: actionTarget(), + files: ["/stage/one.png"], + timeoutMs: 100, + withOverlayHidden: withoutOverlay, + }); + + expect(result).toMatchObject({ + code: "unsupported", + data: { + reason: "upload_mechanism_unsupported", + effect_state: "none", + phase: "drag_enter", + }, + }); + }); + + it("cancels an unresolved drag before returning a pre-commit timeout", async () => { + const { cdp, calls } = dropCdp({ + dispatchError: (type) => (type === "dragEnter" ? new Promise(() => {}) : undefined), + }); + const result = await uploadThroughFileDrop({ + cdp, + actionTarget: actionTarget(), + files: ["/stage/one.png"], + timeoutMs: 10, + withOverlayHidden: withoutOverlay, + }); + + expect(result).toMatchObject({ + code: "timeout", + data: { reason: "file_drop_failed", effect_state: "none", phase: "drag_enter" }, + }); + expect(calls).toContainEqual({ + method: "Input.dispatchDragEvent", + params: expect.objectContaining({ type: "dragCancel" }), + }); + }); + + it("marks an unresolved drop unknown and detaches browser state", async () => { + const { cdp, detach } = dropCdp({ + dispatchError: (type) => (type === "drop" ? new Promise(() => {}) : undefined), + }); + const result = await uploadThroughFileDrop({ + cdp, + actionTarget: actionTarget(), + files: ["/stage/one.png"], + timeoutMs: 10, + withOverlayHidden: withoutOverlay, + }); + + expect(result).toMatchObject({ + code: "timeout", + data: { reason: "file_drop_failed", effect_state: "unknown", phase: "drop" }, + }); + expect(detach).toHaveBeenCalledWith(4); + }); + it("routes one exact-target download through a browser-relative capability", async () => { const manager = sessions(); const ctx = await manager.start("s1"); @@ -665,10 +847,7 @@ describe("file transfer tools", () => { expect(downloads.cancel).not.toHaveBeenCalled(); }); - it.each([ - ["cancel resolves", false], - ["cancel reports that completion won", true], - ])("reconciles a download that completes while %s", async (_label, cancelThrows) => { + it("reconciles a download that completes while cancellation is being requested", async () => { const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>(); const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>(); const onDeterminingFilename = @@ -702,7 +881,7 @@ describe("file transfer tools", () => { onDeterminingFilename, search: vi.fn().mockResolvedValueOnce([initial]).mockResolvedValueOnce([complete]), cancel: vi.fn(async () => { - if (cancelThrows) throw new Error("download already complete"); + throw new Error("download already complete"); }), removeFile: vi.fn(async () => {}), }; diff --git a/apps/extension/src/tools/__tests__/frame-geometry.test.ts b/apps/extension/src/tools/__tests__/frame-geometry.test.ts index c7f82b4..9116f6a 100644 --- a/apps/extension/src/tools/__tests__/frame-geometry.test.ts +++ b/apps/extension/src/tools/__tests__/frame-geometry.test.ts @@ -169,6 +169,7 @@ describe("frame geometry projection", () => { expect(geometry).toMatchObject({ topBounds: { x: 100, y: 100, width: 200, height: 100 }, actionPoint: { x: 200, y: 150 }, + targetActionPoint: { x: 100, y: 50 }, }); }); @@ -261,6 +262,7 @@ describe("frame geometry projection", () => { expect(geometry).toMatchObject({ topBounds: { x: 100, y: 100, width: 200, height: 100 }, actionPoint: { x: 200, y: 150 }, + targetActionPoint: { x: 200, y: 150 }, }); }); diff --git a/apps/extension/src/tools/download-capture.ts b/apps/extension/src/tools/download-capture.ts index 411655e..6e75bff 100644 --- a/apps/extension/src/tools/download-capture.ts +++ b/apps/extension/src/tools/download-capture.ts @@ -9,8 +9,6 @@ import { type CdpRunner, isRpcError } from "./shared"; const CORRELATION_GRACE_MS = 750; const UNIQUE_SETTLE_MS = 50; -const CLEANUP_RECONCILE_MS = 1_000; -const CLEANUP_POLL_MS = 25; const SIZE_POLL_MS = 250; type DeterminingFilenameListener = ( @@ -112,35 +110,26 @@ function captureError( async function cleanupClaimedDownload(downloads: DownloadsApi, downloadId: number): Promise { const lookup = async () => (await downloads.search({ id: downloadId }))[0]; - let item = await lookup(); + const item = await lookup(); if (!item || item.state === "interrupted") return; if (item.state === "complete") { await downloads.removeFile(downloadId); return; } - let cancelError: unknown; try { await downloads.cancel(downloadId); - } catch (error) { - cancelError = error; - } - - const deadline = Date.now() + CLEANUP_RECONCILE_MS; - while (true) { - item = await lookup(); - if (!item || item.state === "interrupted") return; - if (item.state === "complete") { + } catch (cancelError) { + // Completion can win the race after the lookup but before cancellation. + // Reconcile against Chrome's authoritative state before declaring cleanup + // failed so every terminal state has one explicit cleanup path. + const reconciled = await lookup(); + if (!reconciled || reconciled.state === "interrupted") return; + if (reconciled.state === "complete") { await downloads.removeFile(downloadId); return; } - if (Date.now() >= deadline) { - if (cancelError instanceof Error) throw cancelError; - throw new Error("download remained in progress after cancellation"); - } - await new Promise((resolve) => - setTimeout(resolve, Math.min(CLEANUP_POLL_MS, Math.max(1, deadline - Date.now()))), - ); + throw cancelError; } } diff --git a/apps/extension/src/tools/file-drop-transaction.ts b/apps/extension/src/tools/file-drop-transaction.ts new file mode 100644 index 0000000..f806e84 --- /dev/null +++ b/apps/extension/src/tools/file-drop-transaction.ts @@ -0,0 +1,319 @@ +// One native file-drop transaction bound to an explicit page target. CDP drag +// events are sent to the target that owns the node: root targets use top-level +// coordinates, while OOPIF targets require coordinates local to their session. + +import type { RpcError, TransferEffectState } from "@/transport/types"; +import { transferError } from "./errors"; +import { resolveNodeGeometry } from "./frame-geometry"; +import type { ResolvedActionTarget } from "./interaction"; +import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared"; +import { BoundedWaitError, waitBounded } from "./transfer-transaction"; + +const CLEANUP_TIMEOUT_MS = 1_000; + +type FileDropPhase = "resolve_target" | "drag_enter" | "drag_over" | "drop" | "cleanup"; + +interface RuntimeReply { + result?: { value?: unknown; objectId?: string }; + exceptionDetails?: { text?: string; exception?: { description?: string } }; +} + +interface DragData { + items: []; + files: string[]; + dragOperationsMask: number; +} + +export interface FileDropTransactionOptions { + cdp: CdpRunner; + actionTarget: ResolvedActionTarget; + files: string[]; + timeoutMs: number; + signal?: AbortSignal; + withOverlayHidden: (operation: () => Promise) => Promise; +} + +export interface FileDropTransactionResult { + x: number; + y: number; +} + +function failure( + code: RpcError["code"], + reason: "upload_mechanism_unsupported" | "file_drop_target_unavailable" | "file_drop_failed", + message: string, + effectState: TransferEffectState, + phase: FileDropPhase, +): RpcError { + const error = transferError(code, reason, message, { effectState, phase }); + error.data = { ...error.data, mechanism: "drop" }; + return error; +} + +function enrichFailure( + error: RpcError, + effectState: TransferEffectState, + phase: FileDropPhase, +): RpcError { + return { + ...error, + data: { + ...error.data, + mechanism: "drop", + effect_state: effectState, + phase, + }, + }; +} + +function messageOf(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function methodUnavailable(error: unknown): boolean { + return /(?:method|command).*(?:not found|wasn't found|unsupported)|wasn't found|-32601/i.test( + messageOf(error), + ); +} + +function runtimeError(reply: RuntimeReply, fallback: string): Error | null { + if (!reply.exceptionDetails) return null; + return new Error( + reply.exceptionDetails.exception?.description ?? reply.exceptionDetails.text ?? fallback, + ); +} + +async function pointBelongsToTarget( + options: FileDropTransactionOptions, + x: number, + y: number, + objectGroup: string, +): Promise { + const target = options.actionTarget.cdpTarget; + const hit = await sendToCdpTarget<{ backendNodeId?: number; frameId?: string }>( + options.cdp, + target, + "DOM.getNodeForLocation", + { x: Math.round(x), y: Math.round(y), includeUserAgentShadowDOM: true }, + ); + if (typeof hit.backendNodeId !== "number") return false; + if (options.actionTarget.frameId && hit.frameId !== options.actionTarget.frameId) return false; + + const [resolvedTarget, resolvedHit] = await Promise.all([ + sendToCdpTarget<{ object?: { objectId?: string } }>(options.cdp, target, "DOM.resolveNode", { + backendNodeId: options.actionTarget.backendNodeId, + objectGroup, + }), + sendToCdpTarget<{ object?: { objectId?: string } }>(options.cdp, target, "DOM.resolveNode", { + backendNodeId: hit.backendNodeId, + objectGroup, + }), + ]); + const targetObjectId = resolvedTarget.object?.objectId; + const hitObjectId = resolvedHit.object?.objectId; + if (!targetObjectId || !hitObjectId) return false; + + const reply = await sendToCdpTarget(options.cdp, target, "Runtime.callFunctionOn", { + objectId: targetObjectId, + functionDeclaration: `function(hit) { + for (let node = hit; node; ) { + if (node === this) return true; + if (this.contains?.(node)) return true; + const root = node.getRootNode?.(); + node = node.parentNode || (root && root !== node ? root.host : null); + } + return false; + }`, + arguments: [{ objectId: hitObjectId }], + returnByValue: true, + }); + const error = runtimeError(reply, "failed to verify file-drop target"); + if (error) throw error; + return reply.result?.value === true; +} + +export async function uploadThroughFileDrop( + options: FileDropTransactionOptions, +): Promise { + const deadline = Date.now() + options.timeoutMs; + const target = options.actionTarget.cdpTarget; + const objectGroup = `bsk-file-drop-${crypto.randomUUID()}`; + const dragData: DragData = { + items: [], + files: options.files, + dragOperationsMask: 1, + }; + let dragStarted = false; + let dropSent = false; + let dragPoint = { x: 0, y: 0 }; + let cleanupFailed = false; + let outcome: FileDropTransactionResult | RpcError = failure( + "protocol_error", + "file_drop_failed", + "file-drop transaction ended without an outcome", + "none", + "cleanup", + ); + + try { + const geometry = await waitBounded( + resolveNodeGeometry( + options.cdp, + options.actionTarget.tab.tabId, + { + target, + backendNodeId: options.actionTarget.backendNodeId, + ...(options.actionTarget.frameId ? { frameId: options.actionTarget.frameId } : {}), + }, + { scrollIntoView: true }, + ), + deadline, + options.signal, + "resolving file-drop target timed out", + ); + if (isRpcError(geometry)) { + outcome = enrichFailure(geometry, "none", "resolve_target"); + return outcome; + } + const point = geometry.targetActionPoint; + dragPoint = point; + + outcome = await options.withOverlayHidden(async () => { + try { + try { + const belongs = await waitBounded( + pointBelongsToTarget(options, point.x, point.y, objectGroup), + deadline, + options.signal, + "verifying file-drop target timed out", + ); + if (!belongs) { + return failure( + "permission_denied", + "file_drop_target_unavailable", + "file-drop target is not the topmost element at its action point", + "none", + "resolve_target", + ); + } + } catch (error) { + return failure( + error instanceof BoundedWaitError ? "timeout" : "cdp_failed", + "file_drop_target_unavailable", + messageOf(error), + "none", + "resolve_target", + ); + } + + for (const [type, phase] of [ + ["dragEnter", "drag_enter"], + ["dragOver", "drag_over"], + ] as const) { + try { + if (type === "dragEnter") dragStarted = true; + await waitBounded( + sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", { + type, + x: point.x, + y: point.y, + data: dragData, + }), + deadline, + options.signal, + `${type} timed out`, + ); + } catch (error) { + const unsupported = methodUnavailable(error); + if (type === "dragEnter" && unsupported) dragStarted = false; + return failure( + unsupported + ? "unsupported" + : error instanceof BoundedWaitError + ? "timeout" + : "cdp_failed", + unsupported ? "upload_mechanism_unsupported" : "file_drop_failed", + messageOf(error), + "none", + phase, + ); + } + } + + try { + dropSent = true; + await waitBounded( + sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", { + type: "drop", + x: point.x, + y: point.y, + data: dragData, + }), + deadline, + options.signal, + "file drop timed out", + ); + return { x: point.x, y: point.y }; + } catch (error) { + outcome = failure( + error instanceof BoundedWaitError ? "timeout" : "cdp_failed", + "file_drop_failed", + messageOf(error), + "unknown", + "drop", + ); + return outcome; + } + } finally { + if (dragStarted && !dropSent) { + try { + await waitBounded( + sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", { + type: "dragCancel", + x: dragPoint.x, + y: dragPoint.y, + data: dragData, + }), + Date.now() + CLEANUP_TIMEOUT_MS, + undefined, + "cancelling file drag timed out", + ); + } catch { + cleanupFailed = true; + } + } + } + }); + return outcome; + } catch (error) { + outcome = failure( + error instanceof BoundedWaitError ? "timeout" : "cdp_failed", + "file_drop_target_unavailable", + messageOf(error), + "none", + "resolve_target", + ); + return outcome; + } finally { + try { + await waitBounded( + sendToCdpTarget(options.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }), + Date.now() + CLEANUP_TIMEOUT_MS, + undefined, + "releasing file-drop object group timed out", + ); + } catch { + cleanupFailed = true; + } + + const effect = isRpcError(outcome) + ? (outcome.data?.effect_state as TransferEffectState | undefined) + : "committed"; + if (effect === "unknown" || cleanupFailed) { + await options.cdp.detach?.(target.tabId); + } + if (cleanupFailed && isRpcError(outcome)) { + outcome.data = { ...outcome.data, cleanup_state: "failed" }; + } + } +} diff --git a/apps/extension/src/tools/file-input-transaction.ts b/apps/extension/src/tools/file-input-transaction.ts index 6e69b8b..c37dce2 100644 --- a/apps/extension/src/tools/file-input-transaction.ts +++ b/apps/extension/src/tools/file-input-transaction.ts @@ -7,9 +7,9 @@ import type { ClickResult, RpcError, TransferEffectState } from "@/transport/typ import { transferError } from "./errors"; import type { ResolvedActionTarget } from "./interaction"; import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared"; +import { BoundedWaitError, remainingMs, waitBounded } from "./transfer-transaction"; const CLEANUP_TIMEOUT_MS = 1_000; -const DETACH_TIMEOUT_MS = 500; const ACTIVATION_GRACE_MS = 1_000; const PROBE_INTERVAL_MS = 20; @@ -46,51 +46,6 @@ export interface FileInputTransactionResult { multiple: boolean; } -class BoundedWaitError extends Error { - constructor( - readonly kind: "timeout" | "aborted", - message: string, - ) { - super(message); - } -} - -function remainingMs(deadline: number): number { - return Math.max(0, deadline - Date.now()); -} - -async function waitBounded( - operation: () => Promise, - deadline: number, - signal: AbortSignal | undefined, - timeoutMessage: string, -): Promise { - const remaining = remainingMs(deadline); - if (signal?.aborted) throw new BoundedWaitError("aborted", "upload transaction aborted"); - if (remaining === 0) throw new BoundedWaitError("timeout", timeoutMessage); - - let timer: ReturnType | undefined; - let onAbort: (() => void) | undefined; - const boundary = new Promise((_resolve, reject) => { - timer = setTimeout(() => reject(new BoundedWaitError("timeout", timeoutMessage)), remaining); - if (signal) { - onAbort = () => reject(new BoundedWaitError("aborted", "upload transaction aborted")); - signal.addEventListener("abort", onAbort, { once: true }); - } - }); - try { - const work = Promise.resolve().then(() => { - if (signal?.aborted) throw new BoundedWaitError("aborted", "upload transaction aborted"); - if (remainingMs(deadline) === 0) throw new BoundedWaitError("timeout", timeoutMessage); - return operation(); - }); - return await Promise.race([work, boundary]); - } finally { - if (timer) clearTimeout(timer); - if (signal && onAbort) signal.removeEventListener("abort", onAbort); - } -} - function runtimeError(reply: RuntimeReply, fallback: string): Error | null { if (!reply.exceptionDetails) return null; return new Error( @@ -128,18 +83,13 @@ function sameTarget( async function delay(ms: number, signal?: AbortSignal): Promise { await waitBounded( - () => new Promise((resolve) => setTimeout(resolve, ms)), + new Promise((resolve) => setTimeout(resolve, ms)), Date.now() + ms + 1, signal, "upload activation probe timed out", ); } -function waitErrorCode(error: unknown): RpcError["code"] { - if (!(error instanceof BoundedWaitError)) return "cdp_failed"; - return error.kind === "aborted" ? "cancelled" : "timeout"; -} - async function callOnTrigger( options: FileInputTransactionOptions, objectId: string, @@ -164,8 +114,8 @@ export async function uploadThroughActivatedFileInput( const objectGroup = `bsk-upload-${crypto.randomUUID()}`; const stateKey = `__bskUpload_${crypto.randomUUID().replaceAll("-", "")}`; const chooserEvents: ChooserEvent[] = []; - let interceptionState: "not_started" | "possibly_active" | "active" = "not_started"; - let probeState: "not_started" | "possibly_active" | "active" = "not_started"; + let interceptionArmed = false; + let probeArmed = false; let triggerObjectId: string | undefined; let outcome: FileInputTransactionResult | RpcError = transferFailure( "protocol_error", @@ -183,20 +133,18 @@ export async function uploadThroughActivatedFileInput( try { try { - interceptionState = "possibly_active"; await waitBounded( - () => - sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", { - enabled: true, - }), + sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", { + enabled: true, + }), deadline, options.signal, "arming native file chooser interception timed out", ); - interceptionState = "active"; + interceptionArmed = true; } catch (err) { outcome = transferFailure( - waitErrorCode(err), + err instanceof BoundedWaitError ? "timeout" : "cdp_failed", "file_input_probe_failed", err instanceof Error ? err.message : String(err), "none", @@ -207,16 +155,15 @@ export async function uploadThroughActivatedFileInput( try { const resolved = await waitBounded( - () => - sendToCdpTarget<{ object?: { objectId?: string } }>( - options.cdp, - target, - "DOM.resolveNode", - { - backendNodeId: options.actionTarget.backendNodeId, - objectGroup, - }, - ), + sendToCdpTarget<{ object?: { objectId?: string } }>( + options.cdp, + target, + "DOM.resolveNode", + { + backendNodeId: options.actionTarget.backendNodeId, + objectGroup, + }, + ), deadline, options.signal, "resolving upload trigger timed out", @@ -224,13 +171,11 @@ export async function uploadThroughActivatedFileInput( triggerObjectId = resolved.object?.objectId; if (!triggerObjectId) throw new Error("DOM.resolveNode returned no trigger objectId"); - probeState = "possibly_active"; const armed = await waitBounded( - () => - callOnTrigger( - options, - triggerObjectId as string, - `function(key) { + callOnTrigger( + options, + triggerObjectId, + `function(key) { const doc = this.ownerDocument; const owner = doc.defaultView; if (!owner) return false; @@ -248,9 +193,9 @@ export async function uploadThroughActivatedFileInput( doc.addEventListener("click", state.listener, true); return true; }`, - [stateKey], - true, - ), + [stateKey], + true, + ), deadline, options.signal, "arming frame-scoped file input probe timed out", @@ -259,10 +204,10 @@ export async function uploadThroughActivatedFileInput( if (armError || armed.result?.value !== true) { throw armError ?? new Error("file input probe did not arm"); } - probeState = "active"; + probeArmed = true; } catch (err) { outcome = transferFailure( - waitErrorCode(err), + err instanceof BoundedWaitError ? "timeout" : "cdp_failed", "file_input_probe_failed", err instanceof Error ? err.message : String(err), "none", @@ -274,14 +219,14 @@ export async function uploadThroughActivatedFileInput( let click: ClickResult | RpcError; try { click = await waitBounded( - options.trigger, + options.trigger(), deadline, options.signal, "upload trigger timed out", ); } catch (err) { outcome = transferFailure( - waitErrorCode(err), + err instanceof BoundedWaitError ? "timeout" : "cdp_failed", "file_input_probe_failed", err instanceof Error ? err.message : String(err), "none", @@ -299,23 +244,17 @@ export async function uploadThroughActivatedFileInput( let summary: { count: number; multiple: boolean } = { count: 0, multiple: false }; while (Date.now() < activationDeadline) { if (chooserEvents.length > 0) break; - const reply = await waitBounded( - () => - callOnTrigger( - options, - triggerObjectId as string, - `function(key) { + const reply = await callOnTrigger( + options, + triggerObjectId, + `function(key) { const state = this.ownerDocument.defaultView?.[key]; return state ? { count: state.inputs.length, multiple: state.inputs[0]?.multiple === true } : { count: 0, multiple: false }; }`, - [stateKey], - true, - ), - activationDeadline, - options.signal, - "inspecting activated file input timed out", + [stateKey], + true, ); const summaryError = runtimeError(reply, "failed to inspect activated file input"); if (summaryError) throw summaryError; @@ -378,33 +317,21 @@ export async function uploadThroughActivatedFileInput( ); return outcome; } - const input = await waitBounded( - () => - callOnTrigger( - options, - triggerObjectId as string, - `function(key) { return this.ownerDocument.defaultView?.[key]?.inputs[0]; }`, - [stateKey], - false, - ), - deadline, - options.signal, - "resolving activated file input timed out", + const input = await callOnTrigger( + options, + triggerObjectId, + `function(key) { return this.ownerDocument.defaultView?.[key]?.inputs[0]; }`, + [stateKey], + false, ); const inputError = runtimeError(input, "failed to resolve activated file input object"); if (inputError) throw inputError; if (!input.result?.objectId) throw new Error("activated file input returned no objectId"); - const described = await waitBounded( - () => - sendToCdpTarget<{ node?: { backendNodeId?: number } }>( - options.cdp, - target, - "DOM.describeNode", - { objectId: input.result?.objectId }, - ), - deadline, - options.signal, - "describing activated file input timed out", + const described = await sendToCdpTarget<{ node?: { backendNodeId?: number } }>( + options.cdp, + target, + "DOM.describeNode", + { objectId: input.result.objectId }, ); backendNodeId = described.node?.backendNodeId; if (typeof backendNodeId !== "number") { @@ -423,18 +350,17 @@ export async function uploadThroughActivatedFileInput( try { await waitBounded( - () => - sendToCdpTarget(options.cdp, target, "DOM.setFileInputFiles", { - files: options.files, - backendNodeId, - }), + sendToCdpTarget(options.cdp, target, "DOM.setFileInputFiles", { + files: options.files, + backendNodeId, + }), deadline, options.signal, "setting file input files timed out", ); } catch (err) { outcome = transferFailure( - waitErrorCode(err), + err instanceof BoundedWaitError ? "timeout" : "cdp_failed", "set_file_input_failed", err instanceof Error ? err.message : String(err), "unknown", @@ -446,7 +372,7 @@ export async function uploadThroughActivatedFileInput( return outcome; } catch (err) { outcome = transferFailure( - waitErrorCode(err), + err instanceof BoundedWaitError ? "timeout" : "cdp_failed", "file_input_probe_failed", err instanceof Error ? err.message : String(err), "none", @@ -456,70 +382,61 @@ export async function uploadThroughActivatedFileInput( } } finally { chooserSubscription?.dispose(); - const cleanupDeadline = Date.now() + CLEANUP_TIMEOUT_MS; - const cleanupTasks: Promise[] = []; - const cleanup = async (operation: () => Promise, message: string) => { + let cleanupFailed = false; + if (probeArmed && triggerObjectId) { try { - await waitBounded(operation, cleanupDeadline, undefined, message); - return true; - } catch { - return false; - } - }; - if (probeState !== "not_started" && triggerObjectId) { - cleanupTasks.push( - cleanup( - () => - callOnTrigger( - options, - triggerObjectId as string, - `function(key) { + await waitBounded( + callOnTrigger( + options, + triggerObjectId, + `function(key) { const owner = this.ownerDocument.defaultView; const state = owner?.[key]; if (state?.listener) this.ownerDocument.removeEventListener("click", state.listener, true); if (owner) delete owner[key]; }`, - [stateKey], - true, - ), + [stateKey], + true, + ), + Date.now() + CLEANUP_TIMEOUT_MS, + undefined, "cleaning file input probe timed out", - ), - ); + ); + } catch { + cleanupFailed = true; + } } - if (interceptionState !== "not_started") { - cleanupTasks.push( - cleanup( - () => - sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", { - enabled: false, - cancel: true, - }), + if (interceptionArmed) { + try { + await waitBounded( + sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", { + enabled: false, + cancel: true, + }), + Date.now() + CLEANUP_TIMEOUT_MS, + undefined, "disabling file chooser interception timed out", - ), - ); + ); + } catch { + cleanupFailed = true; + } } - let cleanupFailed = (await Promise.all(cleanupTasks)).some((completed) => !completed); - if (!cleanupFailed) { - cleanupFailed = !(await cleanup( - () => sendToCdpTarget(options.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }), + try { + await waitBounded( + sendToCdpTarget(options.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }), + Date.now() + CLEANUP_TIMEOUT_MS, + undefined, "releasing upload object group timed out", - )); + ); + } catch { + cleanupFailed = true; } const effect = isRpcError(outcome) ? (outcome.data?.effect_state as TransferEffectState | undefined) : "committed"; if (effect === "unknown" || cleanupFailed) { - try { - await waitBounded( - () => options.cdp.detach?.(target.tabId) ?? Promise.resolve(), - Date.now() + DETACH_TIMEOUT_MS, - undefined, - "detaching upload target timed out", - ); - } catch { - cleanupFailed = true; - } + await options.cdp.detach?.(target.tabId); } if (cleanupFailed && isRpcError(outcome)) { outcome.data = { ...outcome.data, cleanup_state: "failed" }; diff --git a/apps/extension/src/tools/frame-geometry.ts b/apps/extension/src/tools/frame-geometry.ts index ba2be42..0c788c9 100644 --- a/apps/extension/src/tools/frame-geometry.ts +++ b/apps/extension/src/tools/frame-geometry.ts @@ -29,7 +29,10 @@ export interface NodeAddress { export interface ResolvedNodeGeometry { topVisibleRegions: Region; topBounds: ViewportRect; + /** Point in the top-level tab viewport, used by root-target input events. */ actionPoint: Point; + /** Point in the addressed CDP target's viewport, used by OOPIF-local input events. */ + targetActionPoint: Point; } function geometryError(message: string): RpcError { @@ -262,11 +265,30 @@ export async function resolveNodeGeometry( if (isRpcError(localRegion)) return localRegion; let topVisibleRegions: Region; + let targetActionPoint: Point | null = null; if (address.frameId && graph) { const projection = await resolveFrameProjection(cdp, graph, address.frameId); if (!projection) return geometryError(`could not resolve frame geometry for ${address.frameId}`); topVisibleRegions = projectRegionToViewport(localRegion, projection); + if (address.target.sessionId) { + const localViewport = projection.edges[0]?.sourceViewport ?? projection.topViewport; + const localVisibleRegions = localRegion + .map((polygon) => + clipPolygon( + polygon, + rectPolygon({ + x: 0, + y: 0, + w: localViewport.width, + h: localViewport.height, + }), + ), + ) + .filter((polygon) => polygon.length >= 3); + const localActionRegion = largestRegion(localVisibleRegions); + targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null; + } } else { const viewport = await targetViewport(cdp, address.target); if (!viewport) return geometryError("could not resolve top viewport geometry"); @@ -283,7 +305,11 @@ export async function resolveNodeGeometry( if (!topBounds || !actionPoint) { return { code: "permission_denied", message: "element not visible" }; } - return { topVisibleRegions, topBounds, actionPoint }; + if (address.target.sessionId && !targetActionPoint) { + return { code: "permission_denied", message: "element not visible in its target" }; + } + targetActionPoint ??= actionPoint; + return { topVisibleRegions, topBounds, actionPoint, targetActionPoint }; } catch (error) { return geometryError(error instanceof Error ? error.message : String(error)); } diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index bae4969..9c944ab 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -17,7 +17,7 @@ import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; import type { CdpTarget } from "@/browser-driver/frame-graph"; import { type CaptureSuppressSendToTab, - withOverlaysHiddenForCapture, + withExtensionOverlayHidden, } from "@/lib/capture-suppress-bridge"; import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { @@ -324,7 +324,7 @@ export async function handleScreenshot( tabId: target.tabId, ...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}), }; - const captured = await withOverlaysHiddenForCapture( + const captured = await withExtensionOverlayHidden( target.tabId, () => captureElementScreenshot( @@ -356,7 +356,7 @@ export async function handleScreenshot( ); } - const captured = await withOverlaysHiddenForCapture( + const captured = await withExtensionOverlayHidden( target.tabId, () => captureFullTabPng(deps, ctx, target, signal), deps.sendToTab, diff --git a/apps/extension/src/tools/transfer-transaction.ts b/apps/extension/src/tools/transfer-transaction.ts new file mode 100644 index 0000000..6debaeb --- /dev/null +++ b/apps/extension/src/tools/transfer-transaction.ts @@ -0,0 +1,42 @@ +// Shared time boundary for browser-side file-transfer transactions. Concrete +// upload/download mechanisms own their state machines and effect semantics. + +export class BoundedWaitError extends Error { + constructor( + readonly kind: "timeout" | "aborted", + message: string, + ) { + super(message); + } +} + +export function remainingMs(deadline: number): number { + return Math.max(0, deadline - Date.now()); +} + +export async function waitBounded( + promise: Promise, + deadline: number, + signal: AbortSignal | undefined, + timeoutMessage: string, +): Promise { + const remaining = remainingMs(deadline); + if (signal?.aborted) throw new BoundedWaitError("aborted", "file transfer aborted"); + if (remaining === 0) throw new BoundedWaitError("timeout", timeoutMessage); + + let timer: ReturnType | undefined; + let onAbort: (() => void) | undefined; + const boundary = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new BoundedWaitError("timeout", timeoutMessage)), remaining); + if (signal) { + onAbort = () => reject(new BoundedWaitError("aborted", "file transfer aborted")); + signal.addEventListener("abort", onAbort, { once: true }); + } + }); + try { + return await Promise.race([promise, boundary]); + } finally { + if (timer) clearTimeout(timer); + if (signal && onAbort) signal.removeEventListener("abort", onAbort); + } +} diff --git a/apps/extension/src/tools/upload.ts b/apps/extension/src/tools/upload.ts index 05a43ea..9203dd2 100644 --- a/apps/extension/src/tools/upload.ts +++ b/apps/extension/src/tools/upload.ts @@ -2,8 +2,13 @@ // target, then delegate the browser protocol transaction to the file-input // transaction module. +import { + type CaptureSuppressSendToTab, + withExtensionOverlayHidden, +} from "@/lib/capture-suppress-bridge"; import type { SessionManager } from "@/session-manager/manager"; import type { RpcError, UploadParams, UploadResult } from "@/transport/types"; +import { uploadThroughFileDrop } from "./file-drop-transaction"; import { uploadThroughActivatedFileInput } from "./file-input-transaction"; import { clickResolvedTarget, type InteractionDeps, resolveActionTarget } from "./interaction"; import { @@ -18,6 +23,7 @@ const DEFAULT_TIMEOUT_MS = 120_000; export interface UploadDeps extends InteractionDeps { cdp: CdpRunner; + sendToTab?: CaptureSuppressSendToTab; } export async function handleUpload( @@ -41,19 +47,39 @@ export async function handleUpload( const address = await resolveActionTarget(deps.cdp, ctx, target, params, "upload"); if (isRpcError(address)) return address; const timeoutMs = params.timeout_ms ?? DEFAULT_TIMEOUT_MS; - const transaction = await uploadThroughActivatedFileInput({ - cdp: deps.cdp, - actionTarget: address, - files: params.files.map((file) => file.staged_path as string), - timeoutMs, - signal: deps.signal, - trigger: () => clickResolvedTarget(ctx, address, {}, deps), - }); - if (isRpcError(transaction)) return transaction; + const files = params.files.map((file) => file.staged_path as string); + const mode = params.mode ?? "input"; + let usedRef = address.usedRef; + let usedSelector = address.usedSelector; + if (mode === "drop") { + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const transaction = await uploadThroughFileDrop({ + cdp: deps.cdp, + actionTarget: address, + files, + timeoutMs, + signal: deps.signal, + withOverlayHidden: (operation) => + withExtensionOverlayHidden(target.tabId, operation, deps.sendToTab), + }); + if (isRpcError(transaction)) return transaction; + } else { + const transaction = await uploadThroughActivatedFileInput({ + cdp: deps.cdp, + actionTarget: address, + files, + timeoutMs, + signal: deps.signal, + trigger: () => clickResolvedTarget(ctx, address, {}, deps), + }); + if (isRpcError(transaction)) return transaction; + usedRef = transaction.click.used_ref; + usedSelector = transaction.click.used_selector; + } return { tab_id: target.tabId, - used_ref: transaction.click.used_ref, - used_selector: transaction.click.used_selector, + used_ref: usedRef, + used_selector: usedSelector, file_names: params.files.map((file) => file.name), }; } diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index 1d9c619..fc9968a 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -36,6 +36,9 @@ export type RpcErrorReason = | "file_input_probe_failed" | "file_input_not_activated" | "set_file_input_failed" + | "upload_mechanism_unsupported" + | "file_drop_target_unavailable" + | "file_drop_failed" | "download_capture_failed" | "transfer_outcome_unknown" | "transfer_timeout" @@ -523,12 +526,15 @@ export interface UploadFile { staged_path?: string; } +export type UploadMode = "input" | "drop"; + export interface UploadParams { session_id: string; ref?: string; selector?: string; tab_id?: number; files: UploadFile[]; + mode?: UploadMode; timeout_ms?: number; } diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index 89f447d..357625c 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -116,156 +116,15 @@ navigate navigate-back navigate-forward reload wait-for-navigation wai observe snapshot get-html screenshot console network click hover fill select press evaluate tab list|create|close|select|borrow|return window resize emulate -request-help record start|stop +upload download request-help record start|stop ``` Required flags that are easy to get wrong: -### Tabs (require `--session `) - -| Command | Summary | -|---------|---------| -| `bsk tab list` | List tabs (`--scope user\|agent\|all`, default `all`) | -| `bsk tab create` | New tab in Agent Window (`--url`, `--no-active`, `--index`) | -| `bsk tab close ` | Close an agent tab | -| `bsk tab select ` | Focus an agent tab | -| `bsk tab borrow ` | Move a user tab into the Agent Window | -| `bsk tab return ` | Return a borrowed tab to its original window | - -### Observation (require `--session` unless noted) - -| Command | Summary | -|---------|---------| -| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs | -| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces | -| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) | -| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) | - -### Console & network debugging (read-only; require `--session`) - -| Command | Summary | -|---------|---------| -| `bsk console` | Buffered page console messages, JS exceptions, and browser log entries (`--include-stack` for stack traces) | -| `bsk network` | Buffered network responses (status, method, URL, MIME/resource type) and failures (`net::ERR_*` reason) | - -Both capture from the moment the tab is attached and read a bounded per-tab buffer: `--since ` pages from a cursor (`next_since` in the result), `--limit` (default 50, max 200), `--max-text-chars` (default 1000, max 4096), `--tab-id` to target a non-active tab. Both are strictly read-only — they never intercept or modify traffic, and request/response headers, bodies, and timings are not captured. - -### Navigation - -| Command | Summary | -|---------|---------| -| `bsk navigate ` | Go to URL in agent tab (`--wait-until`, `--timeout`) | -| `bsk navigate-back` | History back one step | -| `bsk navigate-forward` | History forward one step | -| `bsk reload` | Reload current tab (`--hard` bypass cache) | - -(`bsk navigate back` / `bsk navigate forward` are equivalent subcommands.) - -### Interaction - -| Command | Summary | -|---------|---------| -| `bsk click ` | Click element (`--button`, `--click-count`, `--modifiers`) | -| `bsk hover ` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) | -| `bsk fill --value ` | Clear and type into input | -| `bsk select --value ` | Set ``; the page may use a non-input picker such as `window.showOpenFilePicker()`. Do not retry blindly. If human help is available, call `request-help` and tell the user the exact original local path to choose. The staged daemon path is internal and must not be shown to the user. -- `reason=file_input_probe_failed` means BrowserSkill could not safely establish the browser-side upload transaction. Do not repeat the same action; use `request-help` when available. -- `reason=set_file_input_failed` means BrowserSkill found the activated file input but Chrome rejected the staged path or assignment. Check the extension's file-URL access permission; otherwise use `request-help`. -- `reason=download_capture_failed` means BrowserSkill could not attribute exactly one completed download to the requested target. Do not retry blindly or accept an unrelated browser download; use `request-help` when available. -- `effect_state=none` means BrowserSkill confirmed that no file-transfer effect was committed. Follow the accompanying reason; a corrected target or explicit human fallback may be attempted. -- `effect_state=unknown` means the browser may already have attached or created the file. Do not repeat the transfer. Observe the page if that can establish the result; otherwise stop and report the uncertainty. -- `effect_state=committed` means the browser-side effect occurred even if later completion or cleanup failed. Do not repeat it; continue only after verifying the resulting page/download state. - -If `request-help` returns `outcome="disabled"`, do not retry it. Stop gracefully and report the transfer mechanism that requires human intervention. - -### Scripting & timing - -| Command | Summary | -|---------|---------| -| `bsk evaluate ` | Run JS in agent tab (see red lines). JS throw → stderr, **exit 0** (RPC success); use `--json` and check `.ok` to detect JS errors | -| `bsk wait-for-navigation` | Block until load/DOM idle/etc. (`--wait-until`, `--timeout`) | -| `bsk wait-ms ` | Sleep (`500ms`, `2s`, `1m`; **no** `--session`) | - -### Ask the human for help — `bsk request-help` - -When a step needs a human (captcha, login, OTP) or you want the user to -confirm an important action, pause and ask: - - bsk request-help --session --prompt "Solve the captcha, then click Done only after the site accepts it" \ - --title "Captcha required" --target @e7 --target "#submit" --timeout 5m - -- `--prompt` (required): what the user should do. -- `--title` (optional): custom title for the overlay panel. When omitted, - the extension shows its default localized title. -- `--target` (repeatable): a snapshot ref (`@e7`) or CSS selector - (`#submit`) to scroll to and flash-highlight. **Strongly recommended** — - whenever the prompt refers to a concrete element (a button to click, a - field to fill, a checkbox to toggle), pass its `@eN` ref / selector so the - user is guided straight to the right spot instead of hunting for it. For - interaction scenarios, always include the relevant target(s); reserve a - prompt with no `--target` for cases where there is genuinely no specific - element to point at (e.g. "wait for the page to finish loading"). -- `--timeout` (default `5m`): how long to wait. -- `--completion-criteria` (optional): JSON success detector. Use it only - when there is a concrete post-help success signal, e.g. - `{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`. - -The target tab is brought to the foreground; the page stays interactive -while the agent control mask is hidden. The call blocks until the user -explicitly acts, the timeout expires, cancellation arrives, or explicit -completion criteria match. Page reloads, SPA route changes, and captcha -refreshes do not return control by themselves. The result `outcome` is one of: - -- `continued` — the user finished and clicked Done / return control (treat as confirm). -- `cancelled` — the user clicked Cancel (treat as reject/abort). -- `timed_out` — nobody acted within the timeout. -- `completed` — the explicit `--completion-criteria` matched while the user had control. -- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal. - -`note` carries any text the user typed back. `resolved_targets` reports -which refs/selectors matched a live element. - -`request-help` does not refresh the page model after the user returns -control. After a `continued` or `completed` result, issue a separate -observation tool call (usually `bsk snapshot --session `) before using -new refs or reasoning about the post-help page state. -#### Disabling request-help (unattended mode) -Set `BSK_REQUEST_HELP=off` on unattended servers: `bsk request-help` then -returns immediately with `outcome="disabled"` (no overlay, no waiting, -exit 0). Any other value keeps it enabled. If you get `disabled`, do not -retry — complete the task autonomously or stop gracefully. - -### Recording — `bsk record` - -Capture the user's own actions in the Agent Window to a **trace bundle**, for later LLM-driven automation. New CLI builds request **trace v3** (page observations + action chain); older extensions may still return **trace v2** (actions only), which the CLI exports as a single `trace.json` without `states/`. - -```bash -bsk record start --browser \ - [--url https://…] [--purpose "publish a wiki doc"] \ - [--max-page-tokens 3000] [--redact-values] \ - [--output trace] -# `--url` is optional; default https://example.com/ when omitted (must be http(s)). -# Blocks until the user clicks Finish in the recording panel, then writes: -# trace/trace.json — action chain (+ state index when v3) -# trace/states/ — v3 only: one `sN.txt` observe snapshot per settled page state - -bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable +```text +bsk fill --value bsk select --value +bsk screenshot --out bsk emulate --device +bsk upload --file bsk download --out ``` `select` matches an option's `value` attribute, not its visible label. Device preset ids are @@ -280,6 +139,34 @@ lowercase and hyphenated, such as `iphone-14`. - `record` captures a user's actions for later replay. Read `bsk record start --help` before use, and never record banking, SSO, password-manager, or other sensitive pages. +## File transfer + +`upload` and `download` stage files through the daemon; the agent never touches browser-internal +paths. Treat upload as disclosure to the website, download as accepting website-controlled bytes. + +Upload has two independent mechanisms — choose explicitly, never rely on automatic fallback: + +- **Default (input mode):** for upload buttons, file-input labels, or "upload from computer" + actions. The command clicks the target and intercepts the native file chooser. +- **`--mode drop`:** for reliably identified attachment-receiving areas — an explicit drop zone, + chat composer, email editor, or form attachment area. Do not target page whitespace, generic + containers, or areas whose attachment ownership is ambiguous. + +Decision sequence when uploading: + +1. Try input mode (the default). +2. If it returns `reason=file_input_not_activated` with `effect_state=none`, re-observe. When a + reliable attachment target exists, try `--mode drop` once against that target. +3. Otherwise fall back to `request-help`. +4. **Never** switch mechanisms or repeat when `effect_state` is `unknown` or `committed` — the + browser may already have applied the file. + +A successful drop means Chrome dispatched the native file-drop event; it does not prove the site +accepted the attachment. Observe the page once after the command. + +Download default-refuses to overwrite; pass `--overwrite` when replacing an existing file is +intended. Read `bsk upload --help` and `bsk download --help` for all flags and error details. + ## Recover without wandering - Stale ref: observe again and retry the intended action once. diff --git a/crates/bsk-cli/src/cli/mod.rs b/crates/bsk-cli/src/cli/mod.rs index 4776ebe..8140045 100644 --- a/crates/bsk-cli/src/cli/mod.rs +++ b/crates/bsk-cli/src/cli/mod.rs @@ -182,7 +182,7 @@ pub enum Command { /// Set `` option(s) by `value` (repeat `--value` for multi-select) | -| `bsk press ` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) | - -### File transfer (require `--session`) - -| Command | Summary | -|---------|---------| -| `bsk upload --file ` | Click one upload trigger and attach an agent-readable local file (`--file` is repeatable) | -| `bsk download --out ` | Click one download trigger and copy the single completed file to an exact local path (`--overwrite` is opt-in) | - -The agent/harness decides whether a file transfer is appropriate and which local path belongs to the task. Treat upload as disclosure of that file to the current website, and download as accepting website-controlled bytes onto the local filesystem. Use only paths that are necessary for the user's bounded goal. - -BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted capabilities reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. Upload intercepts the native chooser for one transaction, locates the input activated in the resolved target's document, and assigns only the staged file paths. Download uniquely correlates one exact-target browser intent with one Chrome download in either event order, routes it through a daemon-minted relative directory, and lets the daemon validate and import it. Upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive. - -Do not use `request-help` merely because a native file chooser or browser download is involved; try these commands first. For transfer failures, use the structured error instead of retrying blindly: - -- `reason=file_input_not_activated` means the requested click did not activate exactly one ``; the page may use a non-input picker such as `window.showOpenFilePicker()`. Do not retry blindly. If human help is available, call `request-help` and tell the user the exact original local path to choose. The staged daemon path is internal and must not be shown to the user. -- `reason=file_input_probe_failed` means BrowserSkill could not safely establish the browser-side upload transaction. Do not repeat the same action; use `request-help` when available. -- `reason=set_file_input_failed` means BrowserSkill found the activated file input but Chrome rejected the staged path or assignment. Check the extension's file-URL access permission; otherwise use `request-help`. -- `reason=download_capture_failed` means BrowserSkill could not attribute exactly one completed download to the requested target. Do not retry blindly or accept an unrelated browser download; use `request-help` when available. -- `effect_state=none` means BrowserSkill confirmed that no file-transfer effect was committed. Follow the accompanying reason; a corrected target or explicit human fallback may be attempted. -- `effect_state=unknown` means the browser may already have attached or created the file. Do not repeat the transfer. Observe the page if that can establish the result; otherwise stop and report the uncertainty. -- `effect_state=committed` means the browser-side effect occurred even if later completion or cleanup failed. Do not repeat it; continue only after verifying the resulting page/download state. - -If `request-help` returns `outcome="disabled"`, do not retry it. Stop gracefully and report the transfer mechanism that requires human intervention. - -### Scripting & timing - -| Command | Summary | -|---------|---------| -| `bsk evaluate ` | Run JS in agent tab (see red lines). JS throw → stderr, **exit 0** (RPC success); use `--json` and check `.ok` to detect JS errors | -| `bsk wait-for-navigation` | Block until load/DOM idle/etc. (`--wait-until`, `--timeout`) | -| `bsk wait-ms ` | Sleep (`500ms`, `2s`, `1m`; **no** `--session`) | - -### Ask the human for help — `bsk request-help` - -When a step needs a human (captcha, login, OTP) or you want the user to -confirm an important action, pause and ask: - - bsk request-help --session --prompt "Solve the captcha, then click Done only after the site accepts it" \ - --title "Captcha required" --target @e7 --target "#submit" --timeout 5m - -- `--prompt` (required): what the user should do. -- `--title` (optional): custom title for the overlay panel. When omitted, - the extension shows its default localized title. -- `--target` (repeatable): a snapshot ref (`@e7`) or CSS selector - (`#submit`) to scroll to and flash-highlight. **Strongly recommended** — - whenever the prompt refers to a concrete element (a button to click, a - field to fill, a checkbox to toggle), pass its `@eN` ref / selector so the - user is guided straight to the right spot instead of hunting for it. For - interaction scenarios, always include the relevant target(s); reserve a - prompt with no `--target` for cases where there is genuinely no specific - element to point at (e.g. "wait for the page to finish loading"). -- `--timeout` (default `5m`): how long to wait. -- `--completion-criteria` (optional): JSON success detector. Use it only - when there is a concrete post-help success signal, e.g. - `{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`. - -The target tab is brought to the foreground; the page stays interactive -while the agent control mask is hidden. The call blocks until the user -explicitly acts, the timeout expires, cancellation arrives, or explicit -completion criteria match. Page reloads, SPA route changes, and captcha -refreshes do not return control by themselves. The result `outcome` is one of: - -- `continued` — the user finished and clicked Done / return control (treat as confirm). -- `cancelled` — the user clicked Cancel (treat as reject/abort). -- `timed_out` — nobody acted within the timeout. -- `completed` — the explicit `--completion-criteria` matched while the user had control. -- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal. - -`note` carries any text the user typed back. `resolved_targets` reports -which refs/selectors matched a live element. - -`request-help` does not refresh the page model after the user returns -control. After a `continued` or `completed` result, issue a separate -observation tool call (usually `bsk snapshot --session `) before using -new refs or reasoning about the post-help page state. -#### Disabling request-help (unattended mode) -Set `BSK_REQUEST_HELP=off` on unattended servers: `bsk request-help` then -returns immediately with `outcome="disabled"` (no overlay, no waiting, -exit 0). Any other value keeps it enabled. If you get `disabled`, do not -retry — complete the task autonomously or stop gracefully. - -### Recording — `bsk record` - -Capture the user's own actions in the Agent Window to a **trace bundle**, for later LLM-driven automation. New CLI builds request **trace v3** (page observations + action chain); older extensions may still return **trace v2** (actions only), which the CLI exports as a single `trace.json` without `states/`. - -```bash -bsk record start --browser \ - [--url https://…] [--purpose "publish a wiki doc"] \ - [--max-page-tokens 3000] [--redact-values] \ - [--output trace] -# `--url` is optional; default https://example.com/ when omitted (must be http(s)). -# Blocks until the user clicks Finish in the recording panel, then writes: -# trace/trace.json — action chain (+ state index when v3) -# trace/states/ — v3 only: one `sN.txt` observe snapshot per settled page state - -bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable +```text +bsk fill --value bsk select --value +bsk screenshot --out bsk emulate --device +bsk upload --file bsk download --out ``` `select` matches an option's `value` attribute, not its visible label. Device preset ids are @@ -280,6 +139,34 @@ lowercase and hyphenated, such as `iphone-14`. - `record` captures a user's actions for later replay. Read `bsk record start --help` before use, and never record banking, SSO, password-manager, or other sensitive pages. +## File transfer + +`upload` and `download` stage files through the daemon; the agent never touches browser-internal +paths. Treat upload as disclosure to the website, download as accepting website-controlled bytes. + +Upload has two independent mechanisms — choose explicitly, never rely on automatic fallback: + +- **Default (input mode):** for upload buttons, file-input labels, or "upload from computer" + actions. The command clicks the target and intercepts the native file chooser. +- **`--mode drop`:** for reliably identified attachment-receiving areas — an explicit drop zone, + chat composer, email editor, or form attachment area. Do not target page whitespace, generic + containers, or areas whose attachment ownership is ambiguous. + +Decision sequence when uploading: + +1. Try input mode (the default). +2. If it returns `reason=file_input_not_activated` with `effect_state=none`, re-observe. When a + reliable attachment target exists, try `--mode drop` once against that target. +3. Otherwise fall back to `request-help`. +4. **Never** switch mechanisms or repeat when `effect_state` is `unknown` or `committed` — the + browser may already have applied the file. + +A successful drop means Chrome dispatched the native file-drop event; it does not prove the site +accepted the attachment. Observe the page once after the command. + +Download default-refuses to overwrite; pass `--overwrite` when replacing an existing file is +intended. Read `bsk upload --help` and `bsk download --help` for all flags and error details. + ## Recover without wandering - Stale ref: observe again and retry the intended action once.