figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(bsk): support drop to upload

Browse Source
This commit is contained in:
Ljy-0827
2026-08-24 16:55:16 +08:00
parent c74c119a82
commit 8822bba9a5
21 changed files with 997 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/extension/e2e/fixtures/file-transfer.html
+81
View File
@@ -0,0 +1,81 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<title>BrowserSkill file-transfer fixture</title>
<style>
body { font: 16px system-ui; max-width: 760px; margin: 48px auto; }
.actions { display: flex; gap: 12px; align-items: center; margin: 24px 0; }
button, a { padding: 10px 16px; border: 1px solid #777; border-radius: 8px; }
a[aria-disabled="true"] { opacity: .45; pointer-events: none; }
#drop { padding: 36px; border: 3px dashed #777; border-radius: 12px; text-align: center; }
#drop.dragging { border-color: #2563eb; background: #eff6ff; }
img { display: block; max-width: 480px; max-height: 360px; margin-top: 20px; }
</style>
</head>
<body>
<h1>Upload then download the same image</h1>
<p id="status">No image selected</p>
<div class="actions">
<button id="upload" type="button">Upload through file input</button>
<input id="file" type="file" accept="image/png" hidden />
<a id="download" aria-disabled="true">Download uploaded image</a>
</div>
<div id="drop" role="button" tabindex="0">Drop an image here</div>
<img id="preview" alt="Uploaded image preview" hidden />
<script>
const input = document.querySelector('#file');
const upload = document.querySelector('#upload');
const drop = document.querySelector('#drop');
const download = document.querySelector('#download');
const preview = document.querySelector('#preview');
const status = document.querySelector('#status');
let objectUrl;
window.__bskTransferFixture = { events: [], result: null };
function accept(file, mechanism) {
if (!file) return;
if (objectUrl) URL.revokeObjectURL(objectUrl);
objectUrl = URL.createObjectURL(file);
preview.src = objectUrl;
preview.hidden = false;
download.href = objectUrl;
download.download = file.name;
download.removeAttribute('aria-disabled');
status.textContent = `Ready via ${mechanism}: ${file.name} (${file.size} bytes)`;
window.__bskTransferFixture.result = {
mechanism,
name: file.name,
size: file.size,
type: file.type,
hasPathProperty: 'path' in file,
};
}
upload.addEventListener('click', () => input.click());
input.addEventListener('change', () => accept(input.files[0], 'input'));
for (const type of ['dragenter', 'dragover']) {
drop.addEventListener(type, event => {
const types = [...event.dataTransfer.types];
window.__bskTransferFixture.events.push({ type, types });
if (!types.includes('Files')) return;
event.preventDefault();
drop.classList.add('dragging');
});
}
drop.addEventListener('dragleave', () => drop.classList.remove('dragging'));
drop.addEventListener('drop', event => {
window.__bskTransferFixture.events.push({
type: 'drop',
types: [...event.dataTransfer.types],
fileCount: event.dataTransfer.files.length,
});
event.preventDefault();
drop.classList.remove('dragging');
accept(event.dataTransfer.files[0], 'drop');
});
</script>
</body>
</html>
apps/extension/src/lib/__tests__/capture-suppress-bridge.test.ts
+6 -6
View File
@@ -2,7 +2,7 @@ import { describe, expect, it, vi } from "vitest";
import { import {
CAPTURE_SUPPRESS, CAPTURE_SUPPRESS,
type CaptureSuppressMessage, type CaptureSuppressMessage,
withOverlaysHiddenForCapture, withExtensionOverlayHidden,
} from "../capture-suppress-bridge"; } from "../capture-suppress-bridge";
function recordingSendToTab(events: string[]) { function recordingSendToTab(events: string[]) {
@@ -12,12 +12,12 @@ function recordingSendToTab(events: string[]) {
}); });
} }
describe("withOverlaysHiddenForCapture", () => { describe("withExtensionOverlayHidden", () => {
it("sends begin → fn → end and returns fn's result", async () => { it("sends begin → fn → end and returns fn's result", async () => {
const events: string[] = []; const events: string[] = [];
const sendToTab = recordingSendToTab(events); const sendToTab = recordingSendToTab(events);
const result = await withOverlaysHiddenForCapture( const result = await withExtensionOverlayHidden(
7, 7,
async () => { async () => {
events.push("capture"); events.push("capture");
@@ -37,7 +37,7 @@ describe("withOverlaysHiddenForCapture", () => {
const sendToTab = recordingSendToTab(events); const sendToTab = recordingSendToTab(events);
await expect( await expect(
withOverlaysHiddenForCapture( withExtensionOverlayHidden(
7, 7,
async () => { async () => {
events.push("capture"); events.push("capture");
@@ -55,7 +55,7 @@ describe("withOverlaysHiddenForCapture", () => {
}); });
const fn = vi.fn(async () => "shot"); const fn = vi.fn(async () => "shot");
const result = await withOverlaysHiddenForCapture(7, fn, sendToTab); const result = await withExtensionOverlayHidden(7, fn, sendToTab);
expect(result).toBe("shot"); expect(result).toBe("shot");
expect(fn).toHaveBeenCalledTimes(1); expect(fn).toHaveBeenCalledTimes(1);
@@ -69,7 +69,7 @@ describe("withOverlaysHiddenForCapture", () => {
return { type: CAPTURE_SUPPRESS, ok: true }; return { type: CAPTURE_SUPPRESS, ok: true };
}); });
const result = await withOverlaysHiddenForCapture(7, async () => "shot", sendToTab); const result = await withExtensionOverlayHidden(7, async () => "shot", sendToTab);
expect(result).toBe("shot"); expect(result).toBe("shot");
expect(sendToTab).toHaveBeenCalledTimes(2); expect(sendToTab).toHaveBeenCalledTimes(2);
apps/extension/src/lib/capture-suppress-bridge.ts
+5 -5
View File
@@ -1,9 +1,9 @@
/** /**
* Wire protocol and background-side helper for hiding the extension's * Wire protocol and background-side helper for temporarily excluding the
* in-page overlay while a screenshot is captured, so captured frames only * extension's in-page overlay. Screenshots use it to avoid captured chrome;
* contain the page itself. * coordinate-driven native input uses it so Chrome hit-tests the page.
* *
* The background wraps every capture in `withOverlaysHiddenForCapture`, * The background wraps each bounded operation in `withExtensionOverlayHidden`,
* which sends `begin` (the content script hides the overlay host and only * which sends `begin` (the content script hides the overlay host and only
* acks once the compositor has produced an overlay-free frame) and always * acks once the compositor has produced an overlay-free frame) and always
* follows up with `end`, even when the capture throws. Tabs without the * follows up with `end`, even when the capture throws. Tabs without the
@@ -46,7 +46,7 @@ const defaultSendToTab: CaptureSuppressSendToTab = (tabId, message) =>
* (or any other `begin` failure) simply skips suppression — there is no * (or any other `begin` failure) simply skips suppression — there is no
* overlay to hide in that tab. * overlay to hide in that tab.
*/ */
export async function withOverlaysHiddenForCapture<T>( export async function withExtensionOverlayHidden<T>(
tabId: number, tabId: number,
fn: () => Promise<T>, fn: () => Promise<T>,
sendToTab: CaptureSuppressSendToTab = defaultSendToTab, sendToTab: CaptureSuppressSendToTab = defaultSendToTab,
apps/extension/src/tools/__tests__/file-transfer.test.ts
+282 -2
View File
@@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager"; import { SessionManager } from "@/session-manager/manager";
import { type DownloadsApi, handleDownload } from "../download"; import { type DownloadsApi, handleDownload } from "../download";
import { captureBrowserDownload } from "../download-capture"; import { captureBrowserDownload } from "../download-capture";
import { uploadThroughFileDrop } from "../file-drop-transaction";
import { uploadThroughActivatedFileInput } from "../file-input-transaction"; import { uploadThroughActivatedFileInput } from "../file-input-transaction";
import type { ResolvedActionTarget } from "../interaction"; import type { ResolvedActionTarget } from "../interaction";
import type { CdpRunner } from "../shared"; import type { CdpRunner } from "../shared";
@@ -37,16 +38,20 @@ function fakeEvent<T extends (...args: never[]) => unknown>() {
}; };
} }
function actionTarget(frameId?: string): ResolvedActionTarget { function actionTarget(frameId?: string, sessionId?: string): ResolvedActionTarget {
return { return {
tab: { tabId: 4, windowId: 100, active: true }, tab: { tabId: 4, windowId: 100, active: true },
backendNodeId: 123, backendNodeId: 123,
cdpTarget: { tabId: 4 }, cdpTarget: { tabId: 4, ...(sessionId ? { sessionId } : {}) },
...(frameId ? { frameId } : {}), ...(frameId ? { frameId } : {}),
usedRef: "e3", usedRef: "e3",
}; };
} }
function withoutOverlay<T>(operation: () => Promise<T>): Promise<T> {
return operation();
}
function uploadCdp( function uploadCdp(
options: { options: {
inputCount?: number; inputCount?: number;
@@ -59,6 +64,7 @@ function uploadCdp(
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined; let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
const send = vi.fn(async (_tabId: number, method: string, params?: object) => { const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
calls.push({ method, params }); calls.push({ method, params });
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "Page.setInterceptFileChooserDialog") return {}; if (method === "Page.setInterceptFileChooserDialog") return {};
if (method === "Page.getLayoutMetrics") if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } }; return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
@@ -110,6 +116,50 @@ function uploadCdp(
}; };
} }
function dropCdp(
options: {
targetContainsPoint?: boolean;
dispatchError?: (type: string) => Error | Promise<never> | undefined;
onCall?: (method: string) => void;
} = {},
) {
const calls: Array<{ method: string; params?: object }> = [];
const detach = vi.fn(async () => {});
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
calls.push({ method, params });
options.onCall?.(method);
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
}
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] };
}
if (method === "DOM.getNodeForLocation") return { backendNodeId: 124 };
if (method === "DOM.resolveNode") {
const id = (params as { backendNodeId?: number }).backendNodeId;
return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } };
}
if (method === "Runtime.callFunctionOn") {
return { result: { value: options.targetContainsPoint ?? true } };
}
if (method === "Input.dispatchDragEvent") {
const type = (params as { type?: string }).type ?? "";
const error = options.dispatchError?.(type);
if (error instanceof Promise) return error;
if (error) throw error;
return {};
}
if (method === "Runtime.releaseObjectGroup") return {};
throw new Error(`unexpected drop CDP command ${method}`);
});
const cdp: CdpRunner = {
send: send as unknown as CdpRunner["send"],
detach,
};
return { calls, cdp, detach };
}
describe("file transfer tools", () => { describe("file transfer tools", () => {
it("captures the file input activated by the requested click and injects only staged paths", async () => { it("captures the file input activated by the requested click and injects only staged paths", async () => {
const manager = sessions(); const manager = sessions();
@@ -271,6 +321,236 @@ describe("file transfer tools", () => {
expect(detach).toHaveBeenCalledWith(4); expect(detach).toHaveBeenCalledWith(4);
}); });
it("delivers staged files through one explicit native drop without clicking", async () => {
const manager = sessions();
const ctx = await manager.start("s1");
ctx.refStore.set("e3", 123, { tabId: 4 });
const lifecycle: string[] = [];
const { cdp, calls } = dropCdp({ onCall: (method) => lifecycle.push(method) });
const overlayPhases: string[] = [];
const result = await handleUpload(
manager,
{
session_id: "s1",
ref: "@e3",
mode: "drop",
files: [{ transfer_id: "tr_1", name: "one.png", staged_path: "/stage/one.png" }],
},
{
cdp,
tabsApi: tabsApi(),
sendToTab: vi.fn(async (_tabId, message) => {
overlayPhases.push(message.phase);
lifecycle.push(`overlay:${message.phase}`);
}),
},
);
expect(result).toMatchObject({ tab_id: 4, used_ref: "e3", file_names: ["one.png"] });
expect(calls.filter((call) => call.method === "Input.dispatchDragEvent")).toEqual([
{
method: "Input.dispatchDragEvent",
params: {
type: "dragEnter",
x: 110,
y: 70,
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
},
},
{
method: "Input.dispatchDragEvent",
params: {
type: "dragOver",
x: 110,
y: 70,
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
},
},
{
method: "Input.dispatchDragEvent",
params: {
type: "drop",
x: 110,
y: 70,
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
},
},
]);
expect(calls.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
expect(overlayPhases).toEqual(["begin", "end"]);
expect(lifecycle.indexOf("DOM.getContentQuads")).toBeLessThan(
lifecycle.indexOf("overlay:begin"),
);
expect(lifecycle.indexOf("overlay:begin")).toBeLessThan(
lifecycle.indexOf("DOM.getNodeForLocation"),
);
expect(lifecycle.lastIndexOf("Input.dispatchDragEvent")).toBeLessThan(
lifecycle.indexOf("overlay:end"),
);
expect(lifecycle.indexOf("overlay:end")).toBeLessThan(
lifecycle.indexOf("Runtime.releaseObjectGroup"),
);
});
it("fails closed before dragging when another element owns the action point", async () => {
const { cdp, calls } = dropCdp({ targetContainsPoint: false });
const result = await uploadThroughFileDrop({
cdp,
actionTarget: actionTarget(),
files: ["/stage/one.png"],
timeoutMs: 100,
withOverlayHidden: withoutOverlay,
});
expect(result).toMatchObject({
code: "permission_denied",
data: {
reason: "file_drop_target_unavailable",
mechanism: "drop",
effect_state: "none",
phase: "resolve_target",
},
});
expect(calls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false);
});
it("dispatches an OOPIF file drop in the child target's local coordinates", async () => {
const rootCalls: Array<{ method: string; params?: object }> = [];
const childCalls: Array<{ method: string; params?: object }> = [];
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method, params) => {
rootCalls.push({ method, params });
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.getBoxModel") {
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
}
throw new Error(`unexpected root command ${method}`);
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method, params) => {
childCalls.push({ method, params });
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } };
}
if (method === "DOM.getNodeForLocation") {
return { backendNodeId: 124, frameId: "child" };
}
if (method === "DOM.resolveNode") {
const id = (params as { backendNodeId?: number }).backendNodeId;
return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } };
}
if (method === "Runtime.callFunctionOn") return { result: { value: true } };
if (method === "Input.dispatchDragEvent" || method === "Runtime.releaseObjectGroup") {
return {};
}
throw new Error(`unexpected child command ${method}`);
}) as CdpRunner["sendToTarget"],
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
})),
};
const result = await uploadThroughFileDrop({
cdp,
actionTarget: actionTarget("child", "child-session"),
files: ["/stage/one.png"],
timeoutMs: 100,
withOverlayHidden: withoutOverlay,
});
expect(result).toMatchObject({ x: 110, y: 70 });
expect(
childCalls
.filter((call) => call.method === "Input.dispatchDragEvent")
.map((call) => call.params),
).toEqual([
expect.objectContaining({ type: "dragEnter", x: 110, y: 70 }),
expect.objectContaining({ type: "dragOver", x: 110, y: 70 }),
expect.objectContaining({ type: "drop", x: 110, y: 70 }),
]);
expect(rootCalls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false);
});
it("reports a missing native drag command as unsupported before commit", async () => {
const { cdp } = dropCdp({
dispatchError: (type) =>
type === "dragEnter" ? new Error("'Input.dispatchDragEvent' wasn't found") : undefined,
});
const result = await uploadThroughFileDrop({
cdp,
actionTarget: actionTarget(),
files: ["/stage/one.png"],
timeoutMs: 100,
withOverlayHidden: withoutOverlay,
});
expect(result).toMatchObject({
code: "unsupported",
data: {
reason: "upload_mechanism_unsupported",
effect_state: "none",
phase: "drag_enter",
},
});
});
it("cancels an unresolved drag before returning a pre-commit timeout", async () => {
const { cdp, calls } = dropCdp({
dispatchError: (type) => (type === "dragEnter" ? new Promise<never>(() => {}) : undefined),
});
const result = await uploadThroughFileDrop({
cdp,
actionTarget: actionTarget(),
files: ["/stage/one.png"],
timeoutMs: 10,
withOverlayHidden: withoutOverlay,
});
expect(result).toMatchObject({
code: "timeout",
data: { reason: "file_drop_failed", effect_state: "none", phase: "drag_enter" },
});
expect(calls).toContainEqual({
method: "Input.dispatchDragEvent",
params: expect.objectContaining({ type: "dragCancel" }),
});
});
it("marks an unresolved drop unknown and detaches browser state", async () => {
const { cdp, detach } = dropCdp({
dispatchError: (type) => (type === "drop" ? new Promise<never>(() => {}) : undefined),
});
const result = await uploadThroughFileDrop({
cdp,
actionTarget: actionTarget(),
files: ["/stage/one.png"],
timeoutMs: 10,
withOverlayHidden: withoutOverlay,
});
expect(result).toMatchObject({
code: "timeout",
data: { reason: "file_drop_failed", effect_state: "unknown", phase: "drop" },
});
expect(detach).toHaveBeenCalledWith(4);
});
it("routes one exact-target download through a browser-relative capability", async () => { it("routes one exact-target download through a browser-relative capability", async () => {
const manager = sessions(); const manager = sessions();
const ctx = await manager.start("s1"); const ctx = await manager.start("s1");
apps/extension/src/tools/__tests__/frame-geometry.test.ts
+2
View File
@@ -169,6 +169,7 @@ describe("frame geometry projection", () => {
expect(geometry).toMatchObject({ expect(geometry).toMatchObject({
topBounds: { x: 100, y: 100, width: 200, height: 100 }, topBounds: { x: 100, y: 100, width: 200, height: 100 },
actionPoint: { x: 200, y: 150 }, actionPoint: { x: 200, y: 150 },
targetActionPoint: { x: 100, y: 50 },
}); });
}); });
@@ -261,6 +262,7 @@ describe("frame geometry projection", () => {
expect(geometry).toMatchObject({ expect(geometry).toMatchObject({
topBounds: { x: 100, y: 100, width: 200, height: 100 }, topBounds: { x: 100, y: 100, width: 200, height: 100 },
actionPoint: { x: 200, y: 150 }, actionPoint: { x: 200, y: 150 },
targetActionPoint: { x: 200, y: 150 },
}); });
}); });
apps/extension/src/tools/file-drop-transaction.ts
+319
View File
@@ -0,0 +1,319 @@
// One native file-drop transaction bound to an explicit page target. CDP drag
// events are sent to the target that owns the node: root targets use top-level
// coordinates, while OOPIF targets require coordinates local to their session.
import type { RpcError, TransferEffectState } from "@/transport/types";
import { transferError } from "./errors";
import { resolveNodeGeometry } from "./frame-geometry";
import type { ResolvedActionTarget } from "./interaction";
import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared";
import { BoundedWaitError, waitBounded } from "./transfer-transaction";
const CLEANUP_TIMEOUT_MS = 1_000;
type FileDropPhase = "resolve_target" | "drag_enter" | "drag_over" | "drop" | "cleanup";
interface RuntimeReply {
result?: { value?: unknown; objectId?: string };
exceptionDetails?: { text?: string; exception?: { description?: string } };
}
interface DragData {
items: [];
files: string[];
dragOperationsMask: number;
}
export interface FileDropTransactionOptions {
cdp: CdpRunner;
actionTarget: ResolvedActionTarget;
files: string[];
timeoutMs: number;
signal?: AbortSignal;
withOverlayHidden: <T>(operation: () => Promise<T>) => Promise<T>;
}
export interface FileDropTransactionResult {
x: number;
y: number;
}
function failure(
code: RpcError["code"],
reason: "upload_mechanism_unsupported" | "file_drop_target_unavailable" | "file_drop_failed",
message: string,
effectState: TransferEffectState,
phase: FileDropPhase,
): RpcError {
const error = transferError(code, reason, message, { effectState, phase });
error.data = { ...error.data, mechanism: "drop" };
return error;
}
function enrichFailure(
error: RpcError,
effectState: TransferEffectState,
phase: FileDropPhase,
): RpcError {
return {
...error,
data: {
...error.data,
mechanism: "drop",
effect_state: effectState,
phase,
},
};
}
function messageOf(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
function methodUnavailable(error: unknown): boolean {
return /(?:method|command).*(?:not found|wasn't found|unsupported)|wasn't found|-32601/i.test(
messageOf(error),
);
}
function runtimeError(reply: RuntimeReply, fallback: string): Error | null {
if (!reply.exceptionDetails) return null;
return new Error(
reply.exceptionDetails.exception?.description ?? reply.exceptionDetails.text ?? fallback,
);
}
async function pointBelongsToTarget(
options: FileDropTransactionOptions,
x: number,
y: number,
objectGroup: string,
): Promise<boolean> {
const target = options.actionTarget.cdpTarget;
const hit = await sendToCdpTarget<{ backendNodeId?: number; frameId?: string }>(
options.cdp,
target,
"DOM.getNodeForLocation",
{ x: Math.round(x), y: Math.round(y), includeUserAgentShadowDOM: true },
);
if (typeof hit.backendNodeId !== "number") return false;
if (options.actionTarget.frameId && hit.frameId !== options.actionTarget.frameId) return false;
const [resolvedTarget, resolvedHit] = await Promise.all([
sendToCdpTarget<{ object?: { objectId?: string } }>(options.cdp, target, "DOM.resolveNode", {
backendNodeId: options.actionTarget.backendNodeId,
objectGroup,
}),
sendToCdpTarget<{ object?: { objectId?: string } }>(options.cdp, target, "DOM.resolveNode", {
backendNodeId: hit.backendNodeId,
objectGroup,
}),
]);
const targetObjectId = resolvedTarget.object?.objectId;
const hitObjectId = resolvedHit.object?.objectId;
if (!targetObjectId || !hitObjectId) return false;
const reply = await sendToCdpTarget<RuntimeReply>(options.cdp, target, "Runtime.callFunctionOn", {
objectId: targetObjectId,
functionDeclaration: `function(hit) {
for (let node = hit; node; ) {
if (node === this) return true;
if (this.contains?.(node)) return true;
const root = node.getRootNode?.();
node = node.parentNode || (root && root !== node ? root.host : null);
}
return false;
}`,
arguments: [{ objectId: hitObjectId }],
returnByValue: true,
});
const error = runtimeError(reply, "failed to verify file-drop target");
if (error) throw error;
return reply.result?.value === true;
}
export async function uploadThroughFileDrop(
options: FileDropTransactionOptions,
): Promise<FileDropTransactionResult | RpcError> {
const deadline = Date.now() + options.timeoutMs;
const target = options.actionTarget.cdpTarget;
const objectGroup = `bsk-file-drop-${crypto.randomUUID()}`;
const dragData: DragData = {
items: [],
files: options.files,
dragOperationsMask: 1,
};
let dragStarted = false;
let dropSent = false;
let dragPoint = { x: 0, y: 0 };
let cleanupFailed = false;
let outcome: FileDropTransactionResult | RpcError = failure(
"protocol_error",
"file_drop_failed",
"file-drop transaction ended without an outcome",
"none",
"cleanup",
);
try {
const geometry = await waitBounded(
resolveNodeGeometry(
options.cdp,
options.actionTarget.tab.tabId,
{
target,
backendNodeId: options.actionTarget.backendNodeId,
...(options.actionTarget.frameId ? { frameId: options.actionTarget.frameId } : {}),
},
{ scrollIntoView: true },
),
deadline,
options.signal,
"resolving file-drop target timed out",
);
if (isRpcError(geometry)) {
outcome = enrichFailure(geometry, "none", "resolve_target");
return outcome;
}
const point = geometry.targetActionPoint;
dragPoint = point;
outcome = await options.withOverlayHidden(async () => {
try {
try {
const belongs = await waitBounded(
pointBelongsToTarget(options, point.x, point.y, objectGroup),
deadline,
options.signal,
"verifying file-drop target timed out",
);
if (!belongs) {
return failure(
"permission_denied",
"file_drop_target_unavailable",
"file-drop target is not the topmost element at its action point",
"none",
"resolve_target",
);
}
} catch (error) {
return failure(
error instanceof BoundedWaitError ? "timeout" : "cdp_failed",
"file_drop_target_unavailable",
messageOf(error),
"none",
"resolve_target",
);
}
for (const [type, phase] of [
["dragEnter", "drag_enter"],
["dragOver", "drag_over"],
] as const) {
try {
if (type === "dragEnter") dragStarted = true;
await waitBounded(
sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", {
type,
x: point.x,
y: point.y,
data: dragData,
}),
deadline,
options.signal,
`${type} timed out`,
);
} catch (error) {
const unsupported = methodUnavailable(error);
if (type === "dragEnter" && unsupported) dragStarted = false;
return failure(
unsupported
? "unsupported"
: error instanceof BoundedWaitError
? "timeout"
: "cdp_failed",
unsupported ? "upload_mechanism_unsupported" : "file_drop_failed",
messageOf(error),
"none",
phase,
);
}
}
try {
dropSent = true;
await waitBounded(
sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", {
type: "drop",
x: point.x,
y: point.y,
data: dragData,
}),
deadline,
options.signal,
"file drop timed out",
);
return { x: point.x, y: point.y };
} catch (error) {
outcome = failure(
error instanceof BoundedWaitError ? "timeout" : "cdp_failed",
"file_drop_failed",
messageOf(error),
"unknown",
"drop",
);
return outcome;
}
} finally {
if (dragStarted && !dropSent) {
try {
await waitBounded(
sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", {
type: "dragCancel",
x: dragPoint.x,
y: dragPoint.y,
data: dragData,
}),
Date.now() + CLEANUP_TIMEOUT_MS,
undefined,
"cancelling file drag timed out",
);
} catch {
cleanupFailed = true;
}
}
}
});
return outcome;
} catch (error) {
outcome = failure(
error instanceof BoundedWaitError ? "timeout" : "cdp_failed",
"file_drop_target_unavailable",
messageOf(error),
"none",
"resolve_target",
);
return outcome;
} finally {
try {
await waitBounded(
sendToCdpTarget(options.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }),
Date.now() + CLEANUP_TIMEOUT_MS,
undefined,
"releasing file-drop object group timed out",
);
} catch {
cleanupFailed = true;
}
const effect = isRpcError(outcome)
? (outcome.data?.effect_state as TransferEffectState | undefined)
: "committed";
if (effect === "unknown" || cleanupFailed) {
await options.cdp.detach?.(target.tabId);
}
if (cleanupFailed && isRpcError(outcome)) {
outcome.data = { ...outcome.data, cleanup_state: "failed" };
}
}
}
apps/extension/src/tools/file-input-transaction.ts
+1 -40
View File
@@ -7,6 +7,7 @@ import type { ClickResult, RpcError, TransferEffectState } from "@/transport/typ
import { transferError } from "./errors"; import { transferError } from "./errors";
import type { ResolvedActionTarget } from "./interaction"; import type { ResolvedActionTarget } from "./interaction";
import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared"; import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared";
import { BoundedWaitError, remainingMs, waitBounded } from "./transfer-transaction";
const CLEANUP_TIMEOUT_MS = 1_000; const CLEANUP_TIMEOUT_MS = 1_000;
const ACTIVATION_GRACE_MS = 1_000; const ACTIVATION_GRACE_MS = 1_000;
@@ -45,46 +46,6 @@ export interface FileInputTransactionResult {
multiple: boolean; multiple: boolean;
} }
class BoundedWaitError extends Error {
constructor(
readonly kind: "timeout" | "aborted",
message: string,
) {
super(message);
}
}
function remainingMs(deadline: number): number {
return Math.max(0, deadline - Date.now());
}
async function waitBounded<T>(
promise: Promise<T>,
deadline: number,
signal: AbortSignal | undefined,
timeoutMessage: string,
): Promise<T> {
const remaining = remainingMs(deadline);
if (signal?.aborted) throw new BoundedWaitError("aborted", "upload transaction aborted");
if (remaining === 0) throw new BoundedWaitError("timeout", timeoutMessage);
let timer: ReturnType<typeof setTimeout> | undefined;
let onAbort: (() => void) | undefined;
const boundary = new Promise<never>((_resolve, reject) => {
timer = setTimeout(() => reject(new BoundedWaitError("timeout", timeoutMessage)), remaining);
if (signal) {
onAbort = () => reject(new BoundedWaitError("aborted", "upload transaction aborted"));
signal.addEventListener("abort", onAbort, { once: true });
}
});
try {
return await Promise.race([promise, boundary]);
} finally {
if (timer) clearTimeout(timer);
if (signal && onAbort) signal.removeEventListener("abort", onAbort);
}
}
function runtimeError(reply: RuntimeReply, fallback: string): Error | null { function runtimeError(reply: RuntimeReply, fallback: string): Error | null {
if (!reply.exceptionDetails) return null; if (!reply.exceptionDetails) return null;
return new Error( return new Error(
apps/extension/src/tools/frame-geometry.ts
+27 -1
View File
@@ -29,7 +29,10 @@ export interface NodeAddress {
export interface ResolvedNodeGeometry { export interface ResolvedNodeGeometry {
topVisibleRegions: Region; topVisibleRegions: Region;
topBounds: ViewportRect; topBounds: ViewportRect;
/** Point in the top-level tab viewport, used by root-target input events. */
actionPoint: Point; actionPoint: Point;
/** Point in the addressed CDP target's viewport, used by OOPIF-local input events. */
targetActionPoint: Point;
} }
function geometryError(message: string): RpcError { function geometryError(message: string): RpcError {
@@ -262,11 +265,30 @@ export async function resolveNodeGeometry(
if (isRpcError(localRegion)) return localRegion; if (isRpcError(localRegion)) return localRegion;
let topVisibleRegions: Region; let topVisibleRegions: Region;
let targetActionPoint: Point | null = null;
if (address.frameId && graph) { if (address.frameId && graph) {
const projection = await resolveFrameProjection(cdp, graph, address.frameId); const projection = await resolveFrameProjection(cdp, graph, address.frameId);
if (!projection) if (!projection)
return geometryError(`could not resolve frame geometry for ${address.frameId}`); return geometryError(`could not resolve frame geometry for ${address.frameId}`);
topVisibleRegions = projectRegionToViewport(localRegion, projection); topVisibleRegions = projectRegionToViewport(localRegion, projection);
if (address.target.sessionId) {
const localViewport = projection.edges[0]?.sourceViewport ?? projection.topViewport;
const localVisibleRegions = localRegion
.map((polygon) =>
clipPolygon(
polygon,
rectPolygon({
x: 0,
y: 0,
w: localViewport.width,
h: localViewport.height,
}),
),
)
.filter((polygon) => polygon.length >= 3);
const localActionRegion = largestRegion(localVisibleRegions);
targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null;
}
} else { } else {
const viewport = await targetViewport(cdp, address.target); const viewport = await targetViewport(cdp, address.target);
if (!viewport) return geometryError("could not resolve top viewport geometry"); if (!viewport) return geometryError("could not resolve top viewport geometry");
@@ -283,7 +305,11 @@ export async function resolveNodeGeometry(
if (!topBounds || !actionPoint) { if (!topBounds || !actionPoint) {
return { code: "permission_denied", message: "element not visible" }; return { code: "permission_denied", message: "element not visible" };
} }
return { topVisibleRegions, topBounds, actionPoint }; if (address.target.sessionId && !targetActionPoint) {
return { code: "permission_denied", message: "element not visible in its target" };
}
targetActionPoint ??= actionPoint;
return { topVisibleRegions, topBounds, actionPoint, targetActionPoint };
} catch (error) { } catch (error) {
return geometryError(error instanceof Error ? error.message : String(error)); return geometryError(error instanceof Error ? error.message : String(error));
} }
apps/extension/src/tools/observation.ts
+3 -3
View File
@@ -17,7 +17,7 @@ import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
import type { CdpTarget } from "@/browser-driver/frame-graph"; import type { CdpTarget } from "@/browser-driver/frame-graph";
import { import {
type CaptureSuppressSendToTab, type CaptureSuppressSendToTab,
withOverlaysHiddenForCapture, withExtensionOverlayHidden,
} from "@/lib/capture-suppress-bridge"; } from "@/lib/capture-suppress-bridge";
import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { SessionContext, SessionManager } from "@/session-manager/manager";
import type { import type {
@@ -312,7 +312,7 @@ export async function handleScreenshot(
tabId: target.tabId, tabId: target.tabId,
...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}), ...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}),
}; };
const captured = await withOverlaysHiddenForCapture( const captured = await withExtensionOverlayHidden(
target.tabId, target.tabId,
() => () =>
captureElementScreenshot( captureElementScreenshot(
@@ -344,7 +344,7 @@ export async function handleScreenshot(
); );
} }
const captured = await withOverlaysHiddenForCapture( const captured = await withExtensionOverlayHidden(
target.tabId, target.tabId,
() => captureFullTabPng(deps, ctx, target, signal), () => captureFullTabPng(deps, ctx, target, signal),
deps.sendToTab, deps.sendToTab,
apps/extension/src/tools/transfer-transaction.ts
+42
View File
@@ -0,0 +1,42 @@
// Shared time boundary for browser-side file-transfer transactions. Concrete
// upload/download mechanisms own their state machines and effect semantics.
export class BoundedWaitError extends Error {
constructor(
readonly kind: "timeout" | "aborted",
message: string,
) {
super(message);
}
}
export function remainingMs(deadline: number): number {
return Math.max(0, deadline - Date.now());
}
export async function waitBounded<T>(
promise: Promise<T>,
deadline: number,
signal: AbortSignal | undefined,
timeoutMessage: string,
): Promise<T> {
const remaining = remainingMs(deadline);
if (signal?.aborted) throw new BoundedWaitError("aborted", "file transfer aborted");
if (remaining === 0) throw new BoundedWaitError("timeout", timeoutMessage);
let timer: ReturnType<typeof setTimeout> | undefined;
let onAbort: (() => void) | undefined;
const boundary = new Promise<never>((_resolve, reject) => {
timer = setTimeout(() => reject(new BoundedWaitError("timeout", timeoutMessage)), remaining);
if (signal) {
onAbort = () => reject(new BoundedWaitError("aborted", "file transfer aborted"));
signal.addEventListener("abort", onAbort, { once: true });
}
});
try {
return await Promise.race([promise, boundary]);
} finally {
if (timer) clearTimeout(timer);
if (signal && onAbort) signal.removeEventListener("abort", onAbort);
}
}
apps/extension/src/tools/upload.ts
+37 -11
View File
@@ -2,8 +2,13 @@
// target, then delegate the browser protocol transaction to the file-input // target, then delegate the browser protocol transaction to the file-input
// transaction module. // transaction module.
import {
type CaptureSuppressSendToTab,
withExtensionOverlayHidden,
} from "@/lib/capture-suppress-bridge";
import type { SessionManager } from "@/session-manager/manager"; import type { SessionManager } from "@/session-manager/manager";
import type { RpcError, UploadParams, UploadResult } from "@/transport/types"; import type { RpcError, UploadParams, UploadResult } from "@/transport/types";
import { uploadThroughFileDrop } from "./file-drop-transaction";
import { uploadThroughActivatedFileInput } from "./file-input-transaction"; import { uploadThroughActivatedFileInput } from "./file-input-transaction";
import { clickResolvedTarget, type InteractionDeps, resolveActionTarget } from "./interaction"; import { clickResolvedTarget, type InteractionDeps, resolveActionTarget } from "./interaction";
import { import {
@@ -18,6 +23,7 @@ const DEFAULT_TIMEOUT_MS = 120_000;
export interface UploadDeps extends InteractionDeps { export interface UploadDeps extends InteractionDeps {
cdp: CdpRunner; cdp: CdpRunner;
sendToTab?: CaptureSuppressSendToTab;
} }
export async function handleUpload( export async function handleUpload(
@@ -41,19 +47,39 @@ export async function handleUpload(
const address = await resolveActionTarget(deps.cdp, ctx, target, params, "upload"); const address = await resolveActionTarget(deps.cdp, ctx, target, params, "upload");
if (isRpcError(address)) return address; if (isRpcError(address)) return address;
const timeoutMs = params.timeout_ms ?? DEFAULT_TIMEOUT_MS; const timeoutMs = params.timeout_ms ?? DEFAULT_TIMEOUT_MS;
const transaction = await uploadThroughActivatedFileInput({ const files = params.files.map((file) => file.staged_path as string);
cdp: deps.cdp, const mode = params.mode ?? "input";
actionTarget: address, let usedRef = address.usedRef;
files: params.files.map((file) => file.staged_path as string), let usedSelector = address.usedSelector;
timeoutMs, if (mode === "drop") {
signal: deps.signal, deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
trigger: () => clickResolvedTarget(ctx, address, {}, deps), const transaction = await uploadThroughFileDrop({
}); cdp: deps.cdp,
if (isRpcError(transaction)) return transaction; actionTarget: address,
files,
timeoutMs,
signal: deps.signal,
withOverlayHidden: (operation) =>
withExtensionOverlayHidden(target.tabId, operation, deps.sendToTab),
});
if (isRpcError(transaction)) return transaction;
} else {
const transaction = await uploadThroughActivatedFileInput({
cdp: deps.cdp,
actionTarget: address,
files,
timeoutMs,
signal: deps.signal,
trigger: () => clickResolvedTarget(ctx, address, {}, deps),
});
if (isRpcError(transaction)) return transaction;
usedRef = transaction.click.used_ref;
usedSelector = transaction.click.used_selector;
}
return { return {
tab_id: target.tabId, tab_id: target.tabId,
used_ref: transaction.click.used_ref, used_ref: usedRef,
used_selector: transaction.click.used_selector, used_selector: usedSelector,
file_names: params.files.map((file) => file.name), file_names: params.files.map((file) => file.name),
}; };
} }
apps/extension/src/transport/types.ts
+6
View File
@@ -36,6 +36,9 @@ export type RpcErrorReason =
| "file_input_probe_failed" | "file_input_probe_failed"
| "file_input_not_activated" | "file_input_not_activated"
| "set_file_input_failed" | "set_file_input_failed"
| "upload_mechanism_unsupported"
| "file_drop_target_unavailable"
| "file_drop_failed"
| "download_capture_failed" | "download_capture_failed"
| "transfer_outcome_unknown" | "transfer_outcome_unknown"
| "transfer_timeout" | "transfer_timeout"
@@ -518,12 +521,15 @@ export interface UploadFile {
staged_path?: string; staged_path?: string;
} }
export type UploadMode = "input" | "drop";
export interface UploadParams { export interface UploadParams {
session_id: string; session_id: string;
ref?: string; ref?: string;
selector?: string; selector?: string;
tab_id?: number; tab_id?: number;
files: UploadFile[]; files: UploadFile[];
mode?: UploadMode;
timeout_ms?: number; timeout_ms?: number;
} }
crates/bsk-cli/skill/SKILL.md
+7 -2
View File
@@ -210,18 +210,23 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
| Command | Summary | | Command | Summary |
|---------|---------| |---------|---------|
| `bsk upload <ref-or-selector> --file <path>` | Click one upload trigger and attach an agent-readable local file (`--file` is repeatable) | | `bsk upload <ref-or-selector> --file <path>` | Attach a local file through a file-input trigger; use `--mode drop` only for an explicit drop zone (`--file` is repeatable) |
| `bsk download <ref-or-selector> --out <path>` | Click one download trigger and copy the single completed file to an exact local path (`--overwrite` is opt-in) | | `bsk download <ref-or-selector> --out <path>` | Click one download trigger and copy the single completed file to an exact local path (`--overwrite` is opt-in) |
The agent/harness decides whether a file transfer is appropriate and which local path belongs to the task. Treat upload as disclosure of that file to the current website, and download as accepting website-controlled bytes onto the local filesystem. Use only paths that are necessary for the user's bounded goal. The agent/harness decides whether a file transfer is appropriate and which local path belongs to the task. Treat upload as disclosure of that file to the current website, and download as accepting website-controlled bytes onto the local filesystem. Use only paths that are necessary for the user's bounded goal.
BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted capabilities reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. Upload intercepts the native chooser for one transaction, locates the input activated in the resolved target's document, and assigns only the staged file paths. Download uniquely correlates one exact-target browser intent with one Chrome download in either event order, routes it through a daemon-minted relative directory, and lets the daemon validate and import it. Upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive. BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted capabilities reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. The default upload mode intercepts one chooser transaction, locates the file input activated in the resolved target's document, and assigns only the staged paths. `--mode drop` does not click: it delivers the same staged files through one native browser drop bound to the explicitly resolved drop zone. These mechanisms never fall back to each other automatically. Download uniquely correlates one exact-target browser intent with one Chrome download in either event order, routes it through a daemon-minted relative directory, and lets the daemon validate and import it. Upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive.
A successful drop command means Chrome dispatched the native file drop to the resolved zone; it does not prove that the site's application logic accepted the attachment. Observe the page once after the command, and do not repeat the drop when the attachment is already present.
Do not use `request-help` merely because a native file chooser or browser download is involved; try these commands first. For transfer failures, use the structured error instead of retrying blindly: Do not use `request-help` merely because a native file chooser or browser download is involved; try these commands first. For transfer failures, use the structured error instead of retrying blindly:
- `reason=file_input_not_activated` means the requested click did not activate exactly one `<input type="file">`; the page may use a non-input picker such as `window.showOpenFilePicker()`. Do not retry blindly. If human help is available, call `request-help` and tell the user the exact original local path to choose. The staged daemon path is internal and must not be shown to the user. - `reason=file_input_not_activated` means the requested click did not activate exactly one `<input type="file">`; the page may use a non-input picker such as `window.showOpenFilePicker()`. Do not retry blindly. If human help is available, call `request-help` and tell the user the exact original local path to choose. The staged daemon path is internal and must not be shown to the user.
- `reason=file_input_probe_failed` means BrowserSkill could not safely establish the browser-side upload transaction. Do not repeat the same action; use `request-help` when available. - `reason=file_input_probe_failed` means BrowserSkill could not safely establish the browser-side upload transaction. Do not repeat the same action; use `request-help` when available.
- `reason=set_file_input_failed` means BrowserSkill found the activated file input but Chrome rejected the staged path or assignment. Check the extension's file-URL access permission; otherwise use `request-help`. - `reason=set_file_input_failed` means BrowserSkill found the activated file input but Chrome rejected the staged path or assignment. Check the extension's file-URL access permission; otherwise use `request-help`.
- `reason=upload_mechanism_unsupported` means the browser cannot perform a native file drop. Do not retry the same drop; use a standard file-input trigger or `request-help`.
- `reason=file_drop_target_unavailable` means the specified point is no longer owned by the resolved drop zone. When `effect_state=none`, re-observe and select the visible drop zone itself; do not guess another target.
- `reason=file_drop_failed` means the native drop command failed. Do not retry when `effect_state=unknown`; first observe whether the page already received the file.
- `reason=download_capture_failed` means BrowserSkill could not attribute exactly one completed download to the requested target. Do not retry blindly or accept an unrelated browser download; use `request-help` when available. - `reason=download_capture_failed` means BrowserSkill could not attribute exactly one completed download to the requested target. Do not retry blindly or accept an unrelated browser download; use `request-help` when available.
- `effect_state=none` means BrowserSkill confirmed that no file-transfer effect was committed. Follow the accompanying reason; a corrected target or explicit human fallback may be attempted. - `effect_state=none` means BrowserSkill confirmed that no file-transfer effect was committed. Follow the accompanying reason; a corrected target or explicit human fallback may be attempted.
- `effect_state=unknown` means the browser may already have attached or created the file. Do not repeat the transfer. Observe the page if that can establish the result; otherwise stop and report the uncertainty. - `effect_state=unknown` means the browser may already have attached or created the file. Do not repeat the transfer. Observe the page if that can establish the result; otherwise stop and report the uncertainty.
crates/bsk-cli/src/cli/mod.rs
+1 -1
View File
@@ -182,7 +182,7 @@ pub enum Command {
/// Set `<select>` option values by `value` attribute. /// Set `<select>` option values by `value` attribute.
Select(SelectArgs), Select(SelectArgs),
/// Upload one or more local files through a page file chooser. /// Upload files through a page file input or explicit drop target.
Upload(UploadArgs), Upload(UploadArgs),
/// Capture one browser download and write it to a local path. /// Capture one browser download and write it to a local path.
crates/bsk-cli/src/cli/render_error.rs
+50
View File
@@ -52,6 +52,9 @@ pub mod reason {
pub const FILE_INPUT_PROBE_FAILED: &str = "file_input_probe_failed"; pub const FILE_INPUT_PROBE_FAILED: &str = "file_input_probe_failed";
pub const FILE_INPUT_NOT_ACTIVATED: &str = "file_input_not_activated"; pub const FILE_INPUT_NOT_ACTIVATED: &str = "file_input_not_activated";
pub const SET_FILE_INPUT_FAILED: &str = "set_file_input_failed"; pub const SET_FILE_INPUT_FAILED: &str = "set_file_input_failed";
pub const UPLOAD_MECHANISM_UNSUPPORTED: &str = "upload_mechanism_unsupported";
pub const FILE_DROP_TARGET_UNAVAILABLE: &str = "file_drop_target_unavailable";
pub const FILE_DROP_FAILED: &str = "file_drop_failed";
pub const DOWNLOAD_CAPTURE_FAILED: &str = "download_capture_failed"; pub const DOWNLOAD_CAPTURE_FAILED: &str = "download_capture_failed";
pub const TRANSFER_OUTCOME_UNKNOWN: &str = "transfer_outcome_unknown"; pub const TRANSFER_OUTCOME_UNKNOWN: &str = "transfer_outcome_unknown";
pub const TRANSFER_TIMEOUT: &str = "transfer_timeout"; pub const TRANSFER_TIMEOUT: &str = "transfer_timeout";
@@ -323,6 +326,30 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
), ),
exit_code: base.exit_code, exit_code: base.exit_code,
}, },
(ErrorCode::Unsupported, reason::UPLOAD_MECHANISM_UNSUPPORTED) => RenderInfo {
summary: "the browser does not support native file-drop upload",
hint: Some(
"do not retry the same drop; use a standard file-input target or `bsk request-help`",
),
exit_code: base.exit_code,
},
(
ErrorCode::PermissionDenied | ErrorCode::Timeout | ErrorCode::CdpFailed,
reason::FILE_DROP_TARGET_UNAVAILABLE,
) => RenderInfo {
summary: "the file-drop target could not be safely resolved",
hint: Some(
"rerun observe and select the visible drop zone itself; no file was delivered when effect_state is none",
),
exit_code: base.exit_code,
},
(ErrorCode::Timeout | ErrorCode::CdpFailed, reason::FILE_DROP_FAILED) => RenderInfo {
summary: "the browser could not complete the native file drop",
hint: Some(
"do not retry when effect_state is unknown; observe the page first, otherwise use `bsk request-help`",
),
exit_code: base.exit_code,
},
(ErrorCode::CdpFailed, reason::DOWNLOAD_CAPTURE_FAILED) => RenderInfo { (ErrorCode::CdpFailed, reason::DOWNLOAD_CAPTURE_FAILED) => RenderInfo {
summary: "the browser download could not be attributed or completed", summary: "the browser download could not be attributed or completed",
hint: Some( hint: Some(
@@ -487,6 +514,29 @@ mod tests {
); );
assert!(info.hint.unwrap().contains("do not retry")); assert!(info.hint.unwrap().contains("do not retry"));
let unsupported_drop =
serde_json::json!({ "reason": reason::UPLOAD_MECHANISM_UNSUPPORTED });
let info = info_for_error(ErrorCode::Unsupported, Some(&unsupported_drop));
assert_eq!(
info.summary,
"the browser does not support native file-drop upload"
);
assert!(info.hint.unwrap().contains("standard file-input"));
let unavailable_target =
serde_json::json!({ "reason": reason::FILE_DROP_TARGET_UNAVAILABLE });
let info = info_for_error(ErrorCode::PermissionDenied, Some(&unavailable_target));
assert_eq!(
info.summary,
"the file-drop target could not be safely resolved"
);
assert!(info.hint.unwrap().contains("effect_state is none"));
let failed_drop = serde_json::json!({ "reason": reason::FILE_DROP_FAILED });
let info = info_for_error(ErrorCode::Timeout, Some(&failed_drop));
assert!(info.summary.contains("native file drop"));
assert!(info.hint.unwrap().contains("effect_state is unknown"));
let download = serde_json::json!({ "reason": reason::DOWNLOAD_CAPTURE_FAILED }); let download = serde_json::json!({ "reason": reason::DOWNLOAD_CAPTURE_FAILED });
let info = info_for_error(ErrorCode::CdpFailed, Some(&download)); let info = info_for_error(ErrorCode::CdpFailed, Some(&download));
assert!(info.summary.contains("download could not be attributed")); assert!(info.summary.contains("download could not be attributed"));
crates/bsk-cli/src/cli/upload.rs
+24 -4
View File
@@ -10,19 +10,35 @@ use base64::Engine;
use bsk_protocol::Method; use bsk_protocol::Method;
use bsk_protocol::tools::{ use bsk_protocol::tools::{
TransferBeginParams, TransferBeginResult, TransferChunkParams, TransferChunkResult, TransferBeginParams, TransferBeginResult, TransferChunkParams, TransferChunkResult,
TransferIdParams, TransferReadyResult, TransferReleaseResult, UploadFile, UploadParams, TransferIdParams, TransferReadyResult, TransferReleaseResult, UploadFile, UploadMode,
UploadResult, UploadParams, UploadResult,
}; };
use clap::Args; use clap::{Args, ValueEnum};
use crate::cli::ensure_daemon::ensure_daemon; use crate::cli::ensure_daemon::ensure_daemon;
use crate::cli::error::{CliError, Format}; use crate::cli::error::{CliError, Format};
use crate::cli::interaction::split_target; use crate::cli::interaction::split_target;
use crate::cli::navigate::parse_timeout_ms; use crate::cli::navigate::parse_timeout_ms;
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum UploadModeArg {
#[default]
Input,
Drop,
}
impl From<UploadModeArg> for UploadMode {
fn from(value: UploadModeArg) -> Self {
match value {
UploadModeArg::Input => Self::Input,
UploadModeArg::Drop => Self::Drop,
}
}
}
#[derive(Debug, Clone, Args)] #[derive(Debug, Clone, Args)]
pub struct UploadArgs { pub struct UploadArgs {
/// Snapshot ref (`@e3`) or CSS selector for the file input / chooser trigger. /// Snapshot ref (`@e3`) or CSS selector for the chooser trigger / drop target.
pub target: Option<String>, pub target: Option<String>,
#[arg(long = "ref")] #[arg(long = "ref")]
pub ref_: Option<String>, pub ref_: Option<String>,
@@ -31,6 +47,9 @@ pub struct UploadArgs {
/// Local file to upload. Repeat for a multiple-file input. /// Local file to upload. Repeat for a multiple-file input.
#[arg(long = "file", required = true)] #[arg(long = "file", required = true)]
pub files: Vec<PathBuf>, pub files: Vec<PathBuf>,
/// Browser mechanism used to deliver the staged files.
#[arg(long, value_enum, default_value_t)]
pub mode: UploadModeArg,
#[arg(long)] #[arg(long)]
pub session: String, pub session: String,
#[arg(long = "tab-id")] #[arg(long = "tab-id")]
@@ -73,6 +92,7 @@ pub fn dispatch(args: UploadArgs, format: Format) -> Result<(), CliError> {
staged_path: None, staged_path: None,
}) })
.collect(), .collect(),
mode: args.mode.into(),
timeout_ms: Some(args.timeout), timeout_ms: Some(args.timeout),
}; };
let result = crate::cli::business_rpc::call::<_, UploadResult>( let result = crate::cli::business_rpc::call::<_, UploadResult>(
crates/bsk-cli/tests/cli_parse.rs
+49
View File
@@ -6,6 +6,7 @@ use bsk::cli::daemon::{DaemonCmd, parse_duration};
use bsk::cli::navigate::NavigateCmd; use bsk::cli::navigate::NavigateCmd;
use bsk::cli::record::{RecordCmd, RecordSub}; use bsk::cli::record::{RecordCmd, RecordSub};
use bsk::cli::session::{SessionCmd, SessionSub}; use bsk::cli::session::{SessionCmd, SessionSub};
use bsk::cli::upload::UploadModeArg;
use bsk::{Cli, Command}; use bsk::{Cli, Command};
use clap::Parser; use clap::Parser;
@@ -13,6 +14,54 @@ fn parse(args: &[&str]) -> Cli {
Cli::try_parse_from(args).expect("clap parse should succeed") Cli::try_parse_from(args).expect("clap parse should succeed")
} }
#[test]
fn parses_upload_modes() {
let cli = parse(&[
"bsk",
"upload",
"@e1",
"--file",
"image.png",
"--session",
"s1",
]);
let Command::Upload(default_args) = cli.command else {
panic!("expected upload subcommand");
};
assert_eq!(default_args.mode, UploadModeArg::Input);
let cli = parse(&[
"bsk",
"upload",
"@e1",
"--file",
"image.png",
"--session",
"s1",
"--mode",
"drop",
]);
let Command::Upload(drop_args) = cli.command else {
panic!("expected upload subcommand");
};
assert_eq!(drop_args.mode, UploadModeArg::Drop);
assert!(
Cli::try_parse_from([
"bsk",
"upload",
"@e1",
"--file",
"image.png",
"--session",
"s1",
"--mode",
"auto",
])
.is_err()
);
}
#[test] #[test]
fn parses_daemon_start_with_defaults() { fn parses_daemon_start_with_defaults() {
let cli = parse(&["bsk", "daemon", "start"]); let cli = parse(&["bsk", "daemon", "start"]);
crates/bsk-protocol/schema/tool_upload_params.json
+10
View File
@@ -13,6 +13,9 @@
"$ref": "#/definitions/UploadFile" "$ref": "#/definitions/UploadFile"
} }
}, },
"mode": {
"$ref": "#/definitions/UploadMode"
},
"ref": { "ref": {
"type": [ "type": [
"string", "string",
@@ -65,6 +68,13 @@
"type": "string" "type": "string"
} }
} }
},
"UploadMode": {
"type": "string",
"enum": [
"input",
"drop"
]
} }
} }
} }
crates/bsk-protocol/src/tools/file_transfer.rs
+37
View File
@@ -15,6 +15,20 @@ pub struct UploadFile {
pub staged_path: Option<String>, pub staged_path: Option<String>,
} }
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum UploadMode {
#[default]
Input,
Drop,
}
impl UploadMode {
fn is_input(&self) -> bool {
matches!(self, Self::Input)
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct UploadParams { pub struct UploadParams {
pub session_id: String, pub session_id: String,
@@ -30,6 +44,8 @@ pub struct UploadParams {
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub tab_id: Option<i64>, pub tab_id: Option<i64>,
pub files: Vec<UploadFile>, pub files: Vec<UploadFile>,
#[serde(default, skip_serializing_if = "UploadMode::is_input")]
pub mode: UploadMode,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub timeout_ms: Option<u32>, pub timeout_ms: Option<u32>,
} }
@@ -151,11 +167,32 @@ mod tests {
name: "image.png".into(), name: "image.png".into(),
staged_path: None, staged_path: None,
}], }],
mode: UploadMode::Input,
timeout_ms: None, timeout_ms: None,
}) })
.unwrap(); .unwrap();
assert_eq!(value["ref"], "@e3"); assert_eq!(value["ref"], "@e3");
assert!(value["files"][0].get("staged_path").is_none()); assert!(value["files"][0].get("staged_path").is_none());
assert!(value.get("mode").is_none());
}
#[test]
fn upload_drop_mode_is_explicit_on_the_wire() {
let value = serde_json::to_value(UploadParams {
session_id: "s1".into(),
ref_: Some("@e3".into()),
selector: None,
tab_id: None,
files: vec![UploadFile {
transfer_id: "tr_1".into(),
name: "image.png".into(),
staged_path: None,
}],
mode: UploadMode::Drop,
timeout_ms: None,
})
.unwrap();
assert_eq!(value["mode"], "drop");
} }
#[test] #[test]
docs/architecture.md
+1 -1
View File
@@ -148,7 +148,7 @@ flowchart LR
- The invoking agent/harness decides whether a transfer is authorized and supplies the task-local source or destination path. - The invoking agent/harness decides whether a transfer is authorized and supplies the task-local source or destination path.
- The CLI is the only component that reads an upload source or writes the final download destination. Before browser dispatch it owns rollback of partially staged uploads; after dispatch, ownership moves to the session because a transport timeout cannot prove that Chrome did not attach the file. Download output becomes visible through one atomic commit, and replacement is opt-in without a pre-delete window. The extension never receives either agent-facing path. - The CLI is the only component that reads an upload source or writes the final download destination. Before browser dispatch it owns rollback of partially staged uploads; after dispatch, ownership moves to the session because a transport timeout cannot prove that Chrome did not attach the file. Download output becomes visible through one atomic commit, and replacement is opt-in without a pre-delete window. The extension never receives either agent-facing path.
- The daemon is the authority for storage capabilities and limits. It issues opaque session-scoped transfer IDs, stages bounded chunks in a private runtime directory, and injects only private staged upload paths. For download it mints one relative Chrome directory capability. Only after validating the reported path, file type, symlink boundary, and authoritative byte limit does it take ownership of browser-file cleanup and import the bytes. - The daemon is the authority for storage capabilities and limits. It issues opaque session-scoped transfer IDs, stages bounded chunks in a private runtime directory, and injects only private staged upload paths. For download it mints one relative Chrome directory capability. Only after validating the reported path, file type, symlink boundary, and authoritative byte limit does it take ownership of browser-file cleanup and import the bytes.
- The extension owns only the browser transaction. Both tools resolve one `ResolvedActionTarget` and use that same target for protocol setup and the triggering click. Upload arms Chrome's chooser interception before clicking, then accepts either an exact `Page.fileChooserOpened` input node or an independent probe anchored in the trigger node's own document; one verified input is committed with `DOM.setFileInputFiles`. A non-input picker is rejected immediately as unsupported. Download correlates exact-target CDP intent and `chrome.downloads` filename candidates in either arrival order, claims only one unique match, and never cancels an unclaimed candidate. - The extension owns only the browser transaction. Every transfer resolves one `ResolvedActionTarget`. The default upload mechanism arms Chrome's chooser interception before clicking, then accepts either an exact `Page.fileChooserOpened` input node or an independent probe anchored in the trigger node's document; one verified input is committed with `DOM.setFileInputFiles`, while a non-input picker is rejected immediately. Explicit drop mode performs no click and never falls back to the chooser mechanism: after geometry resolution it temporarily excludes BrowserSkill's own overlay, verifies that the resolved drop zone still owns its local action point, and sends one native `dragEnter` / `dragOver` / `drop` transaction to that node's CDP target before restoring the overlay. OOPIF drops use target-local coordinates rather than top-level click coordinates. Download correlates exact-target CDP intent and `chrome.downloads` filename candidates in either arrival order, claims only one unique match, and never cancels an unclaimed candidate.
- Browser-side operations report `effect_state` (`none`, `committed`, or `unknown`), `phase`, and `cleanup_state`. Confirmed success wins over a late cancel; an unknown effect is preserved across timeout or transport loss and must not be retried blindly. A transfer deadline sends cancellation to the extension and keeps the session queue occupied for bounded compensation rather than abandoning an in-flight browser effect. - Browser-side operations report `effect_state` (`none`, `committed`, or `unknown`), `phase`, and `cleanup_state`. Confirmed success wins over a late cancel; an unknown effect is preserved across timeout or transport loss and must not be retried blindly. A transfer deadline sends cancellation to the extension and keeps the session queue occupied for bounded compensation rather than abandoning an in-flight browser effect.
- Download staging is released after CLI commit. Upload staging remains until session teardown because the page may read an attached file only on a later form submission. Remaining staging is released on session stop/browser disconnect and on daemon startup after a crash. BrowserSkill does not inspect content or decide whether a transfer is appropriate. - Download staging is released after CLI commit. Upload staging remains until session teardown because the page may read an attached file only on a later form submission. Remaining staging is released on session stop/browser disconnect and on daemon startup after a crash. BrowserSkill does not inspect content or decide whether a transfer is appropriate.
skill/SKILL.md
+7 -2
View File
@@ -210,18 +210,23 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
| Command | Summary | | Command | Summary |
|---------|---------| |---------|---------|
| `bsk upload <ref-or-selector> --file <path>` | Click one upload trigger and attach an agent-readable local file (`--file` is repeatable) | | `bsk upload <ref-or-selector> --file <path>` | Attach a local file through a file-input trigger; use `--mode drop` only for an explicit drop zone (`--file` is repeatable) |
| `bsk download <ref-or-selector> --out <path>` | Click one download trigger and copy the single completed file to an exact local path (`--overwrite` is opt-in) | | `bsk download <ref-or-selector> --out <path>` | Click one download trigger and copy the single completed file to an exact local path (`--overwrite` is opt-in) |
The agent/harness decides whether a file transfer is appropriate and which local path belongs to the task. Treat upload as disclosure of that file to the current website, and download as accepting website-controlled bytes onto the local filesystem. Use only paths that are necessary for the user's bounded goal. The agent/harness decides whether a file transfer is appropriate and which local path belongs to the task. Treat upload as disclosure of that file to the current website, and download as accepting website-controlled bytes onto the local filesystem. Use only paths that are necessary for the user's bounded goal.
BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted capabilities reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. Upload intercepts the native chooser for one transaction, locates the input activated in the resolved target's document, and assigns only the staged file paths. Download uniquely correlates one exact-target browser intent with one Chrome download in either event order, routes it through a daemon-minted relative directory, and lets the daemon validate and import it. Upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive. BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted capabilities reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. The default upload mode intercepts one chooser transaction, locates the file input activated in the resolved target's document, and assigns only the staged paths. `--mode drop` does not click: it delivers the same staged files through one native browser drop bound to the explicitly resolved drop zone. These mechanisms never fall back to each other automatically. Download uniquely correlates one exact-target browser intent with one Chrome download in either event order, routes it through a daemon-minted relative directory, and lets the daemon validate and import it. Upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive.
A successful drop command means Chrome dispatched the native file drop to the resolved zone; it does not prove that the site's application logic accepted the attachment. Observe the page once after the command, and do not repeat the drop when the attachment is already present.
Do not use `request-help` merely because a native file chooser or browser download is involved; try these commands first. For transfer failures, use the structured error instead of retrying blindly: Do not use `request-help` merely because a native file chooser or browser download is involved; try these commands first. For transfer failures, use the structured error instead of retrying blindly:
- `reason=file_input_not_activated` means the requested click did not activate exactly one `<input type="file">`; the page may use a non-input picker such as `window.showOpenFilePicker()`. Do not retry blindly. If human help is available, call `request-help` and tell the user the exact original local path to choose. The staged daemon path is internal and must not be shown to the user. - `reason=file_input_not_activated` means the requested click did not activate exactly one `<input type="file">`; the page may use a non-input picker such as `window.showOpenFilePicker()`. Do not retry blindly. If human help is available, call `request-help` and tell the user the exact original local path to choose. The staged daemon path is internal and must not be shown to the user.
- `reason=file_input_probe_failed` means BrowserSkill could not safely establish the browser-side upload transaction. Do not repeat the same action; use `request-help` when available. - `reason=file_input_probe_failed` means BrowserSkill could not safely establish the browser-side upload transaction. Do not repeat the same action; use `request-help` when available.
- `reason=set_file_input_failed` means BrowserSkill found the activated file input but Chrome rejected the staged path or assignment. Check the extension's file-URL access permission; otherwise use `request-help`. - `reason=set_file_input_failed` means BrowserSkill found the activated file input but Chrome rejected the staged path or assignment. Check the extension's file-URL access permission; otherwise use `request-help`.
- `reason=upload_mechanism_unsupported` means the browser cannot perform a native file drop. Do not retry the same drop; use a standard file-input trigger or `request-help`.
- `reason=file_drop_target_unavailable` means the specified point is no longer owned by the resolved drop zone. When `effect_state=none`, re-observe and select the visible drop zone itself; do not guess another target.
- `reason=file_drop_failed` means the native drop command failed. Do not retry when `effect_state=unknown`; first observe whether the page already received the file.
- `reason=download_capture_failed` means BrowserSkill could not attribute exactly one completed download to the requested target. Do not retry blindly or accept an unrelated browser download; use `request-help` when available. - `reason=download_capture_failed` means BrowserSkill could not attribute exactly one completed download to the requested target. Do not retry blindly or accept an unrelated browser download; use `request-help` when available.
- `effect_state=none` means BrowserSkill confirmed that no file-transfer effect was committed. Follow the accompanying reason; a corrected target or explicit human fallback may be attempted. - `effect_state=none` means BrowserSkill confirmed that no file-transfer effect was committed. Follow the accompanying reason; a corrected target or explicit human fallback may be attempted.
- `effect_state=unknown` means the browser may already have attached or created the file. Do not repeat the transfer. Observe the page if that can establish the result; otherwise stop and report the uncertainty. - `effect_state=unknown` means the browser may already have attached or created the file. Do not repeat the transfer. Observe the page if that can establish the result; otherwise stop and report the uncertainty.