figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(bsk): support drop to upload

Browse Source
This commit is contained in:
Ljy-0827
2026-08-24 16:55:16 +08:00
parent c74c119a82
commit 8822bba9a5
21 changed files with 997 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/extension/e2e/fixtures/file-transfer.html
+81
View File
@@ -0,0 +1,81 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<title>BrowserSkill file-transfer fixture</title>
<style>
body { font: 16px system-ui; max-width: 760px; margin: 48px auto; }
.actions { display: flex; gap: 12px; align-items: center; margin: 24px 0; }
button, a { padding: 10px 16px; border: 1px solid #777; border-radius: 8px; }
a[aria-disabled="true"] { opacity: .45; pointer-events: none; }
#drop { padding: 36px; border: 3px dashed #777; border-radius: 12px; text-align: center; }
#drop.dragging { border-color: #2563eb; background: #eff6ff; }
img { display: block; max-width: 480px; max-height: 360px; margin-top: 20px; }
</style>
</head>
<body>
<h1>Upload then download the same image</h1>
<p id="status">No image selected</p>
<div class="actions">
<button id="upload" type="button">Upload through file input</button>
<input id="file" type="file" accept="image/png" hidden />
<a id="download" aria-disabled="true">Download uploaded image</a>
</div>
<div id="drop" role="button" tabindex="0">Drop an image here</div>
<img id="preview" alt="Uploaded image preview" hidden />
<script>
const input = document.querySelector('#file');
const upload = document.querySelector('#upload');
const drop = document.querySelector('#drop');
const download = document.querySelector('#download');
const preview = document.querySelector('#preview');
const status = document.querySelector('#status');
let objectUrl;
window.__bskTransferFixture = { events: [], result: null };
function accept(file, mechanism) {
if (!file) return;
if (objectUrl) URL.revokeObjectURL(objectUrl);
objectUrl = URL.createObjectURL(file);
preview.src = objectUrl;
preview.hidden = false;
download.href = objectUrl;
download.download = file.name;
download.removeAttribute('aria-disabled');
status.textContent = `Ready via ${mechanism}: ${file.name} (${file.size} bytes)`;
window.__bskTransferFixture.result = {
mechanism,
name: file.name,
size: file.size,
type: file.type,
hasPathProperty: 'path' in file,
};
}
upload.addEventListener('click', () => input.click());
input.addEventListener('change', () => accept(input.files[0], 'input'));
for (const type of ['dragenter', 'dragover']) {
drop.addEventListener(type, event => {
const types = [...event.dataTransfer.types];
window.__bskTransferFixture.events.push({ type, types });
if (!types.includes('Files')) return;
event.preventDefault();
drop.classList.add('dragging');
});
}
drop.addEventListener('dragleave', () => drop.classList.remove('dragging'));
drop.addEventListener('drop', event => {
window.__bskTransferFixture.events.push({
type: 'drop',
types: [...event.dataTransfer.types],
fileCount: event.dataTransfer.files.length,
});
event.preventDefault();
drop.classList.remove('dragging');
accept(event.dataTransfer.files[0], 'drop');
});
</script>
</body>
</html>
apps/extension/src/lib/__tests__/capture-suppress-bridge.test.ts
+6 -6
View File
@@ -2,7 +2,7 @@ import { describe, expect, it, vi } from "vitest";
import {
CAPTURE_SUPPRESS,
type CaptureSuppressMessage,
withOverlaysHiddenForCapture,
withExtensionOverlayHidden,
} from "../capture-suppress-bridge";
function recordingSendToTab(events: string[]) {
@@ -12,12 +12,12 @@ function recordingSendToTab(events: string[]) {
});
}
describe("withOverlaysHiddenForCapture", () => {
describe("withExtensionOverlayHidden", () => {
it("sends begin → fn → end and returns fn's result", async () => {
const events: string[] = [];
const sendToTab = recordingSendToTab(events);
const result = await withOverlaysHiddenForCapture(
const result = await withExtensionOverlayHidden(
7,
async () => {
events.push("capture");
@@ -37,7 +37,7 @@ describe("withOverlaysHiddenForCapture", () => {
const sendToTab = recordingSendToTab(events);
await expect(
withOverlaysHiddenForCapture(
withExtensionOverlayHidden(
7,
async () => {
events.push("capture");
@@ -55,7 +55,7 @@ describe("withOverlaysHiddenForCapture", () => {
});
const fn = vi.fn(async () => "shot");
const result = await withOverlaysHiddenForCapture(7, fn, sendToTab);
const result = await withExtensionOverlayHidden(7, fn, sendToTab);
expect(result).toBe("shot");
expect(fn).toHaveBeenCalledTimes(1);
@@ -69,7 +69,7 @@ describe("withOverlaysHiddenForCapture", () => {
return { type: CAPTURE_SUPPRESS, ok: true };
});
const result = await withOverlaysHiddenForCapture(7, async () => "shot", sendToTab);
const result = await withExtensionOverlayHidden(7, async () => "shot", sendToTab);
expect(result).toBe("shot");
expect(sendToTab).toHaveBeenCalledTimes(2);
apps/extension/src/lib/capture-suppress-bridge.ts
+5 -5
View File
@@ -1,9 +1,9 @@
/**
* Wire protocol and background-side helper for hiding the extension's
* in-page overlay while a screenshot is captured, so captured frames only
* contain the page itself.
* Wire protocol and background-side helper for temporarily excluding the
* extension's in-page overlay. Screenshots use it to avoid captured chrome;
* coordinate-driven native input uses it so Chrome hit-tests the page.
*
* The background wraps every capture in `withOverlaysHiddenForCapture`,
* The background wraps each bounded operation in `withExtensionOverlayHidden`,
* which sends `begin` (the content script hides the overlay host and only
* acks once the compositor has produced an overlay-free frame) and always
* follows up with `end`, even when the capture throws. Tabs without the
@@ -46,7 +46,7 @@ const defaultSendToTab: CaptureSuppressSendToTab = (tabId, message) =>
* (or any other `begin` failure) simply skips suppression — there is no
* overlay to hide in that tab.
*/
export async function withOverlaysHiddenForCapture<T>(
export async function withExtensionOverlayHidden<T>(
tabId: number,
fn: () => Promise<T>,
sendToTab: CaptureSuppressSendToTab = defaultSendToTab,
apps/extension/src/tools/__tests__/file-transfer.test.ts
+282 -2
View File
@@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import { type DownloadsApi, handleDownload } from "../download";
import { captureBrowserDownload } from "../download-capture";
import { uploadThroughFileDrop } from "../file-drop-transaction";
import { uploadThroughActivatedFileInput } from "../file-input-transaction";
import type { ResolvedActionTarget } from "../interaction";
import type { CdpRunner } from "../shared";
@@ -37,16 +38,20 @@ function fakeEvent<T extends (...args: never[]) => unknown>() {
};
}
function actionTarget(frameId?: string): ResolvedActionTarget {
function actionTarget(frameId?: string, sessionId?: string): ResolvedActionTarget {
return {
tab: { tabId: 4, windowId: 100, active: true },
backendNodeId: 123,
cdpTarget: { tabId: 4 },
cdpTarget: { tabId: 4, ...(sessionId ? { sessionId } : {}) },
...(frameId ? { frameId } : {}),
usedRef: "e3",
};
}
function withoutOverlay<T>(operation: () => Promise<T>): Promise<T> {
return operation();
}
function uploadCdp(
options: {
inputCount?: number;
@@ -59,6 +64,7 @@ function uploadCdp(
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
calls.push({ method, params });
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "Page.setInterceptFileChooserDialog") return {};
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
@@ -110,6 +116,50 @@ function uploadCdp(
};
}
function dropCdp(
options: {
targetContainsPoint?: boolean;
dispatchError?: (type: string) => Error | Promise<never> | undefined;
onCall?: (method: string) => void;
} = {},
) {
const calls: Array<{ method: string; params?: object }> = [];
const detach = vi.fn(async () => {});
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
calls.push({ method, params });
options.onCall?.(method);
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
}
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] };
}
if (method === "DOM.getNodeForLocation") return { backendNodeId: 124 };
if (method === "DOM.resolveNode") {
const id = (params as { backendNodeId?: number }).backendNodeId;
return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } };
}
if (method === "Runtime.callFunctionOn") {
return { result: { value: options.targetContainsPoint ?? true } };
}
if (method === "Input.dispatchDragEvent") {
const type = (params as { type?: string }).type ?? "";
const error = options.dispatchError?.(type);
if (error instanceof Promise) return error;
if (error) throw error;
return {};
}
if (method === "Runtime.releaseObjectGroup") return {};
throw new Error(`unexpected drop CDP command ${method}`);
});
const cdp: CdpRunner = {
send: send as unknown as CdpRunner["send"],
detach,
};
return { calls, cdp, detach };
}
describe("file transfer tools", () => {
it("captures the file input activated by the requested click and injects only staged paths", async () => {
const manager = sessions();
@@ -271,6 +321,236 @@ describe("file transfer tools", () => {
expect(detach).toHaveBeenCalledWith(4);
});
it("delivers staged files through one explicit native drop without clicking", async () => {
const manager = sessions();
const ctx = await manager.start("s1");
ctx.refStore.set("e3", 123, { tabId: 4 });
const lifecycle: string[] = [];
const { cdp, calls } = dropCdp({ onCall: (method) => lifecycle.push(method) });
const overlayPhases: string[] = [];
const result = await handleUpload(
manager,
{
session_id: "s1",
ref: "@e3",
mode: "drop",
files: [{ transfer_id: "tr_1", name: "one.png", staged_path: "/stage/one.png" }],
},
{
cdp,
tabsApi: tabsApi(),
sendToTab: vi.fn(async (_tabId, message) => {
overlayPhases.push(message.phase);
lifecycle.push(`overlay:${message.phase}`);
}),
},
);
expect(result).toMatchObject({ tab_id: 4, used_ref: "e3", file_names: ["one.png"] });
expect(calls.filter((call) => call.method === "Input.dispatchDragEvent")).toEqual([
{
method: "Input.dispatchDragEvent",
params: {
type: "dragEnter",
x: 110,
y: 70,
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
},
},
{
method: "Input.dispatchDragEvent",
params: {
type: "dragOver",
x: 110,
y: 70,
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
},
},
{
method: "Input.dispatchDragEvent",
params: {
type: "drop",
x: 110,
y: 70,
data: { items: [], files: ["/stage/one.png"], dragOperationsMask: 1 },
},
},
]);
expect(calls.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
expect(overlayPhases).toEqual(["begin", "end"]);
expect(lifecycle.indexOf("DOM.getContentQuads")).toBeLessThan(
lifecycle.indexOf("overlay:begin"),
);
expect(lifecycle.indexOf("overlay:begin")).toBeLessThan(
lifecycle.indexOf("DOM.getNodeForLocation"),
);
expect(lifecycle.lastIndexOf("Input.dispatchDragEvent")).toBeLessThan(
lifecycle.indexOf("overlay:end"),
);
expect(lifecycle.indexOf("overlay:end")).toBeLessThan(
lifecycle.indexOf("Runtime.releaseObjectGroup"),
);
});
it("fails closed before dragging when another element owns the action point", async () => {
const { cdp, calls } = dropCdp({ targetContainsPoint: false });
const result = await uploadThroughFileDrop({
cdp,
actionTarget: actionTarget(),
files: ["/stage/one.png"],
timeoutMs: 100,
withOverlayHidden: withoutOverlay,
});
expect(result).toMatchObject({
code: "permission_denied",
data: {
reason: "file_drop_target_unavailable",
mechanism: "drop",
effect_state: "none",
phase: "resolve_target",
},
});
expect(calls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false);
});
it("dispatches an OOPIF file drop in the child target's local coordinates", async () => {
const rootCalls: Array<{ method: string; params?: object }> = [];
const childCalls: Array<{ method: string; params?: object }> = [];
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method, params) => {
rootCalls.push({ method, params });
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.getBoxModel") {
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
}
throw new Error(`unexpected root command ${method}`);
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method, params) => {
childCalls.push({ method, params });
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } };
}
if (method === "DOM.getNodeForLocation") {
return { backendNodeId: 124, frameId: "child" };
}
if (method === "DOM.resolveNode") {
const id = (params as { backendNodeId?: number }).backendNodeId;
return { object: { objectId: id === 123 ? "drop-target" : "hit-target" } };
}
if (method === "Runtime.callFunctionOn") return { result: { value: true } };
if (method === "Input.dispatchDragEvent" || method === "Runtime.releaseObjectGroup") {
return {};
}
throw new Error(`unexpected child command ${method}`);
}) as CdpRunner["sendToTarget"],
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
})),
};
const result = await uploadThroughFileDrop({
cdp,
actionTarget: actionTarget("child", "child-session"),
files: ["/stage/one.png"],
timeoutMs: 100,
withOverlayHidden: withoutOverlay,
});
expect(result).toMatchObject({ x: 110, y: 70 });
expect(
childCalls
.filter((call) => call.method === "Input.dispatchDragEvent")
.map((call) => call.params),
).toEqual([
expect.objectContaining({ type: "dragEnter", x: 110, y: 70 }),
expect.objectContaining({ type: "dragOver", x: 110, y: 70 }),
expect.objectContaining({ type: "drop", x: 110, y: 70 }),
]);
expect(rootCalls.some((call) => call.method === "Input.dispatchDragEvent")).toBe(false);
});
it("reports a missing native drag command as unsupported before commit", async () => {
const { cdp } = dropCdp({
dispatchError: (type) =>
type === "dragEnter" ? new Error("'Input.dispatchDragEvent' wasn't found") : undefined,
});
const result = await uploadThroughFileDrop({
cdp,
actionTarget: actionTarget(),
files: ["/stage/one.png"],
timeoutMs: 100,
withOverlayHidden: withoutOverlay,
});
expect(result).toMatchObject({
code: "unsupported",
data: {
reason: "upload_mechanism_unsupported",
effect_state: "none",
phase: "drag_enter",
},
});
});
it("cancels an unresolved drag before returning a pre-commit timeout", async () => {
const { cdp, calls } = dropCdp({
dispatchError: (type) => (type === "dragEnter" ? new Promise<never>(() => {}) : undefined),
});
const result = await uploadThroughFileDrop({
cdp,
actionTarget: actionTarget(),
files: ["/stage/one.png"],
timeoutMs: 10,
withOverlayHidden: withoutOverlay,
});
expect(result).toMatchObject({
code: "timeout",
data: { reason: "file_drop_failed", effect_state: "none", phase: "drag_enter" },
});
expect(calls).toContainEqual({
method: "Input.dispatchDragEvent",
params: expect.objectContaining({ type: "dragCancel" }),
});
});
it("marks an unresolved drop unknown and detaches browser state", async () => {
const { cdp, detach } = dropCdp({
dispatchError: (type) => (type === "drop" ? new Promise<never>(() => {}) : undefined),
});
const result = await uploadThroughFileDrop({
cdp,
actionTarget: actionTarget(),
files: ["/stage/one.png"],
timeoutMs: 10,
withOverlayHidden: withoutOverlay,
});
expect(result).toMatchObject({
code: "timeout",
data: { reason: "file_drop_failed", effect_state: "unknown", phase: "drop" },
});
expect(detach).toHaveBeenCalledWith(4);
});
it("routes one exact-target download through a browser-relative capability", async () => {
const manager = sessions();
const ctx = await manager.start("s1");
apps/extension/src/tools/__tests__/frame-geometry.test.ts
+2
View File
@@ -169,6 +169,7 @@ describe("frame geometry projection", () => {
expect(geometry).toMatchObject({
topBounds: { x: 100, y: 100, width: 200, height: 100 },
actionPoint: { x: 200, y: 150 },
targetActionPoint: { x: 100, y: 50 },
});
});
@@ -261,6 +262,7 @@ describe("frame geometry projection", () => {
expect(geometry).toMatchObject({
topBounds: { x: 100, y: 100, width: 200, height: 100 },
actionPoint: { x: 200, y: 150 },
targetActionPoint: { x: 200, y: 150 },
});
});
apps/extension/src/tools/file-drop-transaction.ts
+319
View File
@@ -0,0 +1,319 @@
// One native file-drop transaction bound to an explicit page target. CDP drag
// events are sent to the target that owns the node: root targets use top-level
// coordinates, while OOPIF targets require coordinates local to their session.
import type { RpcError, TransferEffectState } from "@/transport/types";
import { transferError } from "./errors";
import { resolveNodeGeometry } from "./frame-geometry";
import type { ResolvedActionTarget } from "./interaction";
import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared";
import { BoundedWaitError, waitBounded } from "./transfer-transaction";
const CLEANUP_TIMEOUT_MS = 1_000;
type FileDropPhase = "resolve_target" | "drag_enter" | "drag_over" | "drop" | "cleanup";
interface RuntimeReply {
result?: { value?: unknown; objectId?: string };
exceptionDetails?: { text?: string; exception?: { description?: string } };
}
interface DragData {
items: [];
files: string[];
dragOperationsMask: number;
}
export interface FileDropTransactionOptions {
cdp: CdpRunner;
actionTarget: ResolvedActionTarget;
files: string[];
timeoutMs: number;
signal?: AbortSignal;
withOverlayHidden: <T>(operation: () => Promise<T>) => Promise<T>;
}
export interface FileDropTransactionResult {
x: number;
y: number;
}
function failure(
code: RpcError["code"],
reason: "upload_mechanism_unsupported" | "file_drop_target_unavailable" | "file_drop_failed",
message: string,
effectState: TransferEffectState,
phase: FileDropPhase,
): RpcError {
const error = transferError(code, reason, message, { effectState, phase });
error.data = { ...error.data, mechanism: "drop" };
return error;
}
function enrichFailure(
error: RpcError,
effectState: TransferEffectState,
phase: FileDropPhase,
): RpcError {
return {
...error,
data: {
...error.data,
mechanism: "drop",
effect_state: effectState,
phase,
},
};
}
function messageOf(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
function methodUnavailable(error: unknown): boolean {
return /(?:method|command).*(?:not found|wasn't found|unsupported)|wasn't found|-32601/i.test(
messageOf(error),
);
}
function runtimeError(reply: RuntimeReply, fallback: string): Error | null {
if (!reply.exceptionDetails) return null;
return new Error(
reply.exceptionDetails.exception?.description ?? reply.exceptionDetails.text ?? fallback,
);
}
async function pointBelongsToTarget(
options: FileDropTransactionOptions,
x: number,
y: number,
objectGroup: string,
): Promise<boolean> {
const target = options.actionTarget.cdpTarget;
const hit = await sendToCdpTarget<{ backendNodeId?: number; frameId?: string }>(
options.cdp,
target,
"DOM.getNodeForLocation",
{ x: Math.round(x), y: Math.round(y), includeUserAgentShadowDOM: true },
);
if (typeof hit.backendNodeId !== "number") return false;
if (options.actionTarget.frameId && hit.frameId !== options.actionTarget.frameId) return false;
const [resolvedTarget, resolvedHit] = await Promise.all([
sendToCdpTarget<{ object?: { objectId?: string } }>(options.cdp, target, "DOM.resolveNode", {
backendNodeId: options.actionTarget.backendNodeId,
objectGroup,
}),
sendToCdpTarget<{ object?: { objectId?: string } }>(options.cdp, target, "DOM.resolveNode", {
backendNodeId: hit.backendNodeId,
objectGroup,
}),
]);
const targetObjectId = resolvedTarget.object?.objectId;
const hitObjectId = resolvedHit.object?.objectId;
if (!targetObjectId || !hitObjectId) return false;
const reply = await sendToCdpTarget<RuntimeReply>(options.cdp, target, "Runtime.callFunctionOn", {
objectId: targetObjectId,
functionDeclaration: `function(hit) {
for (let node = hit; node; ) {
if (node === this) return true;
if (this.contains?.(node)) return true;
const root = node.getRootNode?.();
node = node.parentNode || (root && root !== node ? root.host : null);
}
return false;
}`,
arguments: [{ objectId: hitObjectId }],
returnByValue: true,
});
const error = runtimeError(reply, "failed to verify file-drop target");
if (error) throw error;
return reply.result?.value === true;
}
export async function uploadThroughFileDrop(
options: FileDropTransactionOptions,
): Promise<FileDropTransactionResult | RpcError> {
const deadline = Date.now() + options.timeoutMs;
const target = options.actionTarget.cdpTarget;
const objectGroup = `bsk-file-drop-${crypto.randomUUID()}`;
const dragData: DragData = {
items: [],
files: options.files,
dragOperationsMask: 1,
};
let dragStarted = false;
let dropSent = false;
let dragPoint = { x: 0, y: 0 };
let cleanupFailed = false;
let outcome: FileDropTransactionResult | RpcError = failure(
"protocol_error",
"file_drop_failed",
"file-drop transaction ended without an outcome",
"none",
"cleanup",
);
try {
const geometry = await waitBounded(
resolveNodeGeometry(
options.cdp,
options.actionTarget.tab.tabId,
{
target,
backendNodeId: options.actionTarget.backendNodeId,
...(options.actionTarget.frameId ? { frameId: options.actionTarget.frameId } : {}),
},
{ scrollIntoView: true },
),
deadline,
options.signal,
"resolving file-drop target timed out",
);
if (isRpcError(geometry)) {
outcome = enrichFailure(geometry, "none", "resolve_target");
return outcome;
}
const point = geometry.targetActionPoint;
dragPoint = point;
outcome = await options.withOverlayHidden(async () => {
try {
try {
const belongs = await waitBounded(
pointBelongsToTarget(options, point.x, point.y, objectGroup),
deadline,
options.signal,
"verifying file-drop target timed out",
);
if (!belongs) {
return failure(
"permission_denied",
"file_drop_target_unavailable",
"file-drop target is not the topmost element at its action point",
"none",
"resolve_target",
);
}
} catch (error) {
return failure(
error instanceof BoundedWaitError ? "timeout" : "cdp_failed",
"file_drop_target_unavailable",
messageOf(error),
"none",
"resolve_target",
);
}
for (const [type, phase] of [
["dragEnter", "drag_enter"],
["dragOver", "drag_over"],
] as const) {
try {
if (type === "dragEnter") dragStarted = true;
await waitBounded(
sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", {
type,
x: point.x,
y: point.y,
data: dragData,
}),
deadline,
options.signal,
`${type} timed out`,
);
} catch (error) {
const unsupported = methodUnavailable(error);
if (type === "dragEnter" && unsupported) dragStarted = false;
return failure(
unsupported
? "unsupported"
: error instanceof BoundedWaitError
? "timeout"
: "cdp_failed",
unsupported ? "upload_mechanism_unsupported" : "file_drop_failed",
messageOf(error),
"none",
phase,
);
}
}
try {
dropSent = true;
await waitBounded(
sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", {
type: "drop",
x: point.x,
y: point.y,
data: dragData,
}),
deadline,
options.signal,
"file drop timed out",
);
return { x: point.x, y: point.y };
} catch (error) {
outcome = failure(
error instanceof BoundedWaitError ? "timeout" : "cdp_failed",
"file_drop_failed",
messageOf(error),
"unknown",
"drop",
);
return outcome;
}
} finally {
if (dragStarted && !dropSent) {
try {
await waitBounded(
sendToCdpTarget(options.cdp, target, "Input.dispatchDragEvent", {
type: "dragCancel",
x: dragPoint.x,
y: dragPoint.y,
data: dragData,
}),
Date.now() + CLEANUP_TIMEOUT_MS,
undefined,
"cancelling file drag timed out",
);
} catch {
cleanupFailed = true;
}
}
}
});
return outcome;
} catch (error) {
outcome = failure(
error instanceof BoundedWaitError ? "timeout" : "cdp_failed",
"file_drop_target_unavailable",
messageOf(error),
"none",
"resolve_target",
);
return outcome;
} finally {
try {
await waitBounded(
sendToCdpTarget(options.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }),
Date.now() + CLEANUP_TIMEOUT_MS,
undefined,
"releasing file-drop object group timed out",
);
} catch {
cleanupFailed = true;
}
const effect = isRpcError(outcome)
? (outcome.data?.effect_state as TransferEffectState | undefined)
: "committed";
if (effect === "unknown" || cleanupFailed) {
await options.cdp.detach?.(target.tabId);
}
if (cleanupFailed && isRpcError(outcome)) {
outcome.data = { ...outcome.data, cleanup_state: "failed" };
}
}
}
apps/extension/src/tools/file-input-transaction.ts
+1 -40
View File
@@ -7,6 +7,7 @@ import type { ClickResult, RpcError, TransferEffectState } from "@/transport/typ
import { transferError } from "./errors";
import type { ResolvedActionTarget } from "./interaction";
import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared";
import { BoundedWaitError, remainingMs, waitBounded } from "./transfer-transaction";
const CLEANUP_TIMEOUT_MS = 1_000;
const ACTIVATION_GRACE_MS = 1_000;
@@ -45,46 +46,6 @@ export interface FileInputTransactionResult {
multiple: boolean;
}
class BoundedWaitError extends Error {
constructor(
readonly kind: "timeout" | "aborted",
message: string,
) {
super(message);
}
}
function remainingMs(deadline: number): number {
return Math.max(0, deadline - Date.now());
}
async function waitBounded<T>(
promise: Promise<T>,
deadline: number,
signal: AbortSignal | undefined,
timeoutMessage: string,
): Promise<T> {
const remaining = remainingMs(deadline);
if (signal?.aborted) throw new BoundedWaitError("aborted", "upload transaction aborted");
if (remaining === 0) throw new BoundedWaitError("timeout", timeoutMessage);
let timer: ReturnType<typeof setTimeout> | undefined;
let onAbort: (() => void) | undefined;
const boundary = new Promise<never>((_resolve, reject) => {
timer = setTimeout(() => reject(new BoundedWaitError("timeout", timeoutMessage)), remaining);
if (signal) {
onAbort = () => reject(new BoundedWaitError("aborted", "upload transaction aborted"));
signal.addEventListener("abort", onAbort, { once: true });
}
});
try {
return await Promise.race([promise, boundary]);
} finally {
if (timer) clearTimeout(timer);
if (signal && onAbort) signal.removeEventListener("abort", onAbort);
}
}
function runtimeError(reply: RuntimeReply, fallback: string): Error | null {
if (!reply.exceptionDetails) return null;
return new Error(
apps/extension/src/tools/frame-geometry.ts
+27 -1
View File
@@ -29,7 +29,10 @@ export interface NodeAddress {
export interface ResolvedNodeGeometry {
topVisibleRegions: Region;
topBounds: ViewportRect;
/** Point in the top-level tab viewport, used by root-target input events. */
actionPoint: Point;
/** Point in the addressed CDP target's viewport, used by OOPIF-local input events. */
targetActionPoint: Point;
}
function geometryError(message: string): RpcError {
@@ -262,11 +265,30 @@ export async function resolveNodeGeometry(
if (isRpcError(localRegion)) return localRegion;
let topVisibleRegions: Region;
let targetActionPoint: Point | null = null;
if (address.frameId && graph) {
const projection = await resolveFrameProjection(cdp, graph, address.frameId);
if (!projection)
return geometryError(`could not resolve frame geometry for ${address.frameId}`);
topVisibleRegions = projectRegionToViewport(localRegion, projection);
if (address.target.sessionId) {
const localViewport = projection.edges[0]?.sourceViewport ?? projection.topViewport;
const localVisibleRegions = localRegion
.map((polygon) =>
clipPolygon(
polygon,
rectPolygon({
x: 0,
y: 0,
w: localViewport.width,
h: localViewport.height,
}),
),
)
.filter((polygon) => polygon.length >= 3);
const localActionRegion = largestRegion(localVisibleRegions);
targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null;
}
} else {
const viewport = await targetViewport(cdp, address.target);
if (!viewport) return geometryError("could not resolve top viewport geometry");
@@ -283,7 +305,11 @@ export async function resolveNodeGeometry(
if (!topBounds || !actionPoint) {
return { code: "permission_denied", message: "element not visible" };
}
return { topVisibleRegions, topBounds, actionPoint };
if (address.target.sessionId && !targetActionPoint) {
return { code: "permission_denied", message: "element not visible in its target" };
}
targetActionPoint ??= actionPoint;
return { topVisibleRegions, topBounds, actionPoint, targetActionPoint };
} catch (error) {
return geometryError(error instanceof Error ? error.message : String(error));
}
apps/extension/src/tools/observation.ts
+3 -3
View File
@@ -17,7 +17,7 @@ import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import {
type CaptureSuppressSendToTab,
withOverlaysHiddenForCapture,
withExtensionOverlayHidden,
} from "@/lib/capture-suppress-bridge";
import type { SessionContext, SessionManager } from "@/session-manager/manager";
import type {
@@ -312,7 +312,7 @@ export async function handleScreenshot(
tabId: target.tabId,
...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}),
};
const captured = await withOverlaysHiddenForCapture(
const captured = await withExtensionOverlayHidden(
target.tabId,
() =>
captureElementScreenshot(
@@ -344,7 +344,7 @@ export async function handleScreenshot(
);
}
const captured = await withOverlaysHiddenForCapture(
const captured = await withExtensionOverlayHidden(
target.tabId,
() => captureFullTabPng(deps, ctx, target, signal),
deps.sendToTab,
apps/extension/src/tools/transfer-transaction.ts
+42
View File
@@ -0,0 +1,42 @@
// Shared time boundary for browser-side file-transfer transactions. Concrete
// upload/download mechanisms own their state machines and effect semantics.
export class BoundedWaitError extends Error {
constructor(
readonly kind: "timeout" | "aborted",
message: string,
) {
super(message);
}
}
export function remainingMs(deadline: number): number {
return Math.max(0, deadline - Date.now());
}
export async function waitBounded<T>(
promise: Promise<T>,
deadline: number,
signal: AbortSignal | undefined,
timeoutMessage: string,
): Promise<T> {
const remaining = remainingMs(deadline);
if (signal?.aborted) throw new BoundedWaitError("aborted", "file transfer aborted");
if (remaining === 0) throw new BoundedWaitError("timeout", timeoutMessage);
let timer: ReturnType<typeof setTimeout> | undefined;
let onAbort: (() => void) | undefined;
const boundary = new Promise<never>((_resolve, reject) => {
timer = setTimeout(() => reject(new BoundedWaitError("timeout", timeoutMessage)), remaining);
if (signal) {
onAbort = () => reject(new BoundedWaitError("aborted", "file transfer aborted"));
signal.addEventListener("abort", onAbort, { once: true });
}
});
try {
return await Promise.race([promise, boundary]);
} finally {
if (timer) clearTimeout(timer);
if (signal && onAbort) signal.removeEventListener("abort", onAbort);
}
}
apps/extension/src/tools/upload.ts
+37 -11
View File
@@ -2,8 +2,13 @@
// target, then delegate the browser protocol transaction to the file-input
// transaction module.
import {
type CaptureSuppressSendToTab,
withExtensionOverlayHidden,
} from "@/lib/capture-suppress-bridge";
import type { SessionManager } from "@/session-manager/manager";
import type { RpcError, UploadParams, UploadResult } from "@/transport/types";
import { uploadThroughFileDrop } from "./file-drop-transaction";
import { uploadThroughActivatedFileInput } from "./file-input-transaction";
import { clickResolvedTarget, type InteractionDeps, resolveActionTarget } from "./interaction";
import {
@@ -18,6 +23,7 @@ const DEFAULT_TIMEOUT_MS = 120_000;
export interface UploadDeps extends InteractionDeps {
cdp: CdpRunner;
sendToTab?: CaptureSuppressSendToTab;
}
export async function handleUpload(
@@ -41,19 +47,39 @@ export async function handleUpload(
const address = await resolveActionTarget(deps.cdp, ctx, target, params, "upload");
if (isRpcError(address)) return address;
const timeoutMs = params.timeout_ms ?? DEFAULT_TIMEOUT_MS;
const transaction = await uploadThroughActivatedFileInput({
cdp: deps.cdp,
actionTarget: address,
files: params.files.map((file) => file.staged_path as string),
timeoutMs,
signal: deps.signal,
trigger: () => clickResolvedTarget(ctx, address, {}, deps),
});
if (isRpcError(transaction)) return transaction;
const files = params.files.map((file) => file.staged_path as string);
const mode = params.mode ?? "input";
let usedRef = address.usedRef;
let usedSelector = address.usedSelector;
if (mode === "drop") {
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
const transaction = await uploadThroughFileDrop({
cdp: deps.cdp,
actionTarget: address,
files,
timeoutMs,
signal: deps.signal,
withOverlayHidden: (operation) =>
withExtensionOverlayHidden(target.tabId, operation, deps.sendToTab),
});
if (isRpcError(transaction)) return transaction;
} else {
const transaction = await uploadThroughActivatedFileInput({
cdp: deps.cdp,
actionTarget: address,
files,
timeoutMs,
signal: deps.signal,
trigger: () => clickResolvedTarget(ctx, address, {}, deps),
});
if (isRpcError(transaction)) return transaction;
usedRef = transaction.click.used_ref;
usedSelector = transaction.click.used_selector;
}
return {
tab_id: target.tabId,
used_ref: transaction.click.used_ref,
used_selector: transaction.click.used_selector,
used_ref: usedRef,
used_selector: usedSelector,
file_names: params.files.map((file) => file.name),
};
}
apps/extension/src/transport/types.ts
+6
View File
@@ -36,6 +36,9 @@ export type RpcErrorReason =
| "file_input_probe_failed"
| "file_input_not_activated"
| "set_file_input_failed"
| "upload_mechanism_unsupported"
| "file_drop_target_unavailable"
| "file_drop_failed"
| "download_capture_failed"
| "transfer_outcome_unknown"
| "transfer_timeout"
@@ -518,12 +521,15 @@ export interface UploadFile {
staged_path?: string;
}
export type UploadMode = "input" | "drop";
export interface UploadParams {
session_id: string;
ref?: string;
selector?: string;
tab_id?: number;
files: UploadFile[];
mode?: UploadMode;
timeout_ms?: number;
}