feat(bsk): harden browser file transfer transactions

This commit is contained in:
Ljy-0827
2026-08-24 15:02:52 +08:00
parent a86cbf724f
commit bf59753aa3
24 changed files with 1397 additions and 407 deletions
+1 -1
View File
@@ -51,7 +51,7 @@ The Extension requests the following Chrome permissions. Each is used solely for
- **`alarms`** — Periodically wake the service worker to keep the local WebSocket connection alive.
- **`idle`** — Detect when the device returns from idle/locked so the Extension can promptly re-establish the local WebSocket connection after the machine wakes. No idle data is stored or transmitted.
- **`notifications`** — Show a system notification to obtain user approval before the agent borrows a user-owned tab.
- **`downloads`** — Observe and, on cancellation, stop the one browser download initiated by an active `bsk download` command. It is not used to enumerate download history.
- **`downloads`** — Correlate and route the one browser download initiated by an active `bsk download` command. If that claimed transaction fails, BrowserSkill cancels an in-progress file or removes its completed temporary browser file. It is not used to enumerate download history or alter unclaimed downloads.
- **`storage`** — Persist a random instance ID and optional label in `chrome.storage.local`.
- **Host permission `<all_urls>`** — Inject a small status overlay (showing "Agent Active") on pages controlled by the agent, and enable automation across whatever sites the user directs the agent to. The Extension does **not** read or transmit page content from sites the agent is not actively driving.
@@ -258,7 +258,20 @@ describe("ToolDispatcher", () => {
if (method === "DOM.getContentQuads") {
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] } as T;
}
if (method === "DOM.resolveNode") {
return { object: { objectId: "trigger-object" } } as T;
}
if (method === "DOM.describeNode") return { node: { backendNodeId: 456 } } as T;
if (method === "Runtime.callFunctionOn") {
const declaration = (params as { functionDeclaration?: string }).functionDeclaration ?? "";
if (declaration.includes("count: state.inputs.length")) {
return { result: { value: { count: 1, multiple: false } } } as T;
}
if (declaration.includes("inputs[0]")) {
return { result: { objectId: "input-object" } } as T;
}
return { result: { value: true } } as T;
}
if (method === "Runtime.evaluate") {
const expression = (params as { expression?: string }).expression ?? "";
if (expression.includes("overlayDetails")) {
@@ -3,6 +3,7 @@ import { SessionManager } from "@/session-manager/manager";
import { type DownloadsApi, handleDownload } from "../download";
import { captureBrowserDownload } from "../download-capture";
import { uploadThroughActivatedFileInput } from "../file-input-transaction";
import type { ResolvedActionTarget } from "../interaction";
import type { CdpRunner } from "../shared";
import { handleUpload } from "../upload";
@@ -36,37 +37,76 @@ function fakeEvent<T extends (...args: never[]) => unknown>() {
};
}
function uploadCdp(options: { inputCount?: number; multiple?: boolean; pickerCall?: string } = {}) {
function actionTarget(frameId?: string): ResolvedActionTarget {
return {
tab: { tabId: 4, windowId: 100, active: true },
backendNodeId: 123,
cdpTarget: { tabId: 4 },
...(frameId ? { frameId } : {}),
usedRef: "e3",
};
}
function uploadCdp(
options: {
inputCount?: number;
multiple?: boolean;
chooser?: { frameId?: string; backendNodeId?: number; mode?: string };
pendingResolve?: boolean;
} = {},
) {
const calls: Array<{ method: string; params?: object }> = [];
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
calls.push({ method, params });
if (method === "Page.setInterceptFileChooserDialog") return {};
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
if (method === "Runtime.evaluate") {
const expression = (params as { expression?: string }).expression ?? "";
if (expression.includes("count:")) {
if (method === "DOM.resolveNode") {
if (options.pendingResolve) return new Promise<never>(() => {});
return { object: { objectId: "trigger-object" } };
}
if (method === "Runtime.callFunctionOn") {
const declaration = (params as { functionDeclaration?: string }).functionDeclaration ?? "";
if (declaration.includes("Object.defineProperty")) return { result: { value: true } };
if (declaration.includes("count: state.inputs.length")) {
return {
result: {
value: {
count: options.inputCount ?? 1,
multiple: options.multiple ?? true,
pickerCall: options.pickerCall,
},
},
};
}
if (expression.includes("?.inputs[0]")) {
if (declaration.includes("inputs[0]")) {
return { result: { objectId: "input-object" } };
}
return { result: { value: true } };
}
if (method === "DOM.describeNode") return { node: { backendNodeId: 456 } };
if (
method === "Input.dispatchMouseEvent" &&
(params as { type?: string }).type === "mousePressed" &&
options.chooser
) {
cdpEvent?.({ tabId: 4 }, "Page.fileChooserOpened", options.chooser);
}
return {};
});
const cdp: CdpRunner = {
send: send as unknown as CdpRunner["send"],
onEvent: (handler) => {
cdpEvent = handler;
return { dispose: vi.fn() };
},
};
return {
calls,
cdp: { send: send as unknown as CdpRunner["send"] } satisfies CdpRunner,
emitChooser: (event: { frameId?: string; backendNodeId?: number; mode?: string }) =>
cdpEvent?.({ tabId: 4 }, "Page.fileChooserOpened", event),
cdp,
};
}
@@ -91,15 +131,14 @@ describe("file transfer tools", () => {
);
expect(result).toMatchObject({ tab_id: 4, file_names: ["one.png", "two.png"] });
const armCall = calls.find(
(call) =>
call.method === "Runtime.evaluate" &&
(call.params as { expression?: string }).expression?.includes("showOpenFilePicker"),
);
expect(armCall).toBeDefined();
expect((armCall?.params as { expression?: string }).expression).toContain(
"event.preventDefault()",
);
expect(calls[0]).toEqual({
method: "Page.setInterceptFileChooserDialog",
params: { enabled: true },
});
expect(calls).toContainEqual({
method: "Page.setInterceptFileChooserDialog",
params: { enabled: false, cancel: true },
});
expect(calls).toContainEqual({
method: "DOM.setFileInputFiles",
params: { files: ["/private/stage/one", "/private/stage/two"], backendNodeId: 456 },
@@ -110,7 +149,7 @@ describe("file transfer tools", () => {
const { cdp } = uploadCdp({ inputCount: 0 });
const result = await uploadThroughActivatedFileInput({
cdp,
target: { tabId: 4 },
actionTarget: actionTarget(),
files: ["/private/stage/one"],
timeoutMs: 100,
trigger: async () => ({ tab_id: 4, x: 10, y: 10 }),
@@ -122,31 +161,80 @@ describe("file transfer tools", () => {
});
});
it("reports a File System Access picker without waiting for a timeout", async () => {
const { cdp } = uploadCdp({ inputCount: 0, pickerCall: "showOpenFilePicker" });
it("fails before clicking when chooser interception is unavailable", async () => {
const trigger = vi.fn(async () => ({ tab_id: 4, x: 10, y: 10 }));
const cdp: CdpRunner = {
send: vi.fn(async (_tabId: number, method: string) => {
if (method === "Page.setInterceptFileChooserDialog") {
throw new Error("method unavailable");
}
return {};
}) as CdpRunner["send"],
};
const result = await uploadThroughActivatedFileInput({
cdp,
target: { tabId: 4 },
actionTarget: actionTarget(),
files: ["/private/stage/one"],
timeoutMs: 100,
trigger: async () => ({ tab_id: 4, x: 10, y: 10 }),
trigger,
});
expect(result).toMatchObject({
code: "cdp_failed",
data: { effect_state: "none", phase: "arm_interception" },
});
expect(trigger).not.toHaveBeenCalled();
});
it("uses an exact chooser event as an independent input-location signal", async () => {
const { cdp, calls, emitChooser } = uploadCdp({ inputCount: 0 });
const result = await uploadThroughActivatedFileInput({
cdp,
actionTarget: actionTarget("f1"),
files: ["/private/stage/one"],
timeoutMs: 100,
trigger: async () => {
emitChooser({ frameId: "f1", backendNodeId: 789, mode: "selectSingle" });
return { tab_id: 4, x: 10, y: 10 };
},
});
expect(result).toMatchObject({ multiple: false });
expect(calls).toContainEqual({
method: "DOM.setFileInputFiles",
params: { files: ["/private/stage/one"], backendNodeId: 789 },
});
});
it("reports a File System Access picker without waiting for a timeout", async () => {
const { cdp, emitChooser } = uploadCdp({
inputCount: 0,
});
const result = await uploadThroughActivatedFileInput({
cdp,
actionTarget: actionTarget("f1"),
files: ["/private/stage/one"],
timeoutMs: 100,
trigger: async () => {
emitChooser({ frameId: "f1", mode: "selectSingle" });
return { tab_id: 4, x: 10, y: 10 };
},
});
expect(result).toMatchObject({
code: "unsupported",
message: "upload trigger invoked showOpenFilePicker instead of an input[type=file]",
message: "upload trigger invoked a non-input file picker",
data: { reason: "file_input_not_activated", phase: "resolve_input" },
});
});
it("bounds a stuck file-input probe", async () => {
const cdp: CdpRunner = {
send: vi.fn(() => new Promise<never>(() => {})) as unknown as CdpRunner["send"],
};
const { cdp } = uploadCdp({ pendingResolve: true });
const result = await uploadThroughActivatedFileInput({
cdp,
target: { tabId: 4 },
actionTarget: actionTarget(),
files: ["/private/stage/one"],
timeoutMs: 5,
trigger: vi.fn(),
@@ -158,6 +246,31 @@ describe("file transfer tools", () => {
});
});
it("marks a timed-out file assignment unknown and detaches browser state", async () => {
const fixture = uploadCdp();
const originalSend = fixture.cdp.send;
const detach = vi.fn(async () => {});
fixture.cdp.detach = detach;
fixture.cdp.send = vi.fn((tabId: number, method: string, params?: object) => {
if (method === "DOM.setFileInputFiles") return new Promise<never>(() => {});
return originalSend(tabId, method, params);
}) as CdpRunner["send"];
const result = await uploadThroughActivatedFileInput({
cdp: fixture.cdp,
actionTarget: actionTarget(),
files: ["/private/stage/one"],
timeoutMs: 10,
trigger: async () => ({ tab_id: 4, x: 10, y: 10 }),
});
expect(result).toMatchObject({
code: "timeout",
data: { effect_state: "unknown", phase: "set_files" },
});
expect(detach).toHaveBeenCalledWith(4);
});
it("routes one exact-target download through a browser-relative capability", async () => {
const manager = sessions();
const ctx = await manager.start("s1");
@@ -194,6 +307,7 @@ describe("file transfer tools", () => {
onDeterminingFilename,
search: vi.fn(async () => [completed]),
cancel: vi.fn(async () => {}),
removeFile: vi.fn(async () => {}),
};
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
let suggested: chrome.downloads.DownloadFilenameSuggestion | undefined;
@@ -261,6 +375,7 @@ describe("file transfer tools", () => {
onDeterminingFilename,
search: vi.fn(async () => []),
cancel: vi.fn(async () => {}),
removeFile: vi.fn(async () => {}),
};
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
const cdp: CdpRunner = {
@@ -304,4 +419,137 @@ describe("file transfer tools", () => {
data: { reason: "download_capture_failed" },
});
});
it("correlates a filename candidate that arrives before the CDP intent", async () => {
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
const onDeterminingFilename =
fakeEvent<
(
item: chrome.downloads.DownloadItem,
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
) => void | true
>();
const initial = {
id: 21,
url: "https://example.test/candidate-first.bin",
finalUrl: "https://example.test/candidate-first.bin",
filename: "candidate-first.bin",
state: "in_progress",
fileSize: -1,
totalBytes: 4,
bytesReceived: 0,
} as chrome.downloads.DownloadItem;
const complete = { ...initial, state: "complete", fileSize: 4 } as chrome.downloads.DownloadItem;
const downloads: DownloadsApi = {
onCreated,
onChanged,
onDeterminingFilename,
search: vi.fn(async () => [complete]),
cancel: vi.fn(async () => {}),
removeFile: vi.fn(async () => {}),
};
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
const cdp: CdpRunner = {
send: vi.fn(async () => ({})) as CdpRunner["send"],
onEvent: (handler) => {
cdpEvent = handler;
return { dispose: vi.fn() };
},
};
let suggestion: chrome.downloads.DownloadFilenameSuggestion | undefined;
const result = await captureBrowserDownload({
cdp,
target: { tabId: 4 },
downloads,
browserRelativeDir: "BrowserSkill/tr_21",
timeoutMs: 1_000,
trigger: async () => {
const suggested = new Promise<void>((resolve) => {
onDeterminingFilename.emit(initial, (value) => {
suggestion = value;
resolve();
});
});
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
url: initial.url,
suggestedFilename: initial.filename,
});
await suggested;
onCreated.emit(complete);
return { tab_id: 4, x: 10, y: 10 };
},
});
expect(suggestion).toEqual({
filename: "BrowserSkill/tr_21/candidate-first.bin",
conflictAction: "overwrite",
});
expect(result).toMatchObject({ item: { id: 21, state: "complete" } });
});
it("rejects ambiguous attribution without cancelling either unclaimed download", async () => {
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
const onDeterminingFilename =
fakeEvent<
(
item: chrome.downloads.DownloadItem,
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
) => void | true
>();
const downloads: DownloadsApi = {
onCreated,
onChanged,
onDeterminingFilename,
search: vi.fn(async () => []),
cancel: vi.fn(async () => {}),
removeFile: vi.fn(async () => {}),
};
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
const cdp: CdpRunner = {
send: vi.fn(async () => ({})) as CdpRunner["send"],
onEvent: (handler) => {
cdpEvent = handler;
return { dispose: vi.fn() };
},
};
const defaults: number[] = [];
const candidate = (id: number) =>
({
id,
url: "https://example.test/same.bin",
finalUrl: "https://example.test/same.bin",
filename: "same.bin",
state: "in_progress",
}) as chrome.downloads.DownloadItem;
const result = await captureBrowserDownload({
cdp,
target: { tabId: 4 },
downloads,
browserRelativeDir: "BrowserSkill/tr_ambiguous",
timeoutMs: 100,
trigger: async () => {
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
url: "https://example.test/same.bin",
suggestedFilename: "same.bin",
});
for (const id of [31, 32]) {
onDeterminingFilename.emit(candidate(id), (value) => {
if (value === undefined) defaults.push(id);
});
}
return { tab_id: 4, x: 10, y: 10 };
},
});
expect(defaults.sort()).toEqual([31, 32]);
expect(downloads.cancel).not.toHaveBeenCalled();
expect(result).toMatchObject({
code: "cdp_failed",
data: { effect_state: "unknown", phase: "attribution" },
});
});
});
+196 -60
View File
@@ -1,12 +1,16 @@
// Transaction-scoped capture for one web download. Chrome owns the actual
// download and writes only beneath its Downloads root; BrowserSkill supplies a
// daemon-minted relative directory and reports the completed absolute path
// back to the daemon for validated import.
// Order-independent coordinator for one browser download. CDP supplies the
// exact target/frame intent while chrome.downloads supplies the download id
// and filename routing hook; neither event is assumed to arrive first.
import type { CdpTarget } from "@/browser-driver/frame-graph";
import type { ClickResult, RpcError } from "@/transport/types";
import type { ClickResult, RpcError, TransferEffectState } from "@/transport/types";
import { transferError } from "./errors";
import { type CdpRunner, isRpcError } from "./shared";
const CORRELATION_GRACE_MS = 750;
const UNIQUE_SETTLE_MS = 50;
const SIZE_POLL_MS = 250;
type DeterminingFilenameListener = (
item: chrome.downloads.DownloadItem,
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
@@ -23,6 +27,7 @@ export interface DownloadsApi {
onDeterminingFilename: ListenerEvent<DeterminingFilenameListener>;
search(query: chrome.downloads.DownloadQuery): Promise<chrome.downloads.DownloadItem[]>;
cancel(downloadId: number): Promise<void>;
removeFile(downloadId: number): Promise<void>;
}
export const chromeDownloadsApi: DownloadsApi = {
@@ -37,13 +42,16 @@ export const chromeDownloadsApi: DownloadsApi = {
},
search: (query) => chrome.downloads.search(query),
cancel: (id) => chrome.downloads.cancel(id),
removeFile: (id) => chrome.downloads.removeFile(id),
};
export interface DownloadCaptureOptions {
cdp: CdpRunner;
target: CdpTarget;
expectedFrameId?: string;
downloads: DownloadsApi;
browserRelativeDir: string;
maxByteSize?: number;
timeoutMs: number;
signal?: AbortSignal;
trigger(): Promise<ClickResult | RpcError>;
@@ -54,43 +62,70 @@ export interface DownloadCaptureResult {
item: chrome.downloads.DownloadItem;
}
interface DownloadIntent {
url: string;
suggestedFilename: string;
frameId?: string;
}
interface DownloadCandidate {
item: chrome.downloads.DownloadItem;
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void;
suggested: boolean;
graceTimer: ReturnType<typeof setTimeout>;
}
function safeBasename(filename: string): string {
const basename = filename.split(/[\\/]/).pop()?.trim();
return basename && basename !== "." && basename !== ".." ? basename : "download";
}
function captureError(message: string): RpcError {
return {
code: "cdp_failed",
message,
data: { reason: "download_capture_failed" },
};
}
function sameTarget(source: { tabId?: number; sessionId?: string }, target: CdpTarget): boolean {
return source.tabId === target.tabId && source.sessionId === target.sessionId;
}
function matchesIntent(
item: chrome.downloads.DownloadItem,
intent: { url: string; suggestedFilename: string },
): boolean {
function matchesIntent(item: chrome.downloads.DownloadItem, intent: DownloadIntent): boolean {
const urlMatches = item.url === intent.url || item.finalUrl === intent.url;
return urlMatches && safeBasename(item.filename) === safeBasename(intent.suggestedFilename);
}
function knownSize(item: chrome.downloads.DownloadItem): number | undefined {
if (item.fileSize >= 0) return item.fileSize;
if (item.totalBytes >= 0) return item.totalBytes;
return undefined;
}
function captureError(
message: string,
effectState: TransferEffectState,
phase: string,
cleanupFailed = false,
): RpcError {
return transferError("cdp_failed", "download_capture_failed", message, {
effectState,
phase,
...(cleanupFailed ? { cleanupState: "failed" } : {}),
});
}
export async function captureBrowserDownload(
options: DownloadCaptureOptions,
): Promise<DownloadCaptureResult | RpcError> {
let click: ClickResult | undefined;
let intent: DownloadIntent | undefined;
let capturedId: number | undefined;
let intent: { url: string; suggestedFilename: string } | undefined;
const createdItems = new Map<number, chrome.downloads.DownloadItem>();
let capturedItem: chrome.downloads.DownloadItem | undefined;
let settled = false;
let succeeded = false;
let timer: ReturnType<typeof setTimeout> | undefined;
let rejectCompletion!: (error: Error) => void;
let resolveCompletion!: (item: chrome.downloads.DownloadItem) => void;
let failureResult: RpcError | undefined;
let uniquenessTimer: ReturnType<typeof setTimeout> | undefined;
let operationTimer: ReturnType<typeof setTimeout> | undefined;
let sizePoll: ReturnType<typeof setInterval> | undefined;
const candidates = new Map<number, DownloadCandidate>();
const createdItems = new Map<number, chrome.downloads.DownloadItem>();
let resolveCompletion!: (item: chrome.downloads.DownloadItem) => void;
let rejectCompletion!: (error: Error) => void;
const completion = new Promise<chrome.downloads.DownloadItem>((resolve, reject) => {
resolveCompletion = resolve;
rejectCompletion = reject;
@@ -102,40 +137,95 @@ export async function captureBrowserDownload(
};
const complete = (item: chrome.downloads.DownloadItem) => {
if (settled) return;
const size = knownSize(item);
if (size !== undefined && options.maxByteSize !== undefined && size > options.maxByteSize) {
fail(new Error(`download exceeds transfer limit ${options.maxByteSize}`));
return;
}
settled = true;
capturedItem = item;
resolveCompletion(item);
};
const claim = (item: chrome.downloads.DownloadItem): boolean => {
if (capturedId === undefined) {
capturedId = item.id;
return true;
const suggestDefault = (candidate: DownloadCandidate) => {
if (candidate.suggested) return;
candidate.suggested = true;
candidate.suggest();
};
const matchingCandidates = (): DownloadCandidate[] => {
const currentIntent = intent;
return currentIntent
? [...candidates.values()].filter(
(candidate) => !candidate.suggested && matchesIntent(candidate.item, currentIntent),
)
: [];
};
const claimUnique = () => {
uniquenessTimer = undefined;
if (settled || capturedId !== undefined || !intent) return;
const matches = matchingCandidates();
if (matches.length !== 1) {
if (matches.length > 1) {
for (const candidate of matches) suggestDefault(candidate);
fail(new Error("download attribution is ambiguous"));
}
return;
}
const candidate = matches[0];
candidate.suggested = true;
clearTimeout(candidate.graceTimer);
capturedId = candidate.item.id;
capturedItem = candidate.item;
candidate.suggest({
filename: `${options.browserRelativeDir}/${safeBasename(intent.suggestedFilename)}`,
conflictAction: "overwrite",
});
const size = knownSize(candidate.item);
if (size !== undefined && options.maxByteSize !== undefined && size > options.maxByteSize) {
fail(new Error(`download exceeds transfer limit ${options.maxByteSize}`));
return;
}
const created = createdItems.get(candidate.item.id);
if (created?.state === "interrupted") {
fail(new Error(created.error ?? "download interrupted"));
} else if (created?.state === "complete") {
complete(created);
}
};
const reconcile = () => {
if (settled || capturedId !== undefined || !intent) return;
const matches = matchingCandidates();
if (matches.length > 1) {
for (const candidate of matches) suggestDefault(candidate);
fail(new Error("download attribution is ambiguous"));
return;
}
if (matches.length === 1 && !uniquenessTimer) {
uniquenessTimer = setTimeout(claimUnique, UNIQUE_SETTLE_MS);
}
if (capturedId === item.id) return true;
void options.downloads.cancel(item.id).catch(() => undefined);
fail(new Error("download trigger produced more than one file"));
return false;
};
const determiningListener: DeterminingFilenameListener = (item, suggest) => {
try {
if (!intent || !matchesIntent(item, intent) || !claim(item)) {
suggest();
return;
}
suggest({
filename: `${options.browserRelativeDir}/${safeBasename(intent.suggestedFilename)}`,
conflictAction: "overwrite",
});
const created = createdItems.get(item.id);
if (created?.state === "complete") complete(created);
} catch (err) {
suggest();
fail(err instanceof Error ? err : new Error(String(err)));
}
const candidate: DownloadCandidate = {
item,
suggest,
suggested: false,
graceTimer: setTimeout(() => {
suggestDefault(candidate);
candidates.delete(item.id);
if (intent && matchesIntent(item, intent) && capturedId === undefined) {
fail(new Error("download correlation grace elapsed before unique attribution"));
}
}, CORRELATION_GRACE_MS),
};
candidates.set(item.id, candidate);
reconcile();
return true;
};
const createdListener = (item: chrome.downloads.DownloadItem) => {
createdItems.set(item.id, item);
if (capturedId !== item.id) return;
capturedItem = item;
if (item.state === "interrupted") {
fail(new Error(item.error ?? "download interrupted"));
} else if (item.state === "complete") {
@@ -158,54 +248,100 @@ export async function captureBrowserDownload(
}
}
};
const onAbort = () => {
fail(new DOMException("aborted", "AbortError"));
};
const onAbort = () => fail(new DOMException("aborted", "AbortError"));
const cdpSubscription = options.cdp.onEvent?.((source, method, raw) => {
if (method !== "Page.downloadWillBegin" || !sameTarget(source, options.target)) return;
const event = raw as { url?: unknown; suggestedFilename?: unknown };
const event = raw as { url?: unknown; suggestedFilename?: unknown; frameId?: unknown };
if (typeof event.url !== "string" || typeof event.suggestedFilename !== "string") return;
if (options.expectedFrameId && event.frameId !== options.expectedFrameId) {
fail(new Error("download originated from a different frame"));
return;
}
if (intent) {
fail(new Error("download trigger produced more than one browser download intent"));
return;
}
intent = { url: event.url, suggestedFilename: event.suggestedFilename };
intent = {
url: event.url,
suggestedFilename: event.suggestedFilename,
...(typeof event.frameId === "string" ? { frameId: event.frameId } : {}),
};
reconcile();
});
if (!cdpSubscription) {
return captureError("CDP download intent subscription unavailable");
return captureError("CDP download intent subscription unavailable", "none", "arm");
}
options.downloads.onDeterminingFilename.addListener(determiningListener);
options.downloads.onCreated.addListener(createdListener);
options.downloads.onChanged.addListener(changedListener);
options.signal?.addEventListener("abort", onAbort, { once: true });
timer = setTimeout(
operationTimer = setTimeout(
() => fail(new Error("download did not complete before timeout")),
options.timeoutMs,
);
sizePoll = setInterval(() => {
if (capturedId === undefined || settled || options.maxByteSize === undefined) return;
void options.downloads
.search({ id: capturedId })
.then(([item]) => {
if (!item || settled) return;
capturedItem = item;
if (item.bytesReceived > (options.maxByteSize as number)) {
fail(new Error(`download exceeds transfer limit ${options.maxByteSize}`));
}
})
.catch((err) => fail(err instanceof Error ? err : new Error(String(err))));
}, SIZE_POLL_MS);
try {
const triggered = await options.trigger();
if (isRpcError(triggered)) {
void completion.catch(() => undefined);
return triggered;
const effect: TransferEffectState = capturedId !== undefined ? "committed" : intent ? "unknown" : "none";
failureResult = {
...triggered,
data: { ...triggered.data, effect_state: effect, phase: "trigger" },
};
return failureResult;
}
click = triggered;
const item = await completion;
succeeded = true;
return { click: triggered, item };
return { click, item };
} catch (err) {
if (err instanceof DOMException && err.name === "AbortError") throw err;
return captureError(err instanceof Error ? err.message : String(err));
const effect: TransferEffectState = capturedId !== undefined ? "committed" : click ? "unknown" : "none";
failureResult = captureError(
err instanceof Error ? err.message : String(err),
effect,
capturedId !== undefined ? "download" : "attribution",
);
return failureResult;
} finally {
settled = true;
if (timer) clearTimeout(timer);
if (!succeeded && capturedId !== undefined) {
await options.downloads.cancel(capturedId).catch(() => undefined);
}
if (operationTimer) clearTimeout(operationTimer);
if (uniquenessTimer) clearTimeout(uniquenessTimer);
if (sizePoll) clearInterval(sizePoll);
options.signal?.removeEventListener("abort", onAbort);
options.downloads.onDeterminingFilename.removeListener(determiningListener);
options.downloads.onCreated.removeListener(createdListener);
options.downloads.onChanged.removeListener(changedListener);
cdpSubscription.dispose();
for (const candidate of candidates.values()) {
clearTimeout(candidate.graceTimer);
if (candidate.item.id !== capturedId) suggestDefault(candidate);
}
if (!succeeded && capturedId !== undefined) {
const item = capturedItem ?? createdItems.get(capturedId);
try {
if (item?.state === "complete") {
await options.downloads.removeFile(capturedId);
} else {
await options.downloads.cancel(capturedId);
}
} catch {
if (failureResult?.data) failureResult.data.cleanup_state = "failed";
}
}
}
}
+10 -13
View File
@@ -2,9 +2,13 @@
// browser-global chrome.downloads transaction to download-capture.ts.
import type { SessionManager } from "@/session-manager/manager";
import type { ClickParams, DownloadParams, DownloadResult, RpcError } from "@/transport/types";
import type { DownloadParams, DownloadResult, RpcError } from "@/transport/types";
import { captureBrowserDownload, chromeDownloadsApi, type DownloadsApi } from "./download-capture";
import { handleClick, type InteractionDeps, resolveBackendNode } from "./interaction";
import {
clickResolvedTarget,
type InteractionDeps,
resolveActionTarget,
} from "./interaction";
import { enforceAgentWindow, isRpcError, lookupSession, resolveTargetTab } from "./shared";
let downloadActive = false;
@@ -32,7 +36,7 @@ export async function handleDownload(
if (!params.browser_relative_dir) {
return { code: "invalid_params", message: "download requires a daemon capability directory" };
}
const address = await resolveBackendNode(deps.cdp, ctx, target, params, "download");
const address = await resolveActionTarget(deps.cdp, ctx, target, params, "download");
if (isRpcError(address)) return address;
const capture = await captureBrowserDownload({
@@ -40,18 +44,11 @@ export async function handleDownload(
target: address.cdpTarget,
downloads: deps.downloads ?? chromeDownloadsApi,
browserRelativeDir: params.browser_relative_dir,
maxByteSize: params.max_byte_size,
timeoutMs: params.timeout_ms ?? 120_000,
signal: deps.signal,
trigger: () => {
const clickParams: ClickParams = {
session_id: params.session_id,
ref: params.ref,
selector: params.selector,
tab_id: params.tab_id,
timeout_ms: params.timeout_ms,
};
return handleClick(manager, clickParams, deps);
},
expectedFrameId: address.frameId,
trigger: () => clickResolvedTarget(ctx, address, {}, deps),
});
if (isRpcError(capture)) return capture;
const { click, item } = capture;
+27 -1
View File
@@ -2,10 +2,23 @@
// rendering. Extension handlers attach reasons here; human-facing copy
// lives in bsk-cli `render_error.rs`.
import type { ErrorCode, RpcError, RpcErrorData, RpcErrorReason } from "@/transport/types";
import type {
ErrorCode,
RpcError,
RpcErrorData,
RpcErrorReason,
TransferCleanupState,
TransferEffectState,
} from "@/transport/types";
export type { RpcErrorData, RpcErrorReason };
export interface TransferErrorOptions {
effectState: TransferEffectState;
phase: string;
cleanupState?: TransferCleanupState;
}
export function rpcError(
code: ErrorCode,
reason: RpcErrorReason,
@@ -15,3 +28,16 @@ export function rpcError(
const data: RpcErrorData = { reason, ...extra };
return { code, message, data };
}
export function transferError(
code: ErrorCode,
reason: RpcErrorReason,
message: string,
options: TransferErrorOptions,
): RpcError {
return rpcError(code, reason, message, {
effect_state: options.effectState,
phase: options.phase,
...(options.cleanupState ? { cleanup_state: options.cleanupState } : {}),
});
}
+315 -179
View File
@@ -1,35 +1,43 @@
// One upload transaction, independent of Page.fileChooserOpened delivery.
// A transaction-scoped DOM listener records the file input actually activated
// by the requested click and cancels its native default action. File System
// Access picker entry points are replaced only for the same transaction, so a
// non-input picker fails promptly without opening an OS dialog.
// One upload transaction with an explicit browser-side commit boundary.
// Chrome interception prevents a native chooser from escaping automation;
// chooser events and a frame-scoped DOM probe are independent input-location
// signals, so event delivery is not required for standard file inputs.
import type { CdpTarget } from "@/browser-driver/frame-graph";
import type { ClickResult, RpcError } from "@/transport/types";
import type { ClickResult, RpcError, TransferEffectState } from "@/transport/types";
import { transferError } from "./errors";
import type { ResolvedActionTarget } from "./interaction";
import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared";
const CLEANUP_TIMEOUT_MS = 1_000;
const ACTIVATION_GRACE_MS = 1_000;
const PROBE_INTERVAL_MS = 20;
type UploadPhase = "arm_input_probe" | "trigger" | "resolve_input" | "set_files";
type UploadPhase =
| "arm_interception"
| "arm_input_probe"
| "trigger"
| "resolve_input"
| "set_files"
| "cleanup";
interface RuntimeReply {
result?: {
value?: unknown;
objectId?: string;
};
exceptionDetails?: {
text?: string;
exception?: { description?: string };
};
result?: { value?: unknown; objectId?: string };
exceptionDetails?: { text?: string; exception?: { description?: string } };
}
interface ChooserEvent {
frameId?: string;
backendNodeId?: number;
mode?: "selectSingle" | "selectMultiple";
}
export interface FileInputTransactionOptions {
cdp: CdpRunner;
target: CdpTarget;
actionTarget: ResolvedActionTarget;
files: string[];
timeoutMs: number;
signal?: AbortSignal;
trigger(timeoutMs: number): Promise<ClickResult | RpcError>;
trigger(): Promise<ClickResult | RpcError>;
}
export interface FileInputTransactionResult {
@@ -38,10 +46,7 @@ export interface FileInputTransactionResult {
}
class BoundedWaitError extends Error {
constructor(
readonly kind: "timeout" | "aborted",
message: string,
) {
constructor(readonly kind: "timeout" | "aborted", message: string) {
super(message);
}
}
@@ -77,15 +82,6 @@ async function waitBounded<T>(
}
}
function uploadError(
code: RpcError["code"],
message: string,
reason: "file_input_probe_failed" | "file_input_not_activated" | "set_file_input_failed",
phase: UploadPhase,
): RpcError {
return { code, message, data: { reason, phase } };
}
function runtimeError(reply: RuntimeReply, fallback: string): Error | null {
if (!reply.exceptionDetails) return null;
return new Error(
@@ -93,25 +89,124 @@ function runtimeError(reply: RuntimeReply, fallback: string): Error | null {
);
}
function transferFailure(
code: RpcError["code"],
reason: "file_input_probe_failed" | "file_input_not_activated" | "set_file_input_failed",
message: string,
effectState: TransferEffectState,
phase: UploadPhase,
): RpcError {
return transferError(code, reason, message, { effectState, phase });
}
function enrichFailure(error: RpcError, effectState: TransferEffectState, phase: UploadPhase): RpcError {
return {
...error,
data: { ...error.data, effect_state: effectState, phase },
};
}
function sameTarget(
source: { tabId?: number; sessionId?: string },
target: { tabId: number; sessionId?: string },
): boolean {
return source.tabId === target.tabId && source.sessionId === target.sessionId;
}
async function delay(ms: number, signal?: AbortSignal): Promise<void> {
await waitBounded(
new Promise((resolve) => setTimeout(resolve, ms)),
Date.now() + ms + 1,
signal,
"upload activation probe timed out",
);
}
async function callOnTrigger<T = RuntimeReply>(
options: FileInputTransactionOptions,
objectId: string,
functionDeclaration: string,
args: unknown[],
returnByValue: boolean,
): Promise<T> {
return sendToCdpTarget<T>(options.cdp, options.actionTarget.cdpTarget, "Runtime.callFunctionOn", {
objectId,
functionDeclaration,
arguments: args.map((value) => ({ value })),
returnByValue,
awaitPromise: false,
});
}
export async function uploadThroughActivatedFileInput(
options: FileInputTransactionOptions,
): Promise<FileInputTransactionResult | RpcError> {
const deadline = Date.now() + options.timeoutMs;
const target = options.actionTarget.cdpTarget;
const objectGroup = `bsk-upload-${crypto.randomUUID()}`;
const stateKey = `__bskUpload_${crypto.randomUUID().replaceAll("-", "")}`;
const stateKeyLiteral = JSON.stringify(stateKey);
let probeAttempted = false;
const chooserEvents: ChooserEvent[] = [];
let interceptionArmed = false;
let probeArmed = false;
let triggerObjectId: string | undefined;
let outcome: FileInputTransactionResult | RpcError = transferFailure(
"protocol_error",
"file_input_probe_failed",
"upload transaction ended without an outcome",
"none",
"cleanup",
);
const chooserSubscription = options.cdp.onEvent?.((source, method, raw) => {
if (method !== "Page.fileChooserOpened" || !sameTarget(source, target)) return;
const event = raw as ChooserEvent;
chooserEvents.push(event);
});
try {
try {
probeAttempted = true;
await waitBounded(
sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", {
enabled: true,
}),
deadline,
options.signal,
"arming native file chooser interception timed out",
);
interceptionArmed = true;
} catch (err) {
outcome = transferFailure(
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
"file_input_probe_failed",
err instanceof Error ? err.message : String(err),
"none",
"arm_interception",
);
return outcome;
}
try {
const resolved = await waitBounded(
sendToCdpTarget<{ object?: { objectId?: string } }>(options.cdp, target, "DOM.resolveNode", {
backendNodeId: options.actionTarget.backendNodeId,
objectGroup,
}),
deadline,
options.signal,
"resolving upload trigger timed out",
);
triggerObjectId = resolved.object?.objectId;
if (!triggerObjectId) throw new Error("DOM.resolveNode returned no trigger objectId");
const armed = await waitBounded(
sendToCdpTarget<RuntimeReply>(options.cdp, options.target, "Runtime.evaluate", {
expression: `(() => {
const key = ${stateKeyLiteral};
const owner = globalThis;
const doc = document;
const state = { inputs: [], listener: null, pickerCalls: [], pickers: [] };
callOnTrigger<RuntimeReply>(
options,
triggerObjectId,
`function(key) {
const doc = this.ownerDocument;
const owner = doc.defaultView;
if (!owner) return false;
const state = { inputs: [], listener: null };
Object.defineProperty(owner, key, { value: state, configurable: true });
state.listener = event => {
const path = typeof event.composedPath === "function" ? event.composedPath() : [];
@@ -123,143 +218,169 @@ export async function uploadThroughActivatedFileInput(
}
};
doc.addEventListener("click", state.listener, true);
const win = doc.defaultView;
for (const name of ["showOpenFilePicker", "showSaveFilePicker", "showDirectoryPicker"]) {
if (!win || typeof win[name] !== "function") continue;
const hadOwn = Object.prototype.hasOwnProperty.call(win, name);
const descriptor = Object.getOwnPropertyDescriptor(win, name);
state.pickers.push({ name, hadOwn, descriptor });
Object.defineProperty(win, name, {
configurable: true,
enumerable: descriptor?.enumerable ?? true,
writable: true,
value: () => {
state.pickerCalls.push(name);
return Promise.reject(
new DOMException("Picker intercepted by BrowserSkill", "AbortError")
);
},
});
}
return true;
})()`,
objectGroup,
returnByValue: true,
}),
}`,
[stateKey],
true,
),
deadline,
options.signal,
"arming file input probe timed out",
"arming frame-scoped file input probe timed out",
);
const armError = runtimeError(armed, "failed to arm file input probe");
if (armError) throw armError;
if (armError || armed.result?.value !== true) {
throw armError ?? new Error("file input probe did not arm");
}
probeArmed = true;
} catch (err) {
if (err instanceof BoundedWaitError && err.kind === "aborted") throw err;
return uploadError(
outcome = transferFailure(
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
err instanceof Error ? err.message : String(err),
"file_input_probe_failed",
err instanceof Error ? err.message : String(err),
"none",
"arm_input_probe",
);
return outcome;
}
let click: ClickResult | RpcError;
try {
click = await waitBounded(
options.trigger(remainingMs(deadline)),
options.trigger(),
deadline,
options.signal,
"upload trigger timed out",
);
} catch (err) {
if (err instanceof BoundedWaitError && err.kind === "aborted") throw err;
return uploadError(
outcome = transferFailure(
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
err instanceof Error ? err.message : String(err),
"file_input_probe_failed",
err instanceof Error ? err.message : String(err),
"none",
"trigger",
);
return outcome;
}
if (isRpcError(click)) {
outcome = enrichFailure(click, "none", "trigger");
return outcome;
}
if (isRpcError(click)) return click;
try {
const summary = await waitBounded(
sendToCdpTarget<RuntimeReply>(options.cdp, options.target, "Runtime.evaluate", {
expression: `(() => {
const s = globalThis[${stateKeyLiteral}];
return s
? { count: s.inputs.length, multiple: s.inputs[0]?.multiple === true,
pickerCall: s.pickerCalls[0] }
const activationDeadline = Math.min(deadline, Date.now() + ACTIVATION_GRACE_MS);
let summary: { count: number; multiple: boolean } = { count: 0, multiple: false };
while (Date.now() < activationDeadline) {
if (chooserEvents.length > 0) break;
const reply = await callOnTrigger<RuntimeReply>(
options,
triggerObjectId,
`function(key) {
const state = this.ownerDocument.defaultView?.[key];
return state
? { count: state.inputs.length, multiple: state.inputs[0]?.multiple === true }
: { count: 0, multiple: false };
})()`,
returnByValue: true,
}),
deadline,
options.signal,
"resolving activated file input timed out",
);
const summaryError = runtimeError(summary, "failed to inspect activated file input");
if (summaryError) throw summaryError;
const value = summary.result?.value as
| { count?: unknown; multiple?: unknown; pickerCall?: unknown }
| undefined;
const count = typeof value?.count === "number" ? value.count : 0;
const multiple = value?.multiple === true;
if (typeof value?.pickerCall === "string") {
return uploadError(
"unsupported",
`upload trigger invoked ${value.pickerCall} instead of an input[type=file]`,
"file_input_not_activated",
"resolve_input",
}`,
[stateKey],
true,
);
}
if (count !== 1) {
return uploadError(
"unsupported",
count === 0
? "upload trigger did not activate an input[type=file]"
: "upload trigger activated more than one input[type=file]",
"file_input_not_activated",
"resolve_input",
);
}
if (!multiple && options.files.length !== 1) {
return { code: "invalid_params", message: "file input accepts exactly one file" };
const summaryError = runtimeError(reply, "failed to inspect activated file input");
if (summaryError) throw summaryError;
const value = reply.result?.value as { count?: unknown; multiple?: unknown } | undefined;
summary = {
count: typeof value?.count === "number" ? value.count : 0,
multiple: value?.multiple === true,
};
if (summary.count > 0) break;
await delay(Math.min(PROBE_INTERVAL_MS, remainingMs(activationDeadline)), options.signal);
}
const input = await waitBounded(
sendToCdpTarget<RuntimeReply>(options.cdp, options.target, "Runtime.evaluate", {
expression: `globalThis[${stateKeyLiteral}]?.inputs[0]`,
objectGroup,
returnByValue: false,
}),
deadline,
options.signal,
"resolving activated file input object timed out",
);
const inputError = runtimeError(input, "failed to resolve activated file input object");
if (inputError) throw inputError;
const inputObjectId = input.result?.objectId;
if (!inputObjectId) throw new Error("activated file input returned no objectId");
if (chooserEvents.length > 1) {
outcome = transferFailure(
"unsupported",
"file_input_not_activated",
"upload trigger activated more than one file chooser",
"none",
"resolve_input",
);
return outcome;
}
const described = await waitBounded(
sendToCdpTarget<{ node?: { backendNodeId?: number } }>(
const chooser = chooserEvents[0];
let backendNodeId: number | undefined;
let multiple = summary.multiple;
if (chooser) {
if (
options.actionTarget.frameId &&
chooser.frameId !== options.actionTarget.frameId
) {
outcome = transferFailure(
"unsupported",
"file_input_not_activated",
"upload trigger activated a file chooser in a different frame",
"none",
"resolve_input",
);
return outcome;
}
if (typeof chooser.backendNodeId !== "number") {
outcome = transferFailure(
"unsupported",
"file_input_not_activated",
"upload trigger invoked a non-input file picker",
"none",
"resolve_input",
);
return outcome;
}
backendNodeId = chooser.backendNodeId;
multiple = chooser.mode === "selectMultiple";
} else {
if (summary.count !== 1) {
outcome = transferFailure(
"unsupported",
"file_input_not_activated",
summary.count === 0
? "upload trigger did not activate an input[type=file]"
: "upload trigger activated more than one input[type=file]",
"none",
"resolve_input",
);
return outcome;
}
const input = await callOnTrigger<RuntimeReply>(
options,
triggerObjectId,
`function(key) { return this.ownerDocument.defaultView?.[key]?.inputs[0]; }`,
[stateKey],
false,
);
const inputError = runtimeError(input, "failed to resolve activated file input object");
if (inputError) throw inputError;
if (!input.result?.objectId) throw new Error("activated file input returned no objectId");
const described = await sendToCdpTarget<{ node?: { backendNodeId?: number } }>(
options.cdp,
options.target,
target,
"DOM.describeNode",
{ objectId: inputObjectId },
),
deadline,
options.signal,
"describing activated file input timed out",
);
const backendNodeId = described.node?.backendNodeId;
if (typeof backendNodeId !== "number") {
throw new Error("DOM.describeNode returned no file input backendNodeId");
{ objectId: input.result.objectId },
);
backendNodeId = described.node?.backendNodeId;
if (typeof backendNodeId !== "number") {
throw new Error("DOM.describeNode returned no file input backendNodeId");
}
}
if (!multiple && options.files.length !== 1) {
outcome = enrichFailure(
{ code: "invalid_params", message: "file input accepts exactly one file" },
"none",
"resolve_input",
);
return outcome;
}
try {
await waitBounded(
sendToCdpTarget(options.cdp, options.target, "DOM.setFileInputFiles", {
sendToCdpTarget(options.cdp, target, "DOM.setFileInputFiles", {
files: options.files,
backendNodeId,
}),
@@ -268,72 +389,87 @@ export async function uploadThroughActivatedFileInput(
"setting file input files timed out",
);
} catch (err) {
if (err instanceof BoundedWaitError && err.kind === "aborted") throw err;
return uploadError(
outcome = transferFailure(
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
err instanceof Error ? err.message : String(err),
"set_file_input_failed",
err instanceof Error ? err.message : String(err),
"unknown",
"set_files",
);
return outcome;
}
return { click, multiple };
outcome = { click, multiple };
return outcome;
} catch (err) {
if (err instanceof BoundedWaitError && err.kind === "aborted") throw err;
return uploadError(
outcome = transferFailure(
err instanceof BoundedWaitError ? "timeout" : "cdp_failed",
err instanceof Error ? err.message : String(err),
"file_input_probe_failed",
err instanceof Error ? err.message : String(err),
"none",
"resolve_input",
);
return outcome;
}
} catch (err) {
if (err instanceof BoundedWaitError && err.kind === "aborted") {
return { code: "cancelled", message: "upload transaction aborted" };
}
throw err;
} finally {
if (probeAttempted) {
chooserSubscription?.dispose();
let cleanupFailed = false;
if (probeArmed && triggerObjectId) {
try {
await waitBounded(
sendToCdpTarget(options.cdp, options.target, "Runtime.evaluate", {
expression: `(() => {
const key = ${stateKeyLiteral};
const state = globalThis[key];
if (state?.listener) {
document.removeEventListener("click", state.listener, true);
}
const win = document.defaultView;
if (win) {
for (const picker of state?.pickers || []) {
try {
if (picker.hadOwn && picker.descriptor) {
Object.defineProperty(win, picker.name, picker.descriptor);
} else {
delete win[picker.name];
}
} catch {}
}
}
delete globalThis[key];
})()`,
}),
callOnTrigger(
options,
triggerObjectId,
`function(key) {
const owner = this.ownerDocument.defaultView;
const state = owner?.[key];
if (state?.listener) this.ownerDocument.removeEventListener("click", state.listener, true);
if (owner) delete owner[key];
}`,
[stateKey],
true,
),
Date.now() + CLEANUP_TIMEOUT_MS,
undefined,
"cleaning file input probe timed out",
);
} catch {
// Navigation may have invalidated the object; its document is gone too.
cleanupFailed = true;
}
}
if (interceptionArmed) {
try {
await waitBounded(
sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", {
enabled: false,
cancel: true,
}),
Date.now() + CLEANUP_TIMEOUT_MS,
undefined,
"disabling file chooser interception timed out",
);
} catch {
cleanupFailed = true;
}
}
try {
await waitBounded(
sendToCdpTarget(options.cdp, options.target, "Runtime.releaseObjectGroup", {
objectGroup,
}),
sendToCdpTarget(options.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }),
Date.now() + CLEANUP_TIMEOUT_MS,
undefined,
"releasing upload object group timed out",
);
} catch {}
} catch {
cleanupFailed = true;
}
const effect = isRpcError(outcome)
? (outcome.data?.effect_state as TransferEffectState | undefined)
: "committed";
if (effect === "unknown" || cleanupFailed) {
await options.cdp.detach?.(target.tabId);
}
if (cleanupFailed && isRpcError(outcome)) {
outcome.data = { ...outcome.data, cleanup_state: "failed" };
}
}
}
+38 -8
View File
@@ -40,6 +40,7 @@ import {
enforceAgentWindow,
isRpcError,
lookupSession,
type ResolvedTargetTab,
resolveTargetTab,
} from "./shared";
import { resolveSnapshotRef } from "./snapshot-ref";
@@ -56,6 +57,15 @@ export interface InteractionDeps {
keepOverlayBypassAfterHover?: boolean;
}
export interface ResolvedActionTarget {
tab: ResolvedTargetTab;
backendNodeId: number;
cdpTarget: CdpTarget;
frameId?: string;
usedRef?: string;
usedSelector?: string;
}
const DEFAULT_TIMEOUT_MS = 30_000;
const DEFAULT_HOVER_SETTLE_MS = 200;
@@ -215,6 +225,17 @@ export async function resolveBackendNode(
}
}
export async function resolveActionTarget(
cdp: CdpRunner,
ctx: SessionContext,
target: ResolvedTargetTab,
params: { ref?: string; selector?: string },
toolName: string,
): Promise<ResolvedActionTarget | RpcError> {
const node = await resolveBackendNode(cdp, ctx, target, params, toolName);
return isRpcError(node) ? node : { tab: target, ...node };
}
// ---------------------------------------------------------------------------
// tool.click
// ---------------------------------------------------------------------------
@@ -233,10 +254,19 @@ export async function handleClick(
if (isRpcError(target)) return target;
const denied = enforceAgentWindow(ctx, target, "click");
if (denied) return denied;
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
const resolved = await resolveActionTarget(deps.cdp, ctx, target, params, "click");
if (isRpcError(resolved)) return resolved;
return clickResolvedTarget(ctx, resolved, params, deps);
}
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "click");
if (isRpcError(node)) return node;
export async function clickResolvedTarget(
ctx: SessionContext,
resolved: ResolvedActionTarget,
params: Pick<ClickParams, "button" | "click_count" | "modifiers">,
deps: InteractionDeps,
): Promise<ClickResult | RpcError> {
const { tab: target } = resolved;
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "click aborted" };
@@ -247,9 +277,9 @@ export async function handleClick(
deps.cdp,
target.tabId,
{
target: node.cdpTarget,
backendNodeId: node.backendNodeId,
...(node.frameId ? { frameId: node.frameId } : {}),
target: resolved.cdpTarget,
backendNodeId: resolved.backendNodeId,
...(resolved.frameId ? { frameId: resolved.frameId } : {}),
},
{ scrollIntoView: true },
);
@@ -337,8 +367,8 @@ export async function handleClick(
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
tab_id: target.tabId,
used_ref: node.usedRef,
used_selector: node.usedSelector,
used_ref: resolved.usedRef,
used_selector: resolved.usedSelector,
x: centre.x,
y: centre.y,
});
+9 -14
View File
@@ -3,9 +3,13 @@
// transaction module.
import type { SessionManager } from "@/session-manager/manager";
import type { ClickParams, RpcError, UploadParams, UploadResult } from "@/transport/types";
import type { RpcError, UploadParams, UploadResult } from "@/transport/types";
import { uploadThroughActivatedFileInput } from "./file-input-transaction";
import { handleClick, type InteractionDeps, resolveBackendNode } from "./interaction";
import {
clickResolvedTarget,
type InteractionDeps,
resolveActionTarget,
} from "./interaction";
import {
type CdpRunner,
enforceAgentWindow,
@@ -38,25 +42,16 @@ export async function handleUpload(
) {
return { code: "invalid_params", message: "upload requires daemon-staged files" };
}
const address = await resolveBackendNode(deps.cdp, ctx, target, params, "upload");
const address = await resolveActionTarget(deps.cdp, ctx, target, params, "upload");
if (isRpcError(address)) return address;
const timeoutMs = params.timeout_ms ?? DEFAULT_TIMEOUT_MS;
const transaction = await uploadThroughActivatedFileInput({
cdp: deps.cdp,
target: address.cdpTarget,
actionTarget: address,
files: params.files.map((file) => file.staged_path as string),
timeoutMs,
signal: deps.signal,
trigger: (remaining) => {
const clickParams: ClickParams = {
session_id: params.session_id,
ref: params.ref,
selector: params.selector,
tab_id: params.tab_id,
timeout_ms: Math.max(1, remaining),
};
return handleClick(manager, clickParams, deps);
},
trigger: () => clickResolvedTarget(ctx, address, {}, deps),
});
if (isRpcError(transaction)) return transaction;
return {
+9
View File
@@ -37,10 +37,18 @@ export type RpcErrorReason =
| "file_input_not_activated"
| "set_file_input_failed"
| "download_capture_failed"
| "transfer_outcome_unknown"
| "transfer_timeout"
| "cleanup_failed";
export type TransferEffectState = "none" | "committed" | "unknown";
export type TransferCleanupState = "complete" | "failed";
export interface RpcErrorData {
reason?: RpcErrorReason;
effect_state?: TransferEffectState;
phase?: string;
cleanup_state?: TransferCleanupState;
[key: string]: unknown;
}
@@ -533,6 +541,7 @@ export interface DownloadParams {
tab_id?: number;
timeout_ms?: number;
browser_relative_dir?: string;
max_byte_size?: number;
}
export interface DownloadResult {