feat(bsk): support upload and download

This commit is contained in:
Ljy-0827
2026-08-24 14:59:11 +08:00
parent 4b6cdde168
commit 6800f8fdfe
33 changed files with 2038 additions and 9 deletions
+6 -3
View File
@@ -1,6 +1,6 @@
# BrowserSkill — Privacy Policy
**Last updated:** May 25, 2026
**Last updated:** August 20, 2026
This Privacy Policy describes how the **BrowserSkill** browser extension (the "Extension") handles information when you install and use it. BrowserSkill is published as part of the open-source [BrowserSkill](https://github.com/Tencent/BrowserSkill) project. The source code is publicly auditable.
@@ -14,7 +14,7 @@ BrowserSkill is a local automation bridge that lets AI coding agents (such as Cu
## 2. Single Purpose
The Extension's single purpose is to expose browser automation primitives (navigation, DOM observation, screenshots, clicks, form filling, tab management) to a locally running BrowserSkill daemon over a WebSocket connection on `127.0.0.1`, so that an AI agent invoked by the user can interact with web pages on the user's behalf.
The Extension's single purpose is to expose browser automation primitives (navigation, DOM observation, screenshots, clicks, form filling, task-scoped file transfer, and tab management) to a locally running BrowserSkill daemon over a WebSocket connection on `127.0.0.1`, so that an AI agent invoked by the user can interact with web pages on the user's behalf.
## 3. Data the Extension Accesses
@@ -26,6 +26,7 @@ Depending on the commands the user (via their AI agent) sends to the local daemo
| **User input simulated by the agent** | Mouse clicks, keystrokes, and form values that the AI agent dispatches through the Chrome DevTools Protocol (CDP). | Required to perform automation actions the user has asked the agent to do. |
| **Tab and window metadata** | Tab IDs, URLs, titles, window IDs of the Agent Window and any tabs the user explicitly authorizes. | Required to target automation commands at the correct tab/window. |
| **Local extension storage** | A randomly generated 8-character instance ID and an optional user-supplied label. | Used so the local daemon can recognize this browser instance across reconnects. No personal data is stored. |
| **File transfers requested by the agent** | Local files explicitly supplied to `bsk upload`, and the file created by a single `bsk download` action. | Required to attach a task file to a web page or return a browser-generated download to the invoking local agent. |
| **OS notifications** | Permission to display a system notification when the agent requests to "borrow" one of the user's existing tabs. | Required to obtain explicit, per-tab user consent before the agent touches any pre-existing tab. |
## 4. Data the Extension Does **Not** Collect
@@ -34,7 +35,7 @@ BrowserSkill does **not**:
- Send any data to remote servers, the Extension's authors, or any third party.
- Call any LLM, AI, or cloud API. The Extension contains no API keys, model identifiers, or remote endpoints.
- Read or transmit cookies, browsing history, bookmarks, downloads, saved passwords, or autofill data.
- Read or transmit cookies, browsing history, bookmarks, saved passwords, or autofill data. It observes only the download initiated by an active `bsk download` call, not download history generally.
- Use webcam, microphone, geolocation, or any device sensor.
- Include analytics, telemetry, crash reporting, advertising SDKs, or fingerprinting code.
- Track users across websites or across sessions.
@@ -50,6 +51,7 @@ The Extension requests the following Chrome permissions. Each is used solely for
- **`alarms`** — Periodically wake the service worker to keep the local WebSocket connection alive.
- **`idle`** — Detect when the device returns from idle/locked so the Extension can promptly re-establish the local WebSocket connection after the machine wakes. No idle data is stored or transmitted.
- **`notifications`** — Show a system notification to obtain user approval before the agent borrows a user-owned tab.
- **`downloads`** — Observe and, on cancellation, stop the one browser download initiated by an active `bsk download` command. It is not used to enumerate download history.
- **`storage`** — Persist a random instance ID and optional label in `chrome.storage.local`.
- **Host permission `<all_urls>`** — Inject a small status overlay (showing "Agent Active") on pages controlled by the agent, and enable automation across whatever sites the user directs the agent to. The Extension does **not** read or transmit page content from sites the agent is not actively driving.
@@ -61,6 +63,7 @@ All Extension activity stays on the user's local device. The only network traffi
- The instance ID and optional label persist in `chrome.storage.local` until the user uninstalls the Extension or clears extension storage.
- Page content, screenshots, DOM snapshots, and other observed data are returned to the local daemon in response to commands and are **not retained by the Extension**. They live only as long as the agent's tool call.
- Upload and download bytes are staged by the local daemon in a private, session-scoped directory. Download staging is removed after it is copied to the requested destination. Upload staging is retained until the session ends so a later form submission can still read the attached file. Remaining staging is removed when the session ends or disconnects, or when the daemon next starts after a crash.
## 8. User Control
@@ -0,0 +1,144 @@
import { describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import { type DownloadsApi, handleDownload } from "../download";
import type { CdpRunner } from "../shared";
import { handleUpload } from "../upload";
function sessions() {
return new SessionManager({
agentWindow: {
create: vi.fn(async () => 100),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
},
});
}
function tabsApi() {
return {
get: vi.fn(
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
};
}
function fakeEvent<T extends (...args: never[]) => void>() {
const listeners = new Set<T>();
return {
addListener: (listener: T) => listeners.add(listener),
removeListener: (listener: T) => listeners.delete(listener),
hasListener: (listener: T) => listeners.has(listener),
hasListeners: () => listeners.size > 0,
emit: (...args: Parameters<T>) => {
for (const listener of listeners) listener(...args);
},
};
}
describe("file transfer tools", () => {
it("arms chooser interception before clicking and injects only staged paths", async () => {
const manager = sessions();
const ctx = await manager.start("s1");
ctx.refStore.set("e3", 123, { tabId: 4 });
let eventHandler: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
const calls: Array<{ method: string; params?: object }> = [];
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
calls.push({ method, params });
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
if (
method === "Input.dispatchMouseEvent" &&
(params as { type?: string }).type === "mousePressed"
) {
eventHandler?.({ tabId: 4 }, "Page.fileChooserOpened", {
backendNodeId: 456,
mode: "selectMultiple",
});
}
return {};
});
const cdp: CdpRunner = {
onEvent: (handler) => {
eventHandler = handler;
return { dispose: vi.fn() };
},
send: send as unknown as CdpRunner["send"],
};
const result = await handleUpload(
manager,
{
session_id: "s1",
ref: "@e3",
files: [
{ transfer_id: "tr_1", name: "one.png", staged_path: "/private/stage/one" },
{ transfer_id: "tr_2", name: "two.png", staged_path: "/private/stage/two" },
],
},
{ cdp, tabsApi: tabsApi() },
);
expect(result).toMatchObject({ tab_id: 4, file_names: ["one.png", "two.png"] });
expect(calls[0]).toMatchObject({
method: "Page.setInterceptFileChooserDialog",
params: { enabled: true },
});
expect(calls).toContainEqual({
method: "DOM.setFileInputFiles",
params: { files: ["/private/stage/one", "/private/stage/two"], backendNodeId: 456 },
});
});
it("captures the one download created under daemon staging", async () => {
const manager = sessions();
const ctx = await manager.start("s1");
ctx.refStore.set("e3", 123, { tabId: 4 });
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
const item = {
id: 9,
filename: "/private/stage/tr_1/result.zip",
state: "complete",
fileSize: 12,
totalBytes: 12,
mime: "application/zip",
danger: "safe",
} as chrome.downloads.DownloadItem;
const downloads: DownloadsApi = {
onCreated: onCreated as unknown as DownloadsApi["onCreated"],
onChanged: onChanged as unknown as DownloadsApi["onChanged"],
search: vi.fn(async () => [item]),
cancel: vi.fn(async () => {}),
};
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
if (
method === "Input.dispatchMouseEvent" &&
(params as { type?: string }).type === "mousePressed"
) {
onCreated.emit(item);
}
return {};
});
const cdp: CdpRunner = {
send: send as unknown as CdpRunner["send"],
};
const result = await handleDownload(
manager,
{ session_id: "s1", ref: "@e3", staging_path: "/private/stage/tr_1" },
{ cdp, tabsApi: tabsApi(), downloads },
);
expect(result).toMatchObject({
tab_id: 4,
suggested_filename: "result.zip",
byte_size: 12,
staged_path: item.filename,
});
});
});
+38
View File
@@ -4,6 +4,7 @@ import type { Transport } from "@/transport/transport";
import type {
ClickParams,
ConsoleParams,
DownloadParams,
EmulateParams,
EvaluateParams,
FillParams,
@@ -28,10 +29,12 @@ import type {
ScreenshotParams,
SelectParams,
SnapshotParams,
UploadParams,
WaitForNavigationParams,
} from "@/transport/types";
import { isRequestFrame } from "@/transport/types";
import { handleConsole } from "./console";
import { handleDownload } from "./download";
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
import { handleEvaluate } from "./evaluate";
import { handleRequestHelp } from "./human-loop";
@@ -79,6 +82,7 @@ import {
type TabReturnParams,
type TabSelectParams,
} from "./tabs";
import { handleUpload } from "./upload";
import { handleWaitForNavigation } from "./waits";
import { handleWindowResize, type WindowResizeParams } from "./window";
@@ -513,6 +517,38 @@ export class ToolDispatcher {
),
signal,
);
case "tool.upload":
return this.withHoverReleaseForRequest(
req.params as UploadParams,
() =>
this.cdp
? handleUpload(this.sessions, req.params as UploadParams, {
cdp: this.cdp,
tabsApi: chromeTabsApi,
signal,
})
: Promise.resolve({
code: "unsupported",
message: "upload requires CDP",
} satisfies RpcError),
signal,
);
case "tool.download":
return this.withHoverReleaseForRequest(
req.params as DownloadParams,
() =>
this.cdp
? handleDownload(this.sessions, req.params as DownloadParams, {
cdp: this.cdp,
tabsApi: chromeTabsApi,
signal,
})
: Promise.resolve({
code: "unsupported",
message: "download requires CDP",
} satisfies RpcError),
signal,
);
case "tool.evaluate":
return handleEvaluate(
this.sessions,
@@ -719,6 +755,8 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
case "tool.fill":
case "tool.press":
case "tool.select":
case "tool.upload":
case "tool.download":
case "tool.evaluate":
case "tool.observe":
case "tool.request_help":
+138
View File
@@ -0,0 +1,138 @@
// One-click/one-file browser download capture. chrome.downloads events are
// browser-global, so a coordinator prevents two bsk sessions from racing.
import type { SessionManager } from "@/session-manager/manager";
import type { ClickParams, DownloadParams, DownloadResult, RpcError } from "@/transport/types";
import { handleClick, type InteractionDeps } from "./interaction";
import { enforceAgentWindow, isRpcError, lookupSession, resolveTargetTab } from "./shared";
export interface DownloadsApi {
onCreated: chrome.events.Event<(item: chrome.downloads.DownloadItem) => void>;
onChanged: chrome.events.Event<(delta: chrome.downloads.DownloadDelta) => void>;
search(query: chrome.downloads.DownloadQuery): Promise<chrome.downloads.DownloadItem[]>;
cancel(downloadId: number): Promise<void>;
}
const chromeDownloadsApi: DownloadsApi = {
get onCreated() {
return chrome.downloads.onCreated;
},
get onChanged() {
return chrome.downloads.onChanged;
},
search: (query) => chrome.downloads.search(query),
cancel: (id) => chrome.downloads.cancel(id),
};
let downloadActive = false;
export interface DownloadDeps extends InteractionDeps {
downloads?: DownloadsApi;
}
export async function handleDownload(
manager: SessionManager,
params: DownloadParams,
deps: DownloadDeps,
): Promise<DownloadResult | RpcError> {
if (downloadActive) return { code: "invalid_params", message: "another bsk download is active" };
downloadActive = true;
const downloads = deps.downloads ?? chromeDownloadsApi;
let capturedId: number | undefined;
let createdListener: ((item: chrome.downloads.DownloadItem) => void) | undefined;
let changedListener: ((delta: chrome.downloads.DownloadDelta) => void) | undefined;
let timer: ReturnType<typeof setTimeout> | undefined;
try {
const ctx = lookupSession(manager, params, "download");
if (isRpcError(ctx)) return ctx;
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
if (isRpcError(target)) return target;
const denied = enforceAgentWindow(ctx, target, "download");
if (denied) return denied;
if (!params.staging_path)
return { code: "invalid_params", message: "download requires daemon staging" };
const normalisePath = (path: string) => path.replaceAll("\\", "/");
const normalisedStaging = normalisePath(params.staging_path);
const stagingPrefix = normalisedStaging.endsWith("/")
? normalisedStaging
: `${normalisedStaging}/`;
let rejectCompleted!: (error: Error) => void;
const completed = new Promise<chrome.downloads.DownloadItem>((resolve, reject) => {
rejectCompleted = reject;
createdListener = (item) => {
if (!normalisePath(item.filename).startsWith(stagingPrefix)) return;
if (capturedId !== undefined && capturedId !== item.id) {
reject(new Error("download trigger produced more than one file"));
return;
}
capturedId = item.id;
if (item.state === "complete") resolve(item);
};
changedListener = async (delta) => {
if (capturedId === undefined || delta.id !== capturedId) return;
if (delta.state?.current === "interrupted" || delta.error?.current) {
reject(new Error(delta.error?.current ?? "download interrupted"));
return;
}
if (delta.state?.current === "complete") {
const [item] = await downloads.search({ id: delta.id });
if (item) resolve(item);
else reject(new Error("completed download disappeared"));
}
};
downloads.onCreated.addListener(createdListener);
downloads.onChanged.addListener(changedListener);
timer = setTimeout(
() => reject(new Error("download did not complete before timeout")),
params.timeout_ms ?? 120_000,
);
});
const onAbort = () => {
if (capturedId !== undefined) void downloads.cancel(capturedId).catch(() => undefined);
rejectCompleted(new DOMException("aborted", "AbortError"));
};
deps.signal?.addEventListener("abort", onAbort, { once: true });
try {
await deps.cdp.send(target.tabId, "Page.setDownloadBehavior", {
behavior: "allow",
downloadPath: params.staging_path,
});
const clickParams: ClickParams = {
session_id: params.session_id,
ref: params.ref,
selector: params.selector,
tab_id: params.tab_id,
timeout_ms: params.timeout_ms,
};
const clicked = await handleClick(manager, clickParams, deps);
if (isRpcError(clicked)) {
void completed.catch(() => undefined);
return clicked;
}
const item = await completed;
return {
tab_id: target.tabId,
used_ref: clicked.used_ref,
used_selector: clicked.used_selector,
suggested_filename: item.filename.split(/[\\/]/).pop() ?? "download",
byte_size: item.fileSize >= 0 ? item.fileSize : item.totalBytes,
mime: item.mime || undefined,
danger: item.danger,
staged_path: item.filename,
};
} finally {
deps.signal?.removeEventListener("abort", onAbort);
try {
await deps.cdp.send(target.tabId, "Page.setDownloadBehavior", { behavior: "default" });
} catch {}
}
} catch (err) {
if (err instanceof DOMException && err.name === "AbortError") throw err;
return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err) };
} finally {
if (timer) clearTimeout(timer);
if (createdListener) downloads.onCreated.removeListener(createdListener);
if (changedListener) downloads.onChanged.removeListener(changedListener);
downloadActive = false;
}
}
+148
View File
@@ -0,0 +1,148 @@
// Atomic file-chooser upload: arm interception before clicking, then attach
// daemon-staged files to the chooser node. No local path is accepted from an
// agent-facing call; staged_path is injected by the daemon.
import type { CdpTarget } from "@/browser-driver/frame-graph";
import type { SessionManager } from "@/session-manager/manager";
import type { ClickParams, RpcError, UploadParams, UploadResult } from "@/transport/types";
import { handleClick, type InteractionDeps } from "./interaction";
import {
type CdpRunner,
enforceAgentWindow,
isRpcError,
lookupSession,
resolveTargetTab,
sendToCdpTarget,
} from "./shared";
import { resolveSnapshotRef } from "./snapshot-ref";
const DEFAULT_TIMEOUT_MS = 120_000;
export interface UploadDeps extends InteractionDeps {
cdp: CdpRunner;
}
function chooserTarget(
params: UploadParams,
tabId: number,
manager: SessionManager,
): CdpTarget | RpcError {
const ctx = manager.get(params.session_id);
if (!ctx) return { code: "not_found", message: `session ${params.session_id} unknown` };
if (params.ref) {
const ref = resolveSnapshotRef(ctx, params.ref, tabId);
if (isRpcError(ref)) return ref;
return { tabId, ...(ref.cdpSessionId ? { sessionId: ref.cdpSessionId } : {}) };
}
return { tabId };
}
export async function handleUpload(
manager: SessionManager,
params: UploadParams,
deps: UploadDeps,
): Promise<UploadResult | RpcError> {
const ctx = lookupSession(manager, params, "upload");
if (isRpcError(ctx)) return ctx;
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
if (isRpcError(target)) return target;
const denied = enforceAgentWindow(ctx, target, "upload");
if (denied) return denied;
if (!deps.cdp?.onEvent)
return { code: "unsupported", message: "CDP event subscription unavailable" };
if (
params.files.length === 0 ||
params.files.length > 20 ||
params.files.some((file) => !file.staged_path)
) {
return { code: "invalid_params", message: "upload requires daemon-staged files" };
}
const cdpTarget = chooserTarget(params, target.tabId, manager);
if (isRpcError(cdpTarget)) return cdpTarget;
const timeoutMs = params.timeout_ms ?? DEFAULT_TIMEOUT_MS;
let timer: ReturnType<typeof setTimeout> | undefined;
let subscription: { dispose(): void } | undefined;
let resolveChooser!: (value: {
source: { tabId?: number; sessionId?: string };
backendNodeId: number;
mode?: string;
}) => void;
let rejectChooser!: (error: Error) => void;
const chooser = new Promise<{
source: { tabId?: number; sessionId?: string };
backendNodeId: number;
mode?: string;
}>((resolve, reject) => {
resolveChooser = resolve;
rejectChooser = reject;
});
subscription = deps.cdp.onEvent((source, method, raw) => {
if (method !== "Page.fileChooserOpened" || source.tabId !== target.tabId) return;
if (cdpTarget.sessionId && source.sessionId !== cdpTarget.sessionId) return;
const event = raw as { backendNodeId?: unknown; mode?: unknown };
if (typeof event.backendNodeId !== "number") {
rejectChooser(new Error("file chooser did not expose an input backendNodeId"));
return;
}
resolveChooser({
source,
backendNodeId: event.backendNodeId,
...(typeof event.mode === "string" ? { mode: event.mode } : {}),
});
});
timer = setTimeout(
() => rejectChooser(new Error("file chooser did not open before timeout")),
timeoutMs,
);
const onAbort = () => rejectChooser(new DOMException("aborted", "AbortError"));
deps.signal?.addEventListener("abort", onAbort, { once: true });
try {
await sendToCdpTarget(deps.cdp, cdpTarget, "Page.setInterceptFileChooserDialog", {
enabled: true,
});
const clickParams: ClickParams = {
session_id: params.session_id,
ref: params.ref,
selector: params.selector,
tab_id: params.tab_id,
timeout_ms: params.timeout_ms,
};
const clicked = await handleClick(manager, clickParams, deps);
if (isRpcError(clicked)) {
void chooser.catch(() => undefined);
return clicked;
}
const opened = await chooser;
if (opened.mode === "selectSingle" && params.files.length !== 1) {
return { code: "invalid_params", message: "file chooser accepts exactly one file" };
}
const sourceTarget: CdpTarget = {
tabId: target.tabId,
...(opened.source.sessionId ? { sessionId: opened.source.sessionId } : {}),
};
await sendToCdpTarget(deps.cdp, sourceTarget, "DOM.setFileInputFiles", {
files: params.files.map((file) => file.staged_path as string),
backendNodeId: opened.backendNodeId,
});
return {
tab_id: target.tabId,
used_ref: clicked.used_ref,
used_selector: clicked.used_selector,
file_names: params.files.map((file) => file.name),
};
} catch (err) {
if (err instanceof DOMException && err.name === "AbortError") throw err;
return { code: "cdp_failed", message: err instanceof Error ? err.message : String(err) };
} finally {
if (timer) clearTimeout(timer);
subscription?.dispose();
deps.signal?.removeEventListener("abort", onAbort);
try {
await sendToCdpTarget(deps.cdp, cdpTarget, "Page.setInterceptFileChooserDialog", {
enabled: false,
});
} catch {
// Best-effort compensation; the target may have navigated/closed.
}
}
}
+43
View File
@@ -500,6 +500,49 @@ export interface SelectResult {
dialogs?: JavaScriptDialogInfo[];
}
export interface UploadFile {
transfer_id: string;
name: string;
staged_path?: string;
}
export interface UploadParams {
session_id: string;
ref?: string;
selector?: string;
tab_id?: number;
files: UploadFile[];
timeout_ms?: number;
}
export interface UploadResult {
tab_id: number;
used_ref?: string;
used_selector?: string;
file_names: string[];
}
export interface DownloadParams {
session_id: string;
ref?: string;
selector?: string;
tab_id?: number;
timeout_ms?: number;
staging_path?: string;
}
export interface DownloadResult {
tab_id: number;
used_ref?: string;
used_selector?: string;
suggested_filename: string;
byte_size: number;
mime?: string;
danger?: string;
staged_path?: string;
transfer_id?: string;
}
// --------------------------------------------------------------------------
// M9 tool payloads — evaluate / wait_for_navigation / wait_ms
// --------------------------------------------------------------------------
+1
View File
@@ -30,6 +30,7 @@ export default defineConfig({
permissions: [
"alarms",
"debugger",
"downloads",
"idle",
"notifications",
"tabs",