mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(packages): add dsh-plugin-browserskill — DeepSeek Harness tools for bsk
Add a new monorepo package that registers BrowserSkill (bsk) browser automation as model-visible tools in DeepSeek Harness (dsh): - 11 tools mapping to bsk CLI --json commands: session start/stop/list, navigate, snapshot, observe, click, fill, press, screenshot, emulate - multi-session support: optional session arg with a current-session pointer, session id echoed in every result, configurable concurrency cap (default 5), and dispose-time cleanup of plugin-started sessions - cancellation: exec.signal aborts kill the underlying bsk child process - terminal-style UI cards (command line as the call card, output as the result card); screenshots inline the PNG through the host attachment store when the model route accepts image input, else return the path - config: bskPath / defaultTimeoutMs / maxSessions via Schemastery schema - install guidance when the bsk binary is missing; bsk JSON error envelopes (code/message/hint) surfaced to the model - unit tests mock the bsk runner (no real browser required); the root lint script now also runs the package's typecheck + tests so the existing CI frontend job covers the new package (a dedicated CI job diff is provided in the PR; the pushing token lacks workflow scope)
This commit is contained in:
BB-fat
@@ -0,0 +1,83 @@
|
||||
/**
|
||||
* Screenshot delivery. The canonical tool value stays plain JSON (a PNG file
|
||||
* path plus pixel metadata); when the host mounts a durable attachment store
|
||||
* AND the calling route declares image input, the PNG bytes are additionally
|
||||
* committed through `ctx.attachments` so the render step can attach the image
|
||||
* itself to the tool result. Any uncertainty (no store, unknown route,
|
||||
* text-only model) falls back to the path-only form.
|
||||
*/
|
||||
|
||||
import type { Context } from "@deepseek-ai/cordis";
|
||||
import type { ImageAttachmentRef } from "@deepseek-ai/dsh-attachment";
|
||||
import type { ToolExecution } from "@deepseek-ai/dsh-tools";
|
||||
|
||||
/** Structural view of the attachment seam (avoids a runtime dependency). */
|
||||
interface AttachmentLike {
|
||||
imageLimits: {
|
||||
mediaTypes: readonly string[];
|
||||
maxImageBytes: number;
|
||||
maxMessageImageBytes: number;
|
||||
};
|
||||
saveImage(input: {
|
||||
data: Uint8Array;
|
||||
mediaType: string;
|
||||
name?: string;
|
||||
}): Promise<ImageAttachmentRef>;
|
||||
}
|
||||
|
||||
interface LlmLike {
|
||||
resolveModelInfo(
|
||||
provider: string,
|
||||
model: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ inputModalities?: readonly string[] }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Commit screenshot bytes to the host attachment store when the composition
|
||||
* supports durable images on the current model route.
|
||||
* @returns the durable reference, or undefined to stay in path-only mode.
|
||||
*/
|
||||
export async function trySaveScreenshot(
|
||||
ctx: Context,
|
||||
exec: ToolExecution,
|
||||
data: Uint8Array,
|
||||
name: string,
|
||||
): Promise<ImageAttachmentRef | undefined> {
|
||||
const attachments = ctx.get("attachments") as AttachmentLike | undefined;
|
||||
if (attachments === undefined) return undefined;
|
||||
if (!attachments.imageLimits.mediaTypes.includes("image/png")) return undefined;
|
||||
if (
|
||||
data.byteLength >
|
||||
Math.min(attachments.imageLimits.maxImageBytes, attachments.imageLimits.maxMessageImageBytes)
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
if (!(await isImageCapableRoute(ctx, exec))) return undefined;
|
||||
try {
|
||||
return await attachments.saveImage({ data, mediaType: "image/png", name });
|
||||
} catch {
|
||||
// A store hiccup must not fail the tool call; the PNG path still works.
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Best-effort check that the calling route's model accepts image input,
|
||||
* mirroring dsh-tool-fs `read_image`. Unknown route / missing llm service
|
||||
* answers false (refuse to attach) so history never gains an image block a
|
||||
* text-only adapter cannot replay.
|
||||
*/
|
||||
async function isImageCapableRoute(ctx: Context, exec: ToolExecution): Promise<boolean> {
|
||||
const llm = ctx.get("llm") as LlmLike | undefined;
|
||||
const provider =
|
||||
exec.agent?.session.requestHeader()?.config.provider ?? exec.agent?.options.provider;
|
||||
const model = exec.agent?.session.requestHeader()?.config.model ?? exec.agent?.options.model;
|
||||
if (llm === undefined || provider === undefined || model === undefined) return false;
|
||||
try {
|
||||
const info = await llm.resolveModelInfo(provider, model, exec.signal);
|
||||
return info.inputModalities?.includes("image") ?? false;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
/**
|
||||
* dsh-plugin-browserskill — a DeepSeek Harness bundle that registers
|
||||
* BrowserSkill (`bsk`) browser automation as model-visible tools.
|
||||
*
|
||||
* Each tool spawns `bsk <cmd> --json`, parses the structured output, and
|
||||
* returns a canonical JSON value. The plugin tracks the sessions it starts so
|
||||
* one agent conversation can drive several browsers at once; unloading the
|
||||
* plugin stops every tracked session and kills any in-flight bsk children.
|
||||
*
|
||||
* @module dsh-plugin-browserskill
|
||||
*/
|
||||
|
||||
import type { Context } from "@deepseek-ai/cordis";
|
||||
import Schema from "@deepseek-ai/schemastery";
|
||||
import { createBskRunner } from "./runner";
|
||||
import { SessionRegistry } from "./sessions";
|
||||
import { type PluginConfig, registerTools } from "./tools";
|
||||
|
||||
export const name = "dsh-plugin-browserskill";
|
||||
export const inject = ["tools"];
|
||||
|
||||
/** Runtime configuration schema (validated and defaulted by Cordis). */
|
||||
export const Config = Schema.object({
|
||||
bskPath: Schema.string()
|
||||
.default("bsk")
|
||||
.description("Path to the bsk CLI binary (defaults to resolving `bsk` from PATH)."),
|
||||
defaultTimeoutMs: Schema.number()
|
||||
.default(120_000)
|
||||
.description("Default per-command timeout in milliseconds."),
|
||||
maxSessions: Schema.number()
|
||||
.default(5)
|
||||
.description("Maximum number of concurrent browser sessions started through this plugin."),
|
||||
});
|
||||
|
||||
export type Config = PluginConfig;
|
||||
|
||||
export function apply(ctx: Context, config: Partial<PluginConfig> = {}): void {
|
||||
const resolved: PluginConfig = {
|
||||
bskPath: config.bskPath ?? "bsk",
|
||||
defaultTimeoutMs: config.defaultTimeoutMs ?? 120_000,
|
||||
maxSessions: config.maxSessions ?? 5,
|
||||
};
|
||||
const runner = createBskRunner(resolved.bskPath);
|
||||
const registry = new SessionRegistry(resolved.maxSessions);
|
||||
|
||||
registerTools({ ctx, runner, registry, config: resolved });
|
||||
|
||||
// Non-blocking install probe: warn early when bsk is missing instead of
|
||||
// failing the first tool call with a bare spawn error.
|
||||
runner.run(["status"], { timeoutMs: 10_000 }).then(
|
||||
() => {},
|
||||
(error: unknown) => {
|
||||
const detail = error instanceof Error ? error.message : String(error);
|
||||
console.warn(
|
||||
`[${name}] bsk probe failed (${detail}); browser tools will report install guidance until the bsk CLI is available`,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
// Unload cleanup: kill in-flight children, then stop every session this
|
||||
// plugin started (best effort; sessions started outside the plugin are
|
||||
// left running).
|
||||
ctx.effect(() => {
|
||||
return () => {
|
||||
runner.killAll();
|
||||
const stops = registry
|
||||
.list()
|
||||
.map((session) =>
|
||||
runner.run(["session", "stop", session.sessionId], { timeoutMs: 15_000 }).catch(() => {}),
|
||||
);
|
||||
return Promise.all(stops).then(() => {});
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export type { BskRunner, BskRunResult, SpawnImpl } from "./runner";
|
||||
export { BskError, createBskRunner } from "./runner";
|
||||
export { SessionRegistry } from "./sessions";
|
||||
export type { PluginConfig, ToolDeps } from "./tools";
|
||||
export { registerTools } from "./tools";
|
||||
@@ -0,0 +1,194 @@
|
||||
/**
|
||||
* Process runner for the `bsk` CLI. Every model-facing tool in this plugin
|
||||
* maps to one `bsk <cmd> --json` invocation: spawn the child, capture
|
||||
* stdout/stderr, honor the dsh cancellation signal by killing the child, and
|
||||
* map the CLI's JSON error envelope onto a thrown `BskError`.
|
||||
*/
|
||||
|
||||
import { type ChildProcess, spawn } from "node:child_process";
|
||||
|
||||
/** Shape of the JSON error envelope `bsk --json` prints on failure. */
|
||||
export interface BskErrorBody {
|
||||
code?: string;
|
||||
message?: string;
|
||||
hint?: string;
|
||||
exit_code?: number;
|
||||
data?: unknown;
|
||||
}
|
||||
|
||||
/** A failed `bsk` invocation (non-zero exit, timeout, or spawn failure). */
|
||||
export class BskError extends Error {
|
||||
readonly code?: string;
|
||||
readonly hint?: string;
|
||||
readonly exitCode?: number | null;
|
||||
readonly timedOut: boolean;
|
||||
|
||||
constructor(
|
||||
message: string,
|
||||
options: { code?: string; hint?: string; exitCode?: number | null; timedOut?: boolean } = {},
|
||||
) {
|
||||
super(message);
|
||||
this.name = "BskError";
|
||||
this.code = options.code;
|
||||
this.hint = options.hint;
|
||||
this.exitCode = options.exitCode;
|
||||
this.timedOut = options.timedOut ?? false;
|
||||
}
|
||||
}
|
||||
|
||||
export interface BskRunResult {
|
||||
/** Process exit code, or null when killed by a signal / never started. */
|
||||
code: number | null;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
timedOut: boolean;
|
||||
aborted: boolean;
|
||||
}
|
||||
|
||||
export interface BskRunOptions {
|
||||
signal?: AbortSignal;
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
/** Minimal spawn signature so tests can substitute a fake child process. */
|
||||
export type SpawnImpl = (command: string, args: string[]) => ChildProcess;
|
||||
|
||||
export interface BskRunner {
|
||||
/** Run `bsk <args...> --json` and collect its output. */
|
||||
run(args: string[], options?: BskRunOptions): Promise<BskRunResult>;
|
||||
/** Kill every in-flight child (used when the plugin unloads). */
|
||||
killAll(): void;
|
||||
}
|
||||
|
||||
const KILL_GRACE_MS = 2000;
|
||||
|
||||
export function createBskRunner(bskPath: string, spawnImpl: SpawnImpl = spawn): BskRunner {
|
||||
const live = new Set<ChildProcess>();
|
||||
|
||||
function killChild(child: ChildProcess): void {
|
||||
if (child.exitCode !== null || child.signalCode !== null) return;
|
||||
child.kill("SIGTERM");
|
||||
const force = setTimeout(() => {
|
||||
if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
|
||||
}, KILL_GRACE_MS);
|
||||
force.unref();
|
||||
}
|
||||
|
||||
return {
|
||||
run(args, options = {}) {
|
||||
return new Promise<BskRunResult>((resolve, reject) => {
|
||||
let child: ChildProcess;
|
||||
try {
|
||||
child = spawnImpl(bskPath, [...args, "--json"]);
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
live.add(child);
|
||||
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
let timedOut = false;
|
||||
let aborted = false;
|
||||
child.stdout?.on("data", (chunk: Buffer | string) => {
|
||||
stdout += chunk;
|
||||
});
|
||||
child.stderr?.on("data", (chunk: Buffer | string) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
|
||||
const timeoutMs = options.timeoutMs;
|
||||
const timer =
|
||||
timeoutMs !== undefined && timeoutMs > 0
|
||||
? setTimeout(() => {
|
||||
timedOut = true;
|
||||
killChild(child);
|
||||
}, timeoutMs)
|
||||
: undefined;
|
||||
timer?.unref();
|
||||
|
||||
const onAbort = () => {
|
||||
aborted = true;
|
||||
killChild(child);
|
||||
};
|
||||
if (options.signal?.aborted) {
|
||||
onAbort();
|
||||
} else {
|
||||
options.signal?.addEventListener("abort", onAbort, { once: true });
|
||||
}
|
||||
|
||||
const settle = () => {
|
||||
if (timer !== undefined) clearTimeout(timer);
|
||||
options.signal?.removeEventListener("abort", onAbort);
|
||||
live.delete(child);
|
||||
};
|
||||
|
||||
child.on("error", (error) => {
|
||||
settle();
|
||||
reject(error);
|
||||
});
|
||||
child.on("close", (code) => {
|
||||
settle();
|
||||
resolve({ code, stdout, stderr, timedOut, aborted });
|
||||
});
|
||||
});
|
||||
},
|
||||
killAll() {
|
||||
for (const child of live) killChild(child);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** True when the spawn failure means the bsk binary itself is missing. */
|
||||
export function isCommandNotFound(error: unknown): boolean {
|
||||
return (
|
||||
typeof error === "object" &&
|
||||
error !== null &&
|
||||
"code" in error &&
|
||||
(error as { code?: unknown }).code === "ENOENT"
|
||||
);
|
||||
}
|
||||
|
||||
/** Install guidance shown when the bsk CLI cannot be spawned. */
|
||||
export function bskInstallMessage(bskPath: string): string {
|
||||
return (
|
||||
`the bsk CLI ("${bskPath}") was not found. BrowserSkill must be installed and on PATH ` +
|
||||
"for browser tools to work — install it from https://github.com/Tencent/BrowserSkill " +
|
||||
"(see the README install script or `cargo install`), then retry."
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Interpret one finished run: throw `BskError` on timeout / non-zero exit
|
||||
* (parsing the CLI's JSON error envelope when present), otherwise parse and
|
||||
* return the stdout JSON payload.
|
||||
*/
|
||||
export function parseBskJson(result: BskRunResult, commandLabel: string): unknown {
|
||||
if (result.timedOut) {
|
||||
throw new BskError(`bsk ${commandLabel} timed out`, { timedOut: true });
|
||||
}
|
||||
const body = result.stdout.trim();
|
||||
if (result.code !== 0) {
|
||||
let parsed: BskErrorBody | undefined;
|
||||
try {
|
||||
parsed = JSON.parse(body) as BskErrorBody;
|
||||
} catch {
|
||||
parsed = undefined;
|
||||
}
|
||||
const message =
|
||||
parsed?.message ?? (result.stderr.trim() || body || `bsk ${commandLabel} failed`);
|
||||
throw new BskError(`bsk ${commandLabel} failed: ${message}`, {
|
||||
code: parsed?.code,
|
||||
hint: parsed?.hint,
|
||||
exitCode: result.code,
|
||||
});
|
||||
}
|
||||
try {
|
||||
return JSON.parse(body);
|
||||
} catch {
|
||||
throw new BskError(
|
||||
`bsk ${commandLabel} did not produce JSON output: ${body.slice(0, 200) || "(empty)"}`,
|
||||
{ exitCode: result.code },
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
/**
|
||||
* Tracks the bsk sessions started through this plugin and the "current"
|
||||
* session pointer used when a tool call omits its optional `session` arg.
|
||||
* The pointer moves to whatever session was most recently started, stopped,
|
||||
* or operated on, so a model can run several browsers side by side without
|
||||
* repeating the id on every call.
|
||||
*/
|
||||
|
||||
export interface TrackedSession {
|
||||
sessionId: string;
|
||||
browserInstanceId?: string;
|
||||
startedAtMs: number;
|
||||
}
|
||||
|
||||
export class SessionRegistry {
|
||||
private readonly sessions = new Map<string, TrackedSession>();
|
||||
private currentId: string | undefined;
|
||||
|
||||
constructor(private readonly maxSessions: number) {}
|
||||
|
||||
/**
|
||||
* Register a freshly started session and make it current.
|
||||
* @throws when the configured concurrency cap is already reached.
|
||||
*/
|
||||
add(session: TrackedSession): void {
|
||||
if (!this.sessions.has(session.sessionId) && this.sessions.size >= this.maxSessions) {
|
||||
throw new Error(
|
||||
`session limit reached (${this.maxSessions} concurrent sessions); ` +
|
||||
"stop one with browser_session_stop before starting another",
|
||||
);
|
||||
}
|
||||
this.sessions.set(session.sessionId, session);
|
||||
this.currentId = session.sessionId;
|
||||
}
|
||||
|
||||
/** Forget a stopped session; falls back to the most recent remaining one. */
|
||||
remove(sessionId: string): void {
|
||||
this.sessions.delete(sessionId);
|
||||
if (this.currentId === sessionId) {
|
||||
const rest = [...this.sessions.values()];
|
||||
this.currentId = rest.length > 0 ? rest[rest.length - 1].sessionId : undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** Mark a session as most recently used. Unknown ids are adopted. */
|
||||
touch(sessionId: string): void {
|
||||
const existing = this.sessions.get(sessionId);
|
||||
if (existing !== undefined) {
|
||||
// Re-insert to refresh recency order.
|
||||
this.sessions.delete(sessionId);
|
||||
this.sessions.set(sessionId, existing);
|
||||
} else {
|
||||
this.sessions.set(sessionId, { sessionId, startedAtMs: Date.now() });
|
||||
}
|
||||
this.currentId = sessionId;
|
||||
}
|
||||
|
||||
/** The current session id, if any. */
|
||||
current(): string | undefined {
|
||||
return this.currentId;
|
||||
}
|
||||
|
||||
/** Tracked sessions in least- to most-recently-used order. */
|
||||
list(): TrackedSession[] {
|
||||
return [...this.sessions.values()];
|
||||
}
|
||||
|
||||
size(): number {
|
||||
return this.sessions.size;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the session a tool call acts on: an explicit `session` argument
|
||||
* wins (and becomes current); otherwise fall back to the current session.
|
||||
* @throws when neither is available.
|
||||
*/
|
||||
resolve(explicit: string | undefined, toolName: string): string {
|
||||
if (explicit !== undefined && explicit.trim().length > 0) {
|
||||
this.touch(explicit);
|
||||
return explicit;
|
||||
}
|
||||
const current = this.current();
|
||||
if (current === undefined) {
|
||||
throw new Error(
|
||||
`${toolName} needs a session but none is active — start one with browser_session_start ` +
|
||||
"or pass an explicit `session` id",
|
||||
);
|
||||
}
|
||||
this.touch(current);
|
||||
return current;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,853 @@
|
||||
/**
|
||||
* Model-facing tool definitions. Each tool maps to one `bsk <cmd> --json`
|
||||
* invocation: spawn via the runner, parse the JSON envelope, and return a
|
||||
* canonical value whose `output.render` produces the model-facing text.
|
||||
* UI cards reuse the terminal style: the pending card is the command line,
|
||||
* the completed card carries the output.
|
||||
*/
|
||||
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { Context } from "@deepseek-ai/cordis";
|
||||
import type { ContentBlock } from "@deepseek-ai/dsh-llm";
|
||||
import { defineTool, type ToolResult, type ToolRunContext } from "@deepseek-ai/dsh-tools";
|
||||
import { trySaveScreenshot } from "./image";
|
||||
import { type BskRunner, bskInstallMessage, isCommandNotFound, parseBskJson } from "./runner";
|
||||
import type { SessionRegistry } from "./sessions";
|
||||
|
||||
/** Plugin configuration resolved from the Schemastery schema in index.ts. */
|
||||
export interface PluginConfig {
|
||||
bskPath: string;
|
||||
defaultTimeoutMs: number;
|
||||
maxSessions: number;
|
||||
}
|
||||
|
||||
export interface ToolDeps {
|
||||
ctx: Context;
|
||||
runner: BskRunner;
|
||||
registry: SessionRegistry;
|
||||
config: PluginConfig;
|
||||
}
|
||||
|
||||
/** Device presets supported by `bsk emulate --device`. */
|
||||
const DEVICE_PRESETS = [
|
||||
"iphone-14",
|
||||
"iphone-14-pro-max",
|
||||
"iphone-se",
|
||||
"pixel-7",
|
||||
"galaxy-s23",
|
||||
"ipad-mini",
|
||||
"galaxy-tab-s8",
|
||||
] as const;
|
||||
|
||||
const SESSION_PARAM = {
|
||||
type: "string",
|
||||
description:
|
||||
"bsk session id to act on. Omit to use the current session (the one most recently started or used).",
|
||||
} as const;
|
||||
|
||||
/** Run one bsk command and return its parsed JSON payload, or throw. */
|
||||
async function runBsk(
|
||||
deps: ToolDeps,
|
||||
exec: ToolRunContext,
|
||||
args: string[],
|
||||
label: string,
|
||||
): Promise<unknown> {
|
||||
let result;
|
||||
try {
|
||||
result = await deps.runner.run(args, {
|
||||
signal: exec.signal,
|
||||
timeoutMs: deps.config.defaultTimeoutMs,
|
||||
});
|
||||
} catch (error) {
|
||||
if (isCommandNotFound(error)) {
|
||||
throw new Error(bskInstallMessage(deps.config.bskPath));
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (result.aborted) {
|
||||
const error = new Error("tool call aborted");
|
||||
error.name = "AbortError";
|
||||
throw error;
|
||||
}
|
||||
return parseBskJson(result, label);
|
||||
}
|
||||
|
||||
/** Build the command line shown on the pending terminal card (pure). */
|
||||
function cmdline(deps: ToolDeps, args: string[]): string {
|
||||
return [deps.config.bskPath, ...args].join(" ");
|
||||
}
|
||||
|
||||
/** Shared completed-card presenter: terminal card with the rendered text. */
|
||||
function presentTerminalResult(_args: never, result: ToolResult) {
|
||||
const block = result.content.length === 1 ? result.content[0] : undefined;
|
||||
if (block === undefined || block.type !== "text") return undefined;
|
||||
if (result.isError) return undefined;
|
||||
return { card: "terminal" as const, output: block.text, exitCode: 0 };
|
||||
}
|
||||
|
||||
function abortError(): Error {
|
||||
const error = new Error("tool call aborted");
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
/** Register the full browser tool suite. */
|
||||
export function registerTools(deps: ToolDeps): void {
|
||||
const { ctx, registry } = deps;
|
||||
|
||||
ctx.tools.register(
|
||||
defineTool({
|
||||
name: "browser_session_start",
|
||||
description:
|
||||
"Start a new browser session: opens an Agent Window in the connected browser and returns " +
|
||||
"its session id. The new session becomes the current session for subsequent browser_* calls. " +
|
||||
"Optionally navigate to an initial URL and/or apply a mobile device emulation preset.",
|
||||
parameters: {
|
||||
url: {
|
||||
type: "string",
|
||||
description: "Initial URL to navigate to after the session starts.",
|
||||
},
|
||||
width: {
|
||||
type: "integer",
|
||||
description: "Agent Window outer width in CSS pixels (100..=7680). Requires height.",
|
||||
},
|
||||
height: {
|
||||
type: "integer",
|
||||
description: "Agent Window outer height in CSS pixels (100..=7680). Requires width.",
|
||||
},
|
||||
noFocus: {
|
||||
type: "boolean",
|
||||
description: "Open the Agent Window in the background without stealing focus.",
|
||||
},
|
||||
browser: {
|
||||
type: "string",
|
||||
description:
|
||||
"Target browser instance id (only needed when multiple browsers are connected).",
|
||||
},
|
||||
device: {
|
||||
type: "string",
|
||||
enum: DEVICE_PRESETS,
|
||||
description: "Mobile device emulation preset applied to the tab after start.",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
sessionId: { type: "string", required: true },
|
||||
browserInstanceId: { type: "string", required: true },
|
||||
url: { type: "string" },
|
||||
device: { type: "string" },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text:
|
||||
`started browser session ${value.sessionId}` +
|
||||
(value.url !== undefined ? ` and navigated to ${value.url}` : "") +
|
||||
(value.device !== undefined ? ` (device: ${value.device})` : ""),
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.url !== undefined && args.url.trim().length === 0) {
|
||||
throw new Error("url must be a non-empty string");
|
||||
}
|
||||
if ((args.width === undefined) !== (args.height === undefined)) {
|
||||
throw new Error("width and height must be given together");
|
||||
}
|
||||
const startArgs = ["session", "start"];
|
||||
if (args.width !== undefined && args.height !== undefined) {
|
||||
startArgs.push("--width", String(args.width), "--height", String(args.height));
|
||||
}
|
||||
if (args.noFocus === true) startArgs.push("--no-focus");
|
||||
if (args.browser !== undefined) startArgs.push("--browser", args.browser);
|
||||
const reply = (await runBsk(deps, exec, startArgs, "session start")) as {
|
||||
session_id: string;
|
||||
browser_instance_id: string;
|
||||
};
|
||||
registry.add({
|
||||
sessionId: reply.session_id,
|
||||
browserInstanceId: reply.browser_instance_id,
|
||||
startedAtMs: Date.now(),
|
||||
});
|
||||
try {
|
||||
if (args.device !== undefined) {
|
||||
await runBsk(
|
||||
deps,
|
||||
exec,
|
||||
["emulate", "--session", reply.session_id, "--device", args.device],
|
||||
"emulate",
|
||||
);
|
||||
}
|
||||
if (args.url !== undefined) {
|
||||
await runBsk(
|
||||
deps,
|
||||
exec,
|
||||
["navigate", "--session", reply.session_id, args.url],
|
||||
"navigate",
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
// A half-initialized session must not leak: stop it before surfacing.
|
||||
await deps.runner
|
||||
.run(["session", "stop", reply.session_id], { timeoutMs: 30_000 })
|
||||
.catch(() => {});
|
||||
registry.remove(reply.session_id);
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
sessionId: reply.session_id,
|
||||
browserInstanceId: reply.browser_instance_id,
|
||||
...(args.url !== undefined ? { url: args.url } : {}),
|
||||
...(args.device !== undefined ? { device: args.device } : {}),
|
||||
};
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: cmdline(deps, [
|
||||
"session",
|
||||
"start",
|
||||
...(args.device !== undefined ? ["+ emulate", args.device] : []),
|
||||
...(args.url !== undefined ? ["+ navigate", args.url] : []),
|
||||
]),
|
||||
description: "Start a browser session",
|
||||
}),
|
||||
presentResult: presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
ctx.tools.register(
|
||||
defineTool({
|
||||
name: "browser_session_stop",
|
||||
description:
|
||||
"Stop a browser session and close its Agent Window. Stops the given session, or the " +
|
||||
"current session when `session` is omitted.",
|
||||
parameters: { session: SESSION_PARAM },
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: { stopped: { type: "string", required: true } },
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{ type: "text", text: `stopped browser session ${value.stopped}` },
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_session_stop");
|
||||
await runBsk(deps, exec, ["session", "stop", sessionId], "session stop");
|
||||
registry.remove(sessionId);
|
||||
return { stopped: sessionId };
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: cmdline(deps, ["session", "stop", args.session ?? "(current session)"]),
|
||||
description: "Stop a browser session",
|
||||
}),
|
||||
presentResult: presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
ctx.tools.register(
|
||||
defineTool({
|
||||
name: "browser_session_list",
|
||||
description:
|
||||
"List active browser sessions known to the bsk daemon. The session marked `current` is " +
|
||||
"the one browser_* tools act on when no explicit `session` is passed.",
|
||||
parameters: {},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
sessions: {
|
||||
type: "array",
|
||||
required: true,
|
||||
items: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
sessionId: { type: "string", required: true },
|
||||
browserInstanceId: { type: "string", required: true },
|
||||
current: { type: "boolean", required: true },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text:
|
||||
value.sessions.length === 0
|
||||
? "no active browser sessions"
|
||||
: value.sessions
|
||||
.map(
|
||||
(s) =>
|
||||
`${s.sessionId} (browser ${s.browserInstanceId})${s.current ? " [current]" : ""}`,
|
||||
)
|
||||
.join("\n"),
|
||||
},
|
||||
],
|
||||
},
|
||||
isConcurrencySafe: () => true,
|
||||
async execute(_args, exec) {
|
||||
const reply = (await runBsk(deps, exec, ["session", "list"], "session list")) as {
|
||||
session_id: string;
|
||||
browser_instance_id: string;
|
||||
}[];
|
||||
const current = registry.current();
|
||||
return {
|
||||
sessions: reply.map((entry) => ({
|
||||
sessionId: entry.session_id,
|
||||
browserInstanceId: entry.browser_instance_id,
|
||||
current: entry.session_id === current,
|
||||
})),
|
||||
};
|
||||
},
|
||||
presentCall: () => ({
|
||||
card: "terminal",
|
||||
title: cmdline(deps, ["session", "list"]),
|
||||
description: "List browser sessions",
|
||||
}),
|
||||
presentResult: presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
ctx.tools.register(
|
||||
defineTool({
|
||||
name: "browser_navigate",
|
||||
description:
|
||||
"Navigate the session's active tab to a URL and wait for a page lifecycle phase " +
|
||||
"(default: load). Returns the final URL after redirects.",
|
||||
parameters: {
|
||||
url: { type: "string", required: true, description: "Destination URL." },
|
||||
session: SESSION_PARAM,
|
||||
waitUntil: {
|
||||
type: "string",
|
||||
enum: ["load", "domcontentloaded", "networkidle", "commit"],
|
||||
description: "Lifecycle phase to wait for (default: load).",
|
||||
},
|
||||
timeoutMs: {
|
||||
type: "integer",
|
||||
description: "Navigation wait timeout in milliseconds (default 30000).",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
url: { type: "string", required: true },
|
||||
finalUrl: { type: "string" },
|
||||
reached: { type: "string", required: true },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text:
|
||||
`[session ${value.session}] navigated to ${value.finalUrl ?? value.url} (reached: ${value.reached})` +
|
||||
(value.reached === "timeout"
|
||||
? " — the wait timed out before the requested phase"
|
||||
: ""),
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.url.trim().length === 0) throw new Error("url must be a non-empty string");
|
||||
const sessionId = registry.resolve(args.session, "browser_navigate");
|
||||
const cmdArgs = ["navigate", "--session", sessionId];
|
||||
if (args.waitUntil !== undefined) cmdArgs.push("--wait-until", args.waitUntil);
|
||||
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
|
||||
cmdArgs.push(args.url);
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "navigate")) as {
|
||||
tab_id: number;
|
||||
url: string;
|
||||
final_url?: string;
|
||||
reached: string;
|
||||
};
|
||||
return {
|
||||
session: sessionId,
|
||||
tabId: reply.tab_id,
|
||||
url: reply.url,
|
||||
...(reply.final_url !== undefined ? { finalUrl: reply.final_url } : {}),
|
||||
reached: reply.reached,
|
||||
};
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: cmdline(deps, ["navigate", "--session", args.session ?? "(current)", args.url]),
|
||||
description: "Navigate to a URL",
|
||||
}),
|
||||
presentResult: presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
const registerObservationTool = (kind: "snapshot" | "observe") => {
|
||||
const isSnapshot = kind === "snapshot";
|
||||
const name = isSnapshot ? "browser_snapshot" : "browser_observe";
|
||||
const description = isSnapshot
|
||||
? "Capture an indented aria-tree snapshot of the session's active tab. Interactive elements " +
|
||||
"carry @eN refs for browser_click / browser_fill. Prefer browser_observe for a richer semantic view."
|
||||
: "Produce a semantic VOM observation of the session's active tab (roles, states, perception " +
|
||||
"probes) with @eN refs for browser_click / browser_fill. Read-only: never submits input.";
|
||||
ctx.tools.register(
|
||||
defineTool({
|
||||
name,
|
||||
description,
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
maxDepth: { type: "integer", description: "Cap on tree depth before truncating." },
|
||||
maxTokens: {
|
||||
type: "integer",
|
||||
description: "Soft cap on rendered tokens (~4 chars/token).",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
text: { type: "string", required: true },
|
||||
refCount: { type: "integer", required: true },
|
||||
truncated: { type: "boolean", required: true },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text:
|
||||
value.text.length > 0
|
||||
? value.text + (value.truncated ? "\n(truncated — re-run with looser caps)" : "")
|
||||
: "(empty observation — page may still be loading)",
|
||||
},
|
||||
],
|
||||
},
|
||||
isConcurrencySafe: () => true,
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, name);
|
||||
const cmdArgs = [kind, "--session", sessionId];
|
||||
if (args.maxDepth !== undefined) cmdArgs.push("--max-depth", String(args.maxDepth));
|
||||
if (args.maxTokens !== undefined) cmdArgs.push("--max-tokens", String(args.maxTokens));
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, kind)) as {
|
||||
text: string;
|
||||
ref_count: number;
|
||||
tab_id: number;
|
||||
truncated?: boolean;
|
||||
};
|
||||
return {
|
||||
session: sessionId,
|
||||
tabId: reply.tab_id,
|
||||
text: reply.text,
|
||||
refCount: reply.ref_count,
|
||||
truncated: reply.truncated ?? false,
|
||||
};
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: cmdline(deps, [kind, "--session", args.session ?? "(current)"]),
|
||||
description: isSnapshot ? "Capture an aria snapshot" : "Observe the page semantically",
|
||||
}),
|
||||
presentResult: presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
};
|
||||
registerObservationTool("snapshot");
|
||||
registerObservationTool("observe");
|
||||
|
||||
ctx.tools.register(
|
||||
defineTool({
|
||||
name: "browser_click",
|
||||
description:
|
||||
"Click an element in the session's active tab. Target is a snapshot ref (@e3) from the " +
|
||||
"last browser_snapshot/browser_observe, or a CSS selector.",
|
||||
parameters: {
|
||||
target: {
|
||||
type: "string",
|
||||
required: true,
|
||||
description: "Snapshot ref (@e3 / e3) or CSS selector of the element to click.",
|
||||
},
|
||||
session: SESSION_PARAM,
|
||||
button: {
|
||||
type: "string",
|
||||
enum: ["left", "middle", "right"],
|
||||
description: "Mouse button (default: left).",
|
||||
},
|
||||
clickCount: {
|
||||
type: "integer",
|
||||
description: "Number of consecutive presses (double-click = 2).",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
x: { type: "number", required: true },
|
||||
y: { type: "number", required: true },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text: `[session ${value.session}] clicked at (${value.x}, ${value.y}) on tab ${value.tabId}`,
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.target.trim().length === 0) throw new Error("target must be a non-empty string");
|
||||
const sessionId = registry.resolve(args.session, "browser_click");
|
||||
const cmdArgs = ["click", "--session", sessionId];
|
||||
if (args.button !== undefined) cmdArgs.push("--button", args.button);
|
||||
if (args.clickCount !== undefined) cmdArgs.push("--click-count", String(args.clickCount));
|
||||
cmdArgs.push(args.target);
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "click")) as {
|
||||
tab_id: number;
|
||||
x: number;
|
||||
y: number;
|
||||
};
|
||||
return { session: sessionId, tabId: reply.tab_id, x: reply.x, y: reply.y };
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: cmdline(deps, ["click", args.target, "--session", args.session ?? "(current)"]),
|
||||
description: "Click an element",
|
||||
}),
|
||||
presentResult: presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
ctx.tools.register(
|
||||
defineTool({
|
||||
name: "browser_fill",
|
||||
description:
|
||||
"Fill an input / textarea / contenteditable element, clearing it first by default. " +
|
||||
"Target is a snapshot ref (@e3) or a CSS selector.",
|
||||
parameters: {
|
||||
target: {
|
||||
type: "string",
|
||||
required: true,
|
||||
description: "Snapshot ref (@e3 / e3) or CSS selector of the field.",
|
||||
},
|
||||
value: { type: "string", required: true, description: "Text to type into the element." },
|
||||
session: SESSION_PARAM,
|
||||
noClear: {
|
||||
type: "boolean",
|
||||
description: "Skip the default wipe-the-field-first pass (append instead).",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
valueLength: { type: "integer", required: true },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text: `[session ${value.session}] filled field on tab ${value.tabId} (${value.valueLength} chars)`,
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.target.trim().length === 0) throw new Error("target must be a non-empty string");
|
||||
const sessionId = registry.resolve(args.session, "browser_fill");
|
||||
const cmdArgs = ["fill", "--session", sessionId, "--value", args.value];
|
||||
if (args.noClear === true) cmdArgs.push("--no-clear");
|
||||
cmdArgs.push(args.target);
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "fill")) as {
|
||||
tab_id: number;
|
||||
value_length: number;
|
||||
};
|
||||
return { session: sessionId, tabId: reply.tab_id, valueLength: reply.value_length };
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: cmdline(deps, ["fill", args.target, "--session", args.session ?? "(current)"]),
|
||||
description: `Fill a field with ${args.value.length} chars`,
|
||||
}),
|
||||
presentResult: presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
ctx.tools.register(
|
||||
defineTool({
|
||||
name: "browser_press",
|
||||
description:
|
||||
"Dispatch a keyboard key or combo (Enter, Escape, ArrowDown, Ctrl+A, …) to the session's " +
|
||||
"active tab, optionally focusing a target element first.",
|
||||
parameters: {
|
||||
key: {
|
||||
type: "string",
|
||||
required: true,
|
||||
description: "Key spec: a CDP key name (Enter, a, ArrowLeft) or a combo (Ctrl+A).",
|
||||
},
|
||||
session: SESSION_PARAM,
|
||||
target: {
|
||||
type: "string",
|
||||
description: "Snapshot ref (@e3) or CSS selector to focus before pressing.",
|
||||
},
|
||||
holdMs: {
|
||||
type: "integer",
|
||||
description: "Hold the key down for N milliseconds between keyDown and keyUp.",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
key: { type: "string", required: true },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text: `[session ${value.session}] pressed ${value.key} on tab ${value.tabId}`,
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.key.trim().length === 0) throw new Error("key must be a non-empty string");
|
||||
const sessionId = registry.resolve(args.session, "browser_press");
|
||||
const cmdArgs = ["press", "--session", sessionId];
|
||||
if (args.target !== undefined) {
|
||||
if (args.target.trim().length === 0) throw new Error("target must be a non-empty string");
|
||||
// bsk distinguishes ref vs selector via flags; positional detection is CLI-side for
|
||||
// click/fill only, so pass the explicit flag that matches the target's shape.
|
||||
if (/^@?e\d+$/.test(args.target)) {
|
||||
cmdArgs.push("--ref", args.target);
|
||||
} else {
|
||||
cmdArgs.push("--selector", args.target);
|
||||
}
|
||||
}
|
||||
if (args.holdMs !== undefined) cmdArgs.push("--hold-ms", String(args.holdMs));
|
||||
cmdArgs.push(args.key);
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "press")) as {
|
||||
tab_id: number;
|
||||
key: string;
|
||||
};
|
||||
return { session: sessionId, tabId: reply.tab_id, key: reply.key };
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: cmdline(deps, ["press", args.key, "--session", args.session ?? "(current)"]),
|
||||
description: "Press a key",
|
||||
}),
|
||||
presentResult: presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
ctx.tools.register(
|
||||
defineTool({
|
||||
name: "browser_screenshot",
|
||||
description:
|
||||
"Capture a PNG screenshot of the session's active tab, or crop to a snapshot ref element. " +
|
||||
"Returns the image itself when the deployment supports image input, otherwise a file path.",
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
ref: {
|
||||
type: "string",
|
||||
description: "Snapshot ref (@e3) from the last snapshot/observe; crops to that element.",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
path: { type: "string", required: true },
|
||||
width: { type: "integer", required: true },
|
||||
height: { type: "integer", required: true },
|
||||
byteSize: { type: "integer", required: true },
|
||||
image: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
attachmentId: { type: "string", required: true },
|
||||
mediaType: { type: "string", required: true, const: "image/png" },
|
||||
bytes: { type: "integer", required: true },
|
||||
width: { type: "integer", required: true },
|
||||
height: { type: "integer", required: true },
|
||||
name: { type: "string" },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
render: (_args, value) => {
|
||||
const text =
|
||||
value.image !== undefined
|
||||
? `[session ${value.session}] screenshot of tab ${value.tabId} (${value.width}x${value.height}px)`
|
||||
: `[session ${value.session}] screenshot saved to ${value.path} (${value.width}x${value.height}px, ${value.byteSize} bytes) — this deployment cannot inline images; read the file to view it`;
|
||||
const blocks: ContentBlock[] = [{ type: "text", text }];
|
||||
if (value.image !== undefined) {
|
||||
blocks.push({
|
||||
type: "image",
|
||||
// Structurally an ImageAttachmentRef; branded without a runtime import.
|
||||
attachment: {
|
||||
attachmentId: value.image.attachmentId,
|
||||
mediaType: value.image.mediaType,
|
||||
bytes: value.image.bytes,
|
||||
width: value.image.width,
|
||||
height: value.image.height,
|
||||
...(value.image.name !== undefined ? { name: value.image.name } : {}),
|
||||
} as never,
|
||||
});
|
||||
}
|
||||
return blocks;
|
||||
},
|
||||
},
|
||||
isConcurrencySafe: () => true,
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_screenshot");
|
||||
const outPath = join(
|
||||
tmpdir(),
|
||||
`bsk-screenshot-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.png`,
|
||||
);
|
||||
const cmdArgs = ["screenshot", "--session", sessionId, "--out", outPath];
|
||||
if (args.ref !== undefined) cmdArgs.push("--ref", args.ref);
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "screenshot")) as {
|
||||
tab_id: number;
|
||||
width: number;
|
||||
height: number;
|
||||
path: string;
|
||||
byte_size: number;
|
||||
};
|
||||
const data = await readFile(reply.path);
|
||||
if (exec.signal.aborted) throw abortError();
|
||||
const ref = await trySaveScreenshot(deps.ctx, exec, data, `screenshot-${sessionId}.png`);
|
||||
return {
|
||||
session: sessionId,
|
||||
tabId: reply.tab_id,
|
||||
path: reply.path,
|
||||
width: reply.width,
|
||||
height: reply.height,
|
||||
byteSize: reply.byte_size,
|
||||
...(ref !== undefined
|
||||
? {
|
||||
image: {
|
||||
attachmentId: String(ref.attachmentId),
|
||||
mediaType: "image/png" as const,
|
||||
bytes: ref.bytes,
|
||||
width: ref.width,
|
||||
height: ref.height,
|
||||
...(ref.name !== undefined ? { name: ref.name } : {}),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: cmdline(deps, [
|
||||
"screenshot",
|
||||
"--session",
|
||||
args.session ?? "(current)",
|
||||
...(args.ref !== undefined ? ["--ref", args.ref] : []),
|
||||
]),
|
||||
description: "Capture a screenshot",
|
||||
}),
|
||||
presentResult: presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
ctx.tools.register(
|
||||
defineTool({
|
||||
name: "browser_emulate",
|
||||
description:
|
||||
"Apply (or clear) mobile device emulation on the session's active tab: viewport metrics, " +
|
||||
"user agent, and touch. Overrides are per-tab and not inherited by new tabs.",
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
device: {
|
||||
type: "string",
|
||||
enum: DEVICE_PRESETS,
|
||||
description: "Built-in device preset.",
|
||||
},
|
||||
width: { type: "integer", description: "Viewport width in CSS pixels (requires height)." },
|
||||
height: { type: "integer", description: "Viewport height in CSS pixels (requires width)." },
|
||||
mobile: { type: "boolean", description: "Emulate a mobile viewport." },
|
||||
off: { type: "boolean", description: "Clear every emulation override on the tab." },
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
cleared: { type: "boolean", required: true },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text: value.cleared
|
||||
? `[session ${value.session}] cleared emulation on tab ${value.tabId}`
|
||||
: `[session ${value.session}] applied emulation on tab ${value.tabId}`,
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_emulate");
|
||||
if (args.off === true) {
|
||||
if (args.device !== undefined || args.width !== undefined || args.height !== undefined) {
|
||||
throw new Error("off is mutually exclusive with device/width/height");
|
||||
}
|
||||
} else if (args.device === undefined && args.width === undefined) {
|
||||
throw new Error("nothing to apply: pass device, width+height, or off");
|
||||
}
|
||||
if ((args.width === undefined) !== (args.height === undefined)) {
|
||||
throw new Error("width and height must be given together");
|
||||
}
|
||||
const cmdArgs = ["emulate", "--session", sessionId];
|
||||
if (args.off === true) {
|
||||
cmdArgs.push("--off");
|
||||
} else {
|
||||
if (args.device !== undefined) cmdArgs.push("--device", args.device);
|
||||
if (args.width !== undefined && args.height !== undefined) {
|
||||
cmdArgs.push("--width", String(args.width), "--height", String(args.height));
|
||||
}
|
||||
if (args.mobile === true) cmdArgs.push("--mobile");
|
||||
}
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "emulate")) as {
|
||||
tab_id: number;
|
||||
cleared: boolean;
|
||||
};
|
||||
return { session: sessionId, tabId: reply.tab_id, cleared: reply.cleared };
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: cmdline(deps, [
|
||||
"emulate",
|
||||
"--session",
|
||||
args.session ?? "(current)",
|
||||
...(args.off === true ? ["--off"] : []),
|
||||
...(args.device !== undefined ? ["--device", args.device] : []),
|
||||
]),
|
||||
description: "Emulate a device environment",
|
||||
}),
|
||||
presentResult: presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user