mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
388c1283bc | ||
|
|
b449be1544 |
@@ -7,10 +7,6 @@ dist/
|
||||
.output/
|
||||
.wxt/
|
||||
.web-extension-id
|
||||
.pnpm-store/
|
||||
|
||||
# Git worktrees
|
||||
.worktrees/
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { i18n } from "@browser-skill/i18n";
|
||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import { ConnectionController } from "@/lib/connection-controller";
|
||||
import { DiagnosticLogger } from "@/lib/diagnostics";
|
||||
import { startHeartbeat } from "@/lib/heartbeat";
|
||||
import {
|
||||
getConnectionEnabled,
|
||||
@@ -46,9 +47,20 @@ import type { Transport } from "@/transport/transport";
|
||||
import { WSTransport } from "@/transport/ws-transport";
|
||||
|
||||
export default defineBackground(() => {
|
||||
const controller = new ConnectionController();
|
||||
const transport = new WSTransport({ url: __BSK_DAEMON_WS_URL__ });
|
||||
const sessions = new SessionManager();
|
||||
const diagnostics = new DiagnosticLogger();
|
||||
diagnostics.record("extension.worker.started", {
|
||||
diagnostic_build: "lumi-session-loss-v2",
|
||||
extension_version: __BSK_EXT_VERSION__,
|
||||
extension_runtime_id: chrome.runtime.id,
|
||||
daemon_ws_url: __BSK_DAEMON_WS_URL__,
|
||||
});
|
||||
const controller = new ConnectionController(diagnostics.sink);
|
||||
const transport = new WSTransport({
|
||||
url: __BSK_DAEMON_WS_URL__,
|
||||
diagnostics: diagnostics.sink,
|
||||
});
|
||||
diagnostics.bindTransport((frame) => transport.send(frame));
|
||||
const sessions = new SessionManager({ diagnostics: diagnostics.sink });
|
||||
const cdp = new ChromiumCdp();
|
||||
const sessionsLive = attachSessionsLiveFlag({ manager: sessions });
|
||||
let overlayGeneration = 0;
|
||||
@@ -109,6 +121,9 @@ export default defineBackground(() => {
|
||||
}
|
||||
|
||||
function onOverlaySessionStateChanged(): void {
|
||||
diagnostics.record("background.sessions.changed", {
|
||||
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
|
||||
});
|
||||
void sessionsLive.syncFromManager();
|
||||
const liveSessionIds = new Set(sessions.list().map((ctx) => ctx.sessionId));
|
||||
for (const sessionId of controlModes.keys()) {
|
||||
@@ -146,6 +161,7 @@ export default defineBackground(() => {
|
||||
onSessionsChanged: () => {
|
||||
void sessionsLive.syncFromManager();
|
||||
},
|
||||
diagnostics: diagnostics.sink,
|
||||
});
|
||||
const recordDeps = {
|
||||
tabsApi: chrome.tabs,
|
||||
@@ -193,6 +209,7 @@ export default defineBackground(() => {
|
||||
title: i18n.t("helpRequest.notificationTitle", { ns: "extension" }),
|
||||
body: "",
|
||||
}),
|
||||
diagnostics: diagnostics.sink,
|
||||
});
|
||||
dispatcher.start();
|
||||
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
|
||||
@@ -225,6 +242,7 @@ export default defineBackground(() => {
|
||||
transport,
|
||||
cdp,
|
||||
onSessionsChanged: onOverlaySessionStateChanged,
|
||||
diagnostics: diagnostics.sink,
|
||||
});
|
||||
|
||||
// MV3 service worker keepalive + reconnect supervisor (review M4/M5
|
||||
@@ -236,6 +254,7 @@ export default defineBackground(() => {
|
||||
startKeepalive({
|
||||
transport,
|
||||
shouldConnect: () => controller.isConnectionEnabled,
|
||||
diagnostics: diagnostics.sink,
|
||||
});
|
||||
|
||||
// Application-level heartbeat (Chrome 116+): while the post-handshake
|
||||
@@ -249,6 +268,19 @@ export default defineBackground(() => {
|
||||
controller.subscribe((snap) =>
|
||||
cb(snap.state === "connected" || snap.state === "version_skew"),
|
||||
),
|
||||
diagnostics: diagnostics.sink,
|
||||
});
|
||||
|
||||
controller.subscribe((snapshot) => {
|
||||
const ready = snapshot.state === "connected" || snapshot.state === "version_skew";
|
||||
diagnostics.setTransportReady(ready);
|
||||
diagnostics.record("background.controller.snapshot", {
|
||||
state: snapshot.state,
|
||||
connection_enabled: snapshot.connectionEnabled,
|
||||
instance_id: snapshot.instanceId,
|
||||
last_error: snapshot.lastError,
|
||||
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
|
||||
});
|
||||
});
|
||||
|
||||
// Wake-driven reconnect (best effort). An MV3 service worker is killed
|
||||
@@ -257,7 +289,12 @@ export default defineBackground(() => {
|
||||
// the worker and let us reconnect immediately instead of waiting for
|
||||
// the next 30s alarm tick. They only help when the daemon is actually
|
||||
// running; a cold daemon is (re)spawned by the next `bsk` command.
|
||||
const reconnectIfNeeded = () => {
|
||||
const reconnectIfNeeded = (source: string) => {
|
||||
diagnostics.record("background.wake_reconnect.requested", {
|
||||
source,
|
||||
controller_enabled: controller.isConnectionEnabled,
|
||||
transport_state: transport.state,
|
||||
});
|
||||
if (!controller.isConnectionEnabled) return;
|
||||
if (transport.state === "connected") return;
|
||||
void transport.connect().catch((err) => {
|
||||
@@ -265,13 +302,14 @@ export default defineBackground(() => {
|
||||
});
|
||||
};
|
||||
if (typeof chrome.runtime?.onStartup?.addListener === "function") {
|
||||
chrome.runtime.onStartup.addListener(reconnectIfNeeded);
|
||||
chrome.runtime.onStartup.addListener(() => reconnectIfNeeded("runtime.onStartup"));
|
||||
}
|
||||
if (typeof chrome.idle?.onStateChanged?.addListener === "function") {
|
||||
chrome.idle.onStateChanged.addListener((state) => {
|
||||
diagnostics.record("chrome.idle.state_changed", { state });
|
||||
// "active" fires when the user returns from idle/locked — the most
|
||||
// reliable "machine just woke" signal we get.
|
||||
if (state === "active") reconnectIfNeeded();
|
||||
if (state === "active") reconnectIfNeeded("idle.active");
|
||||
});
|
||||
// Treat >60s of no input as idle so the active transition is timely.
|
||||
chrome.idle.setDetectionInterval?.(60);
|
||||
@@ -279,14 +317,21 @@ export default defineBackground(() => {
|
||||
|
||||
void (async () => {
|
||||
const connectionEnabled = await getConnectionEnabled();
|
||||
diagnostics.record("background.controller.attach_started", { connectionEnabled });
|
||||
await controller.attach(transport, detectBrowserMeta(), connectionEnabled, {
|
||||
beforeDisconnect: async () => {
|
||||
diagnostics.record("background.before_disconnect_cleanup", {
|
||||
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
|
||||
});
|
||||
const report = await cleanupAfterDisconnect();
|
||||
if (report.failures.length > 0) {
|
||||
console.warn("[browser-skill] session cleanup before disconnect was incomplete", report);
|
||||
}
|
||||
},
|
||||
onDisconnected: async () => {
|
||||
diagnostics.record("background.after_disconnect_cleanup", {
|
||||
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
|
||||
});
|
||||
const report = await cleanupAfterDisconnect();
|
||||
if (report.failures.length > 0) {
|
||||
console.warn("[browser-skill] session cleanup after disconnect was incomplete", report);
|
||||
@@ -294,6 +339,7 @@ export default defineBackground(() => {
|
||||
},
|
||||
});
|
||||
})().catch((err) => {
|
||||
diagnostics.record("background.controller.attach_failed", { error: err });
|
||||
console.error("[browser-skill] controller failed to attach", err);
|
||||
});
|
||||
|
||||
@@ -374,6 +420,15 @@ export default defineBackground(() => {
|
||||
dbg.__bhDispatcher = dispatcher;
|
||||
}
|
||||
|
||||
if (typeof chrome.runtime?.onSuspend?.addListener === "function") {
|
||||
chrome.runtime.onSuspend.addListener(() => {
|
||||
diagnostics.record("extension.worker.suspending", {
|
||||
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
|
||||
transport_state: transport.state,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
console.info("[browser-skill] background worker initialised");
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { DiagnosticLogger } from "../diagnostics";
|
||||
|
||||
class MemoryStorage {
|
||||
value: Record<string, unknown> = {};
|
||||
|
||||
async get(key: string): Promise<Record<string, unknown>> {
|
||||
return { [key]: this.value[key] };
|
||||
}
|
||||
|
||||
async set(items: Record<string, unknown>): Promise<void> {
|
||||
this.value = { ...this.value, ...items };
|
||||
}
|
||||
}
|
||||
|
||||
describe("DiagnosticLogger", () => {
|
||||
it("persists evidence before transport readiness and flushes it after reconnect", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const send = vi.fn();
|
||||
const logger = new DiagnosticLogger(storage);
|
||||
logger.bindTransport(send);
|
||||
|
||||
logger.record("transport.socket.closed", { code: 1006 });
|
||||
await logger.flush();
|
||||
expect(send).not.toHaveBeenCalled();
|
||||
|
||||
logger.setTransportReady(true);
|
||||
await logger.flush();
|
||||
|
||||
expect(send).toHaveBeenCalledOnce();
|
||||
expect(send.mock.calls[0]?.[0]).toMatchObject({
|
||||
event: "browser.diagnostic",
|
||||
payload: {
|
||||
schema_version: 1,
|
||||
entries: [
|
||||
{
|
||||
event: "transport.socket.closed",
|
||||
fields: { code: 1006 },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
expect(storage.value.bsk_diagnostic_events_v1).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps persisted evidence when the transport rejects the flush", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const logger = new DiagnosticLogger(storage);
|
||||
logger.bindTransport(() => {
|
||||
throw new Error("socket closed");
|
||||
});
|
||||
logger.record("session.stop.requested", { session_id: "abcd" });
|
||||
|
||||
logger.setTransportReady(true);
|
||||
await logger.flush();
|
||||
|
||||
expect(storage.value.bsk_diagnostic_events_v1).toMatchObject([
|
||||
{ event: "session.stop.requested", fields: { session_id: "abcd" } },
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not resend an acknowledged event when send records a new event", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const batches: Array<{ payload?: { entries?: Array<{ event: string }> } }> = [];
|
||||
const logger = new DiagnosticLogger(storage);
|
||||
logger.record("dispatcher.rpc.received", { rpc_id: "one" });
|
||||
await logger.flush();
|
||||
logger.bindTransport((frame) => {
|
||||
batches.push(frame as { payload?: { entries?: Array<{ event: string }> } });
|
||||
if (batches.length === 1) {
|
||||
logger.record("dispatcher.rpc.response_sent", { rpc_id: "one" });
|
||||
}
|
||||
});
|
||||
|
||||
logger.setTransportReady(true);
|
||||
await logger.flush();
|
||||
await logger.flush();
|
||||
|
||||
expect(batches.map((batch) => batch.payload?.entries?.map((entry) => entry.event))).toEqual([
|
||||
["dispatcher.rpc.received"],
|
||||
["dispatcher.rpc.response_sent"],
|
||||
]);
|
||||
expect(storage.value.bsk_diagnostic_events_v1).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
} from "../transport/handshake";
|
||||
import type { Transport } from "../transport/transport";
|
||||
import type { ConnectionState, HandshakeResult } from "../transport/types";
|
||||
import type { DiagnosticSink } from "./diagnostics";
|
||||
import { getLabel, getOrCreateInstanceId } from "./instance-id";
|
||||
import { compareProtocol, parseProtocolMajor } from "./semver";
|
||||
|
||||
@@ -51,6 +52,11 @@ export class ConnectionController {
|
||||
private handshakeAbort: AbortController | null = null;
|
||||
private handshakeRetryTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private suppressDisconnectRecovery = false;
|
||||
private readonly diagnostics?: DiagnosticSink;
|
||||
|
||||
constructor(diagnostics?: DiagnosticSink) {
|
||||
this.diagnostics = diagnostics;
|
||||
}
|
||||
|
||||
get isConnectionEnabled(): boolean {
|
||||
return this.connectionEnabled;
|
||||
@@ -89,8 +95,20 @@ export class ConnectionController {
|
||||
this.lifecycleHooks = lifecycleHooks;
|
||||
this.instanceId = await getOrCreateInstanceId();
|
||||
this.label = await getLabel();
|
||||
this.diagnostics?.("controller.attach", {
|
||||
instance_id: this.instanceId,
|
||||
connection_enabled: connectionEnabled,
|
||||
transport_state: transport.state,
|
||||
});
|
||||
|
||||
transport.onConnectionStateChange((s) => {
|
||||
this.diagnostics?.("controller.transport_state", {
|
||||
transport_state: s,
|
||||
controller_state: this.currentState,
|
||||
connection_generation: this.connectionGeneration,
|
||||
suppress_disconnect_recovery: this.suppressDisconnectRecovery,
|
||||
recovery_inflight: this.disconnectRecovery !== null,
|
||||
});
|
||||
if (s === "disconnected") {
|
||||
// An in-flight handshake belongs to the dead connection — cancel it so
|
||||
// a late resolution can never clobber the next attempt.
|
||||
@@ -103,6 +121,9 @@ export class ConnectionController {
|
||||
// handshake bounce); the caller owns the reconnect policy there.
|
||||
this.suppressDisconnectRecovery = false;
|
||||
} else {
|
||||
this.diagnostics?.("controller.disconnect_recovery.requested", {
|
||||
connection_generation: this.connectionGeneration,
|
||||
});
|
||||
void this.recoverFromDisconnect();
|
||||
}
|
||||
}
|
||||
@@ -161,6 +182,12 @@ export class ConnectionController {
|
||||
const generation = ++this.connectionGeneration;
|
||||
const abort = new AbortController();
|
||||
this.handshakeAbort = abort;
|
||||
this.diagnostics?.("controller.handshake.started", {
|
||||
connection_generation: generation,
|
||||
browser_name: browser.name,
|
||||
browser_version: browser.version,
|
||||
instance_id: this.instanceId,
|
||||
});
|
||||
void this.runHandshake(browser, generation, abort.signal);
|
||||
}
|
||||
|
||||
@@ -197,11 +224,21 @@ export class ConnectionController {
|
||||
}
|
||||
this.clearHandshakeRetry();
|
||||
this.lastError = null;
|
||||
this.diagnostics?.("controller.handshake.completed", {
|
||||
connection_generation: generation,
|
||||
verdict: verdict.kind,
|
||||
daemon_version: outcome.result.version,
|
||||
daemon_protocol: outcome.result.protocol_version,
|
||||
});
|
||||
this.setState(verdict.kind);
|
||||
} catch (err) {
|
||||
if (generation !== this.connectionGeneration || signal.aborted || isAbortError(err)) return;
|
||||
this.handshake = null;
|
||||
this.lastError = err instanceof Error ? err.message : String(err);
|
||||
this.diagnostics?.("controller.handshake.failed", {
|
||||
connection_generation: generation,
|
||||
error: err,
|
||||
});
|
||||
// Bounce the half-open socket deliberately; the retry timer below owns
|
||||
// the reconnect, so skip unexpected-loss recovery for this disconnect.
|
||||
this.suppressDisconnectRecovery = true;
|
||||
@@ -228,23 +265,57 @@ export class ConnectionController {
|
||||
}
|
||||
|
||||
private recoverFromDisconnect(): Promise<void> {
|
||||
if (this.disconnectRecovery) return this.disconnectRecovery;
|
||||
if (this.disconnectRecovery) {
|
||||
this.diagnostics?.("controller.disconnect_recovery.coalesced", {
|
||||
connection_generation: this.connectionGeneration,
|
||||
});
|
||||
return this.disconnectRecovery;
|
||||
}
|
||||
const recovery = (async () => {
|
||||
const recoveryGeneration = this.connectionGeneration;
|
||||
this.diagnostics?.("controller.disconnect_recovery.started", {
|
||||
recovery_generation: recoveryGeneration,
|
||||
});
|
||||
// WSTransport schedules its reconnect timer immediately after notifying
|
||||
// state listeners. Yield once, then disconnect explicitly so that timer
|
||||
// is cancelled before local session teardown begins.
|
||||
await Promise.resolve();
|
||||
if (!this.connectionEnabled || !this.transport) return;
|
||||
if (!this.connectionEnabled || !this.transport) {
|
||||
this.diagnostics?.("controller.disconnect_recovery.skipped", {
|
||||
recovery_generation: recoveryGeneration,
|
||||
reason: "disabled_or_missing_transport",
|
||||
});
|
||||
return;
|
||||
}
|
||||
// Another path already reconnected while we yielded — nothing to do.
|
||||
if (this.currentState !== "disconnected") return;
|
||||
if (this.currentState !== "disconnected") {
|
||||
this.diagnostics?.("controller.disconnect_recovery.skipped", {
|
||||
recovery_generation: recoveryGeneration,
|
||||
reason: "state_already_changed",
|
||||
current_state: this.currentState,
|
||||
});
|
||||
return;
|
||||
}
|
||||
await this.transport.disconnect().catch(() => {});
|
||||
try {
|
||||
this.diagnostics?.("controller.disconnect_cleanup.started", {
|
||||
recovery_generation: recoveryGeneration,
|
||||
current_generation: this.connectionGeneration,
|
||||
});
|
||||
await this.lifecycleHooks.onDisconnected?.();
|
||||
this.diagnostics?.("controller.disconnect_cleanup.completed", {
|
||||
recovery_generation: recoveryGeneration,
|
||||
current_generation: this.connectionGeneration,
|
||||
});
|
||||
} catch (err) {
|
||||
console.warn("[browser-skill] session cleanup after disconnect failed", err);
|
||||
}
|
||||
if (!this.connectionEnabled) return;
|
||||
try {
|
||||
this.diagnostics?.("controller.disconnect_reconnect.started", {
|
||||
recovery_generation: recoveryGeneration,
|
||||
current_generation: this.connectionGeneration,
|
||||
});
|
||||
await this.transport.connect();
|
||||
} catch (err) {
|
||||
this.lastError = err instanceof Error ? err.message : String(err);
|
||||
@@ -252,22 +323,39 @@ export class ConnectionController {
|
||||
}
|
||||
})();
|
||||
this.disconnectRecovery = recovery.finally(() => {
|
||||
this.diagnostics?.("controller.disconnect_recovery.completed", {
|
||||
connection_generation: this.connectionGeneration,
|
||||
current_state: this.currentState,
|
||||
});
|
||||
this.disconnectRecovery = null;
|
||||
});
|
||||
return this.disconnectRecovery;
|
||||
}
|
||||
|
||||
private cancelHandshake(): void {
|
||||
const previousGeneration = this.connectionGeneration;
|
||||
this.connectionGeneration += 1;
|
||||
this.diagnostics?.("controller.handshake.cancelled", {
|
||||
previous_generation: previousGeneration,
|
||||
next_generation: this.connectionGeneration,
|
||||
had_abort_controller: this.handshakeAbort !== null,
|
||||
});
|
||||
this.handshakeAbort?.abort();
|
||||
this.handshakeAbort = null;
|
||||
}
|
||||
|
||||
private scheduleHandshakeRetry(browser: { name: string; version: string }): void {
|
||||
if (this.handshakeRetryTimer || !this.connectionEnabled) return;
|
||||
this.diagnostics?.("controller.handshake_retry.scheduled", {
|
||||
delay_ms: HANDSHAKE_RETRY_DELAY_MS,
|
||||
connection_generation: this.connectionGeneration,
|
||||
});
|
||||
this.handshakeRetryTimer = setTimeout(() => {
|
||||
this.handshakeRetryTimer = null;
|
||||
if (!this.connectionEnabled || !this.transport) return;
|
||||
this.diagnostics?.("controller.handshake_retry.timer_fired", {
|
||||
connection_generation: this.connectionGeneration,
|
||||
});
|
||||
void this.transport.connect().catch((err) => {
|
||||
this.lastError = err instanceof Error ? err.message : String(err);
|
||||
this.setState("disconnected");
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
import type { ProtocolFrame } from "@/transport/types";
|
||||
|
||||
const STORAGE_KEY = "bsk_diagnostic_events_v1";
|
||||
const MAX_ENTRIES = 500;
|
||||
|
||||
export type DiagnosticFields = Record<string, unknown>;
|
||||
export type DiagnosticSink = (event: string, fields?: DiagnosticFields) => void;
|
||||
|
||||
export interface DiagnosticEntry {
|
||||
id: string;
|
||||
sequence: number;
|
||||
timestamp: string;
|
||||
monotonic_ms: number;
|
||||
worker_boot_id: string;
|
||||
event: string;
|
||||
fields: DiagnosticFields;
|
||||
}
|
||||
|
||||
interface DiagnosticStorage {
|
||||
get(key: string): Promise<Record<string, unknown>>;
|
||||
set(items: Record<string, unknown>): Promise<void>;
|
||||
}
|
||||
|
||||
function defaultStorage(): DiagnosticStorage | null {
|
||||
if (typeof chrome === "undefined" || !chrome.storage?.local) return null;
|
||||
return {
|
||||
get: (key) => chrome.storage.local.get(key),
|
||||
set: (items) => chrome.storage.local.set(items),
|
||||
};
|
||||
}
|
||||
|
||||
function makeBootId(): string {
|
||||
try {
|
||||
return crypto.randomUUID();
|
||||
} catch {
|
||||
return `worker-${Date.now()}-${Math.random().toString(16).slice(2)}`;
|
||||
}
|
||||
}
|
||||
|
||||
function normalise(value: unknown): unknown {
|
||||
if (value instanceof Error) {
|
||||
return { name: value.name, message: value.message, stack: value.stack };
|
||||
}
|
||||
if (typeof value === "bigint") return value.toString();
|
||||
if (Array.isArray(value)) return value.map(normalise);
|
||||
if (value && typeof value === "object") {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value as Record<string, unknown>).map(([key, child]) => [
|
||||
key,
|
||||
normalise(child),
|
||||
]),
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Persistent diagnostics for the MV3 background worker.
|
||||
*
|
||||
* Events are first written to chrome.storage.local so a worker termination or
|
||||
* dead WebSocket does not erase the evidence. Once the post-handshake link is
|
||||
* ready, pending entries are forwarded to the daemon as one
|
||||
* `browser.diagnostic` event and removed from the local queue.
|
||||
*/
|
||||
export class DiagnosticLogger {
|
||||
readonly workerBootId: string;
|
||||
readonly sink: DiagnosticSink;
|
||||
|
||||
private sequence = 0;
|
||||
private storageQueue: Promise<void> = Promise.resolve();
|
||||
private flushPromise: Promise<void> | null = null;
|
||||
private flushRequested = false;
|
||||
private send: ((frame: ProtocolFrame) => void) | null = null;
|
||||
private transportReady = false;
|
||||
private readonly storage: DiagnosticStorage | null;
|
||||
|
||||
constructor(storage: DiagnosticStorage | null = defaultStorage()) {
|
||||
this.workerBootId = makeBootId();
|
||||
this.storage = storage;
|
||||
this.sink = (event, fields = {}) => this.record(event, fields);
|
||||
}
|
||||
|
||||
bindTransport(send: (frame: ProtocolFrame) => void): void {
|
||||
this.send = send;
|
||||
}
|
||||
|
||||
setTransportReady(ready: boolean): void {
|
||||
this.transportReady = ready;
|
||||
if (ready) void this.flush();
|
||||
}
|
||||
|
||||
record(event: string, fields: DiagnosticFields = {}): void {
|
||||
const sequence = ++this.sequence;
|
||||
const entry: DiagnosticEntry = {
|
||||
id: `${this.workerBootId}:${sequence}`,
|
||||
sequence,
|
||||
timestamp: new Date().toISOString(),
|
||||
monotonic_ms: Math.round(performance.now()),
|
||||
worker_boot_id: this.workerBootId,
|
||||
event,
|
||||
fields: normalise(fields) as DiagnosticFields,
|
||||
};
|
||||
console.info("[bsk diagnostic]", entry);
|
||||
this.storageQueue = this.storageQueue
|
||||
.then(async () => {
|
||||
if (!this.storage) return;
|
||||
const stored = await this.storage.get(STORAGE_KEY);
|
||||
const previous = Array.isArray(stored[STORAGE_KEY])
|
||||
? (stored[STORAGE_KEY] as DiagnosticEntry[])
|
||||
: [];
|
||||
await this.storage.set({
|
||||
[STORAGE_KEY]: [...previous, entry].slice(-MAX_ENTRIES),
|
||||
});
|
||||
})
|
||||
.catch((error) => {
|
||||
console.warn("[bsk diagnostic] failed to persist event", error);
|
||||
});
|
||||
if (this.transportReady) void this.flush();
|
||||
}
|
||||
|
||||
flush(): Promise<void> {
|
||||
if (this.flushPromise) {
|
||||
this.flushRequested = true;
|
||||
return this.flushPromise;
|
||||
}
|
||||
const running = (async () => {
|
||||
await this.storageQueue;
|
||||
if (!this.transportReady || !this.send || !this.storage) return;
|
||||
const stored = await this.storage.get(STORAGE_KEY);
|
||||
const entries = Array.isArray(stored[STORAGE_KEY])
|
||||
? (stored[STORAGE_KEY] as DiagnosticEntry[])
|
||||
: [];
|
||||
if (entries.length === 0) return;
|
||||
try {
|
||||
this.send({
|
||||
event: "browser.diagnostic",
|
||||
payload: {
|
||||
schema_version: 1,
|
||||
worker_boot_id: this.workerBootId,
|
||||
entries,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn("[bsk diagnostic] daemon flush failed", error);
|
||||
return;
|
||||
}
|
||||
const sentIds = new Set(entries.map((entry) => entry.id));
|
||||
// Serialize acknowledgement behind every record write that may have
|
||||
// started while send() was in progress. Without this, a concurrent
|
||||
// record can read the pre-ack queue and write an already-sent entry
|
||||
// back into storage, causing the next batch to duplicate it.
|
||||
this.storageQueue = this.storageQueue
|
||||
.then(async () => {
|
||||
const current = await this.storage?.get(STORAGE_KEY);
|
||||
const currentEntries = Array.isArray(current?.[STORAGE_KEY])
|
||||
? (current[STORAGE_KEY] as DiagnosticEntry[])
|
||||
: [];
|
||||
await this.storage?.set({
|
||||
[STORAGE_KEY]: currentEntries.filter((entry) => !sentIds.has(entry.id)),
|
||||
});
|
||||
})
|
||||
.catch((error) => {
|
||||
console.warn("[bsk diagnostic] failed to acknowledge event batch", error);
|
||||
});
|
||||
await this.storageQueue;
|
||||
})().catch((error) => {
|
||||
console.warn("[bsk diagnostic] flush failed", error);
|
||||
});
|
||||
this.flushPromise = running.finally(() => {
|
||||
this.flushPromise = null;
|
||||
if (this.flushRequested && this.transportReady) {
|
||||
this.flushRequested = false;
|
||||
void this.flush();
|
||||
}
|
||||
});
|
||||
return this.flushPromise;
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,7 @@
|
||||
* after the handshake has landed.
|
||||
*/
|
||||
|
||||
import type { DiagnosticSink } from "@/lib/diagnostics";
|
||||
import type { ProtocolFrame } from "@/transport/types";
|
||||
|
||||
export const HEARTBEAT_EVENT = "system.heartbeat";
|
||||
@@ -46,6 +47,7 @@ export interface HeartbeatOptions {
|
||||
intervalMs?: number;
|
||||
/** Inject fake timers (tests). */
|
||||
timers?: HeartbeatTimers;
|
||||
diagnostics?: DiagnosticSink;
|
||||
}
|
||||
|
||||
export interface HeartbeatHandle {
|
||||
@@ -74,7 +76,9 @@ export function startHeartbeat(options: HeartbeatOptions): HeartbeatHandle {
|
||||
const beat = () => {
|
||||
try {
|
||||
options.send({ event: HEARTBEAT_EVENT, payload: {} });
|
||||
options.diagnostics?.("heartbeat.sent");
|
||||
} catch (err) {
|
||||
options.diagnostics?.("heartbeat.send_failed", { error: err });
|
||||
// The socket raced a close between the state change and this
|
||||
// tick. The transport's own reconnect path recovers; swallow so
|
||||
// the interval keeps running (it is cleared on the disconnect
|
||||
@@ -87,12 +91,14 @@ export function startHeartbeat(options: HeartbeatOptions): HeartbeatHandle {
|
||||
if (timer !== null) {
|
||||
timers.clearInterval(timer);
|
||||
timer = null;
|
||||
options.diagnostics?.("heartbeat.timer.stopped");
|
||||
}
|
||||
};
|
||||
|
||||
const start = () => {
|
||||
if (timer !== null) return;
|
||||
timer = timers.setInterval(beat, intervalMs);
|
||||
options.diagnostics?.("heartbeat.timer.started", { interval_ms: intervalMs });
|
||||
};
|
||||
|
||||
const unsubscribe = options.onActiveChange((active) => {
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
* SW idle threshold.
|
||||
*/
|
||||
|
||||
import type { DiagnosticSink } from "@/lib/diagnostics";
|
||||
import type { Transport } from "@/transport/transport";
|
||||
|
||||
export const KEEPALIVE_ALARM_NAME = "bh-keepalive";
|
||||
@@ -59,6 +60,7 @@ export interface KeepaliveOptions {
|
||||
alarmName?: string;
|
||||
/** Override the alarm period (tests). */
|
||||
periodMin?: number;
|
||||
diagnostics?: DiagnosticSink;
|
||||
}
|
||||
|
||||
export interface KeepaliveHandle {
|
||||
@@ -79,11 +81,24 @@ export function startKeepalive(options: KeepaliveOptions): KeepaliveHandle {
|
||||
const period = options.periodMin ?? KEEPALIVE_PERIOD_MIN;
|
||||
|
||||
const tick = async () => {
|
||||
if (options.shouldConnect && !options.shouldConnect()) return;
|
||||
if (options.transport.state === "connected") return;
|
||||
if (options.shouldConnect && !options.shouldConnect()) {
|
||||
options.diagnostics?.("keepalive.tick.skipped", { reason: "connection_disabled" });
|
||||
return;
|
||||
}
|
||||
if (options.transport.state === "connected") {
|
||||
options.diagnostics?.("keepalive.tick.healthy", { transport_state: "connected" });
|
||||
return;
|
||||
}
|
||||
options.diagnostics?.("keepalive.reconnect.started", {
|
||||
transport_state: options.transport.state,
|
||||
});
|
||||
try {
|
||||
await options.transport.connect();
|
||||
options.diagnostics?.("keepalive.reconnect.completed", {
|
||||
transport_state: options.transport.state,
|
||||
});
|
||||
} catch (err) {
|
||||
options.diagnostics?.("keepalive.reconnect.failed", { error: err });
|
||||
console.debug("[bsk keepalive] connect attempt failed", err);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { DiagnosticSink } from "@/lib/diagnostics";
|
||||
import { handleSessionStop, type SessionStopDeps } from "@/tools/session";
|
||||
import type { SessionManager } from "./manager";
|
||||
|
||||
@@ -15,6 +16,7 @@ export interface DisconnectCleanupOptions {
|
||||
manager: SessionManager;
|
||||
sessionStopDeps?: SessionStopDeps;
|
||||
onSessionsChanged?: () => void;
|
||||
diagnostics?: DiagnosticSink;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -29,7 +31,15 @@ export function createDisconnectCleanup(options: DisconnectCleanupOptions) {
|
||||
let inFlight: Promise<DisconnectCleanupReport> | null = null;
|
||||
|
||||
return (): Promise<DisconnectCleanupReport> => {
|
||||
if (inFlight) return inFlight;
|
||||
if (inFlight) {
|
||||
options.diagnostics?.("disconnect_cleanup.coalesced", {
|
||||
live_session_ids: options.manager.list().map((ctx) => ctx.sessionId),
|
||||
});
|
||||
return inFlight;
|
||||
}
|
||||
options.diagnostics?.("disconnect_cleanup.scheduled", {
|
||||
live_session_ids: options.manager.list().map((ctx) => ctx.sessionId),
|
||||
});
|
||||
inFlight = cleanupSessions(options).finally(() => {
|
||||
inFlight = null;
|
||||
});
|
||||
@@ -42,8 +52,17 @@ async function cleanupSessions(
|
||||
): Promise<DisconnectCleanupReport> {
|
||||
const stoppedSessionIds: string[] = [];
|
||||
const failures: DisconnectCleanupFailure[] = [];
|
||||
const capturedSessions = options.manager.list();
|
||||
options.diagnostics?.("disconnect_cleanup.started", {
|
||||
captured_session_ids: capturedSessions.map((ctx) => ctx.sessionId),
|
||||
});
|
||||
|
||||
for (const ctx of options.manager.list()) {
|
||||
for (const ctx of capturedSessions) {
|
||||
options.diagnostics?.("disconnect_cleanup.session_stop.started", {
|
||||
session_id: ctx.sessionId,
|
||||
agent_window_id: ctx.agentWindowId,
|
||||
still_registered: options.manager.has(ctx.sessionId),
|
||||
});
|
||||
try {
|
||||
const result = await handleSessionStop(
|
||||
options.manager,
|
||||
@@ -62,14 +81,26 @@ async function cleanupSessions(
|
||||
continue;
|
||||
}
|
||||
stoppedSessionIds.push(ctx.sessionId);
|
||||
options.diagnostics?.("disconnect_cleanup.session_stop.completed", {
|
||||
session_id: ctx.sessionId,
|
||||
});
|
||||
} catch (err) {
|
||||
failures.push({
|
||||
sessionId: ctx.sessionId,
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
options.diagnostics?.("disconnect_cleanup.session_stop.failed", {
|
||||
session_id: ctx.sessionId,
|
||||
error: err,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
options.onSessionsChanged?.();
|
||||
options.diagnostics?.("disconnect_cleanup.completed", {
|
||||
stopped_session_ids: stoppedSessionIds,
|
||||
failures,
|
||||
remaining_session_ids: options.manager.list().map((ctx) => ctx.sessionId),
|
||||
});
|
||||
return { stoppedSessionIds, failures };
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { DiagnosticSink } from "@/lib/diagnostics";
|
||||
import type { Transport } from "@/transport/transport";
|
||||
import type { EventFrame } from "@/transport/types";
|
||||
import type { SessionManager } from "./manager";
|
||||
@@ -24,6 +25,7 @@ export interface SessionEventHandlerOptions {
|
||||
* `chrome.storage.session` "sessions live" flag (review M4/M5 I3).
|
||||
*/
|
||||
onSessionsChanged?: () => void;
|
||||
diagnostics?: DiagnosticSink;
|
||||
}
|
||||
|
||||
function chromeWindowEvents(): WindowRemovedListener {
|
||||
@@ -50,6 +52,11 @@ export function attachSessionEventHandler(options: SessionEventHandlerOptions):
|
||||
|
||||
const onRemoved = (windowId: number): void => {
|
||||
const ctx = manager.findByWindowId(windowId);
|
||||
options.diagnostics?.("chrome.window.removed", {
|
||||
window_id: windowId,
|
||||
matched_session_id: ctx?.sessionId ?? null,
|
||||
live_session_ids: manager.list().map((entry) => entry.sessionId),
|
||||
});
|
||||
if (!ctx) return;
|
||||
const returnFailures = Array.from(ctx.borrowedTabs.keys()).map((tabId) => ({
|
||||
tab_id: tabId,
|
||||
@@ -81,7 +88,16 @@ export function attachSessionEventHandler(options: SessionEventHandlerOptions):
|
||||
};
|
||||
try {
|
||||
transport.send(event);
|
||||
options.diagnostics?.("session.window_closed.sent", {
|
||||
session_id: ctx.sessionId,
|
||||
window_id: windowId,
|
||||
});
|
||||
} catch (err) {
|
||||
options.diagnostics?.("session.window_closed.send_failed", {
|
||||
session_id: ctx.sessionId,
|
||||
window_id: windowId,
|
||||
error: err,
|
||||
});
|
||||
console.warn("[bh] could not push session.window_closed event", err);
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { DiagnosticSink } from "@/lib/diagnostics";
|
||||
import { AGENT_WINDOW_HOME, type AgentWindowApi, chromeAgentWindowApi } from "./agent-window";
|
||||
import { RefStore } from "./ref-store";
|
||||
|
||||
@@ -23,6 +24,7 @@ export interface BorrowReservation {
|
||||
export interface SessionManagerOptions {
|
||||
agentWindow?: AgentWindowApi;
|
||||
now?: () => number;
|
||||
diagnostics?: DiagnosticSink;
|
||||
}
|
||||
|
||||
/** Options for starting a session's Agent Window. */
|
||||
@@ -83,10 +85,12 @@ export class SessionManager {
|
||||
private readonly borrowReservations = new Map<number, string>();
|
||||
private readonly agentWindow: AgentWindowApi;
|
||||
private readonly now: () => number;
|
||||
private readonly diagnostics?: DiagnosticSink;
|
||||
|
||||
constructor(options: SessionManagerOptions = {}) {
|
||||
this.agentWindow = options.agentWindow ?? chromeAgentWindowApi;
|
||||
this.now = options.now ?? Date.now;
|
||||
this.diagnostics = options.diagnostics;
|
||||
}
|
||||
|
||||
has(sessionId: string): boolean {
|
||||
@@ -169,6 +173,13 @@ export class SessionManager {
|
||||
* daemon in the `tool.session_start` reply.
|
||||
*/
|
||||
async start(sessionId: string, opts: SessionStartOptions = {}): Promise<SessionContext> {
|
||||
this.diagnostics?.("session_manager.start.requested", {
|
||||
session_id: sessionId,
|
||||
existing_session_ids: this.list().map((ctx) => ctx.sessionId),
|
||||
focused: opts.focused ?? null,
|
||||
size: opts.size ?? null,
|
||||
signal_aborted: opts.signal?.aborted ?? false,
|
||||
});
|
||||
if (this.sessions.has(sessionId)) {
|
||||
throw new Error(`[bh] session ${sessionId} already exists`);
|
||||
}
|
||||
@@ -178,6 +189,10 @@ export class SessionManager {
|
||||
try {
|
||||
const { signal: _signal, ...createOptions } = opts;
|
||||
windowId = await this.agentWindow.create(AGENT_WINDOW_HOME, createOptions);
|
||||
this.diagnostics?.("session_manager.window.created", {
|
||||
session_id: sessionId,
|
||||
agent_window_id: windowId,
|
||||
});
|
||||
throwIfSessionStartAborted(opts.signal);
|
||||
await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
|
||||
throwIfSessionStartAborted(opts.signal);
|
||||
@@ -191,8 +206,18 @@ export class SessionManager {
|
||||
};
|
||||
this.sessions.set(sessionId, ctx);
|
||||
this.windowIndex.set(windowId, sessionId);
|
||||
this.diagnostics?.("session_manager.start.committed", {
|
||||
session_id: sessionId,
|
||||
agent_window_id: windowId,
|
||||
live_session_ids: this.list().map((entry) => entry.sessionId),
|
||||
});
|
||||
return ctx;
|
||||
} catch (startupError) {
|
||||
this.diagnostics?.("session_manager.start.failed", {
|
||||
session_id: sessionId,
|
||||
agent_window_id: windowId,
|
||||
error: startupError,
|
||||
});
|
||||
if (windowId !== null) {
|
||||
try {
|
||||
await this.agentWindow.remove(windowId);
|
||||
@@ -216,12 +241,28 @@ export class SessionManager {
|
||||
options: { dropOnly?: boolean } = {},
|
||||
): Promise<SessionContext | null> {
|
||||
const ctx = this.sessions.get(sessionId);
|
||||
this.diagnostics?.("session_manager.stop.requested", {
|
||||
session_id: sessionId,
|
||||
drop_only: options.dropOnly ?? false,
|
||||
existed: ctx !== undefined,
|
||||
live_session_ids: this.list().map((entry) => entry.sessionId),
|
||||
});
|
||||
if (!ctx) return null;
|
||||
if (!options.dropOnly) {
|
||||
await this.agentWindow.remove(ctx.agentWindowId);
|
||||
this.diagnostics?.("session_manager.window.removed", {
|
||||
session_id: sessionId,
|
||||
agent_window_id: ctx.agentWindowId,
|
||||
});
|
||||
}
|
||||
this.sessions.delete(sessionId);
|
||||
this.windowIndex.delete(ctx.agentWindowId);
|
||||
this.diagnostics?.("session_manager.stop.committed", {
|
||||
session_id: sessionId,
|
||||
agent_window_id: ctx.agentWindowId,
|
||||
drop_only: options.dropOnly ?? false,
|
||||
live_session_ids: this.list().map((entry) => entry.sessionId),
|
||||
});
|
||||
return ctx;
|
||||
}
|
||||
|
||||
|
||||
@@ -456,6 +456,30 @@ describe("handleClick", () => {
|
||||
expect(res).toMatchObject({ code: "not_found", data: { reason: "selector_not_found" } });
|
||||
});
|
||||
|
||||
it("returns invalid_selector when Chrome rejects non-CSS selector syntax", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const fake = makeFakeCdp({
|
||||
"DOM.getDocument": () => ({ root: { nodeId: 1 } }),
|
||||
"DOM.querySelector": () => {
|
||||
throw new Error('{"code":-32000,"message":"DOM Error while querying"}');
|
||||
},
|
||||
});
|
||||
const res = await handleClick(
|
||||
sm,
|
||||
{ session_id: "aa11", selector: "text=下午 12~18点" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
expect(res).toMatchObject({
|
||||
code: "invalid_params",
|
||||
message: "selector is not valid CSS: text=下午 12~18点",
|
||||
data: {
|
||||
reason: "invalid_selector",
|
||||
selector: "text=下午 12~18点",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("respects the AbortSignal", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { DiagnosticSink } from "@/lib/diagnostics";
|
||||
import { OVERLAY_AUTOMATION_BYPASS } from "@/lib/overlay-bridge";
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import type { Transport } from "@/transport/transport";
|
||||
@@ -118,6 +119,7 @@ export interface DispatcherDeps {
|
||||
approveBorrow?: BorrowConfirmationApprover;
|
||||
/** i18n notification copy for `tool.request_help` (resolved per-call). */
|
||||
helpNotificationCopy?: () => { title: string; body: string };
|
||||
diagnostics?: DiagnosticSink;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -145,6 +147,7 @@ export class ToolDispatcher {
|
||||
private readonly onBrowserControlResumed?: (sessionId: string) => void;
|
||||
private readonly approveBorrow?: BorrowConfirmationApprover;
|
||||
private readonly helpNotificationCopy?: () => { title: string; body: string };
|
||||
private readonly diagnostics?: DiagnosticSink;
|
||||
private subscription: { dispose(): void } | null = null;
|
||||
private readonly hoverBypassTabs = new Map<number, string>();
|
||||
private readonly hoverLatches = new Map<number, HoverLatch>();
|
||||
@@ -165,6 +168,7 @@ export class ToolDispatcher {
|
||||
this.onBrowserControlResumed = deps.onBrowserControlResumed;
|
||||
this.approveBorrow = deps.approveBorrow;
|
||||
this.helpNotificationCopy = deps.helpNotificationCopy;
|
||||
this.diagnostics = deps.diagnostics;
|
||||
}
|
||||
|
||||
start(): void {
|
||||
@@ -192,6 +196,16 @@ export class ToolDispatcher {
|
||||
private async dispatch(msg: ProtocolFrame): Promise<void> {
|
||||
if (!isRequestFrame(msg)) return;
|
||||
const req = msg as RequestFrame;
|
||||
const requestSessionId = (req.params as { session_id?: unknown } | undefined)?.session_id;
|
||||
const startedAt = performance.now();
|
||||
this.diagnostics?.("dispatcher.rpc.received", {
|
||||
rpc_id: req.id,
|
||||
method: req.method,
|
||||
session_id: typeof requestSessionId === "string" ? requestSessionId : null,
|
||||
transport_state: this.transport.state,
|
||||
live_session_ids: this.sessions.list().map((ctx) => ctx.sessionId),
|
||||
inflight_rpc_ids: Array.from(this.inflightAbortControllers.keys()),
|
||||
});
|
||||
|
||||
// Cancel frames take a fast path: trip the matching controller
|
||||
// (if any), reply with `{cancelled}` so the daemon can answer
|
||||
@@ -200,6 +214,12 @@ export class ToolDispatcher {
|
||||
const params = (req.params as { rpc_id?: string } | undefined) ?? {};
|
||||
const target = typeof params.rpc_id === "string" ? params.rpc_id : "";
|
||||
const ac = target ? this.inflightAbortControllers.get(target) : undefined;
|
||||
this.diagnostics?.("dispatcher.cancel.received", {
|
||||
rpc_id: req.id,
|
||||
target_rpc_id: target,
|
||||
target_found: ac !== undefined,
|
||||
inflight_rpc_ids: Array.from(this.inflightAbortControllers.keys()),
|
||||
});
|
||||
if (ac) {
|
||||
try {
|
||||
ac.abort();
|
||||
@@ -213,7 +233,17 @@ export class ToolDispatcher {
|
||||
};
|
||||
try {
|
||||
this.transport.send(reply);
|
||||
this.diagnostics?.("dispatcher.cancel.ack_sent", {
|
||||
rpc_id: req.id,
|
||||
target_rpc_id: target,
|
||||
cancelled: ac !== undefined,
|
||||
});
|
||||
} catch (sendErr) {
|
||||
this.diagnostics?.("dispatcher.cancel.ack_failed", {
|
||||
rpc_id: req.id,
|
||||
target_rpc_id: target,
|
||||
error: sendErr,
|
||||
});
|
||||
console.warn("[bsk dispatcher] failed to ack cancel", sendErr);
|
||||
}
|
||||
return;
|
||||
@@ -238,6 +268,13 @@ export class ToolDispatcher {
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
this.diagnostics?.("dispatcher.rpc.invoke_failed", {
|
||||
rpc_id: req.id,
|
||||
method: req.method,
|
||||
session_id: typeof requestSessionId === "string" ? requestSessionId : null,
|
||||
abort_signal: ac.signal.aborted,
|
||||
error: err,
|
||||
});
|
||||
if (isAbortLikeError(err)) {
|
||||
body = {
|
||||
id: req.id,
|
||||
@@ -258,8 +295,27 @@ export class ToolDispatcher {
|
||||
let sent = true;
|
||||
try {
|
||||
this.transport.send(body);
|
||||
this.diagnostics?.("dispatcher.rpc.response_sent", {
|
||||
rpc_id: req.id,
|
||||
method: req.method,
|
||||
session_id: typeof requestSessionId === "string" ? requestSessionId : null,
|
||||
response_kind: "error" in body ? "error" : "ok",
|
||||
error_code: "error" in body ? body.error.code : null,
|
||||
elapsed_ms: Math.round(performance.now() - startedAt),
|
||||
transport_state: this.transport.state,
|
||||
live_session_ids: this.sessions.list().map((ctx) => ctx.sessionId),
|
||||
});
|
||||
} catch (sendErr) {
|
||||
sent = false;
|
||||
this.diagnostics?.("dispatcher.rpc.response_send_failed", {
|
||||
rpc_id: req.id,
|
||||
method: req.method,
|
||||
session_id: typeof requestSessionId === "string" ? requestSessionId : null,
|
||||
started_session_id: startedSession,
|
||||
elapsed_ms: Math.round(performance.now() - startedAt),
|
||||
transport_state: this.transport.state,
|
||||
error: sendErr,
|
||||
});
|
||||
// Transport is dead by the time we want to reply. Drop the link
|
||||
// proactively so the alarm-driven keepalive reconnects sooner
|
||||
// and the daemon's pending RPC times out cleanly instead of
|
||||
@@ -278,12 +334,23 @@ export class ToolDispatcher {
|
||||
try {
|
||||
const ctx = await this.sessions.stop(startedSession);
|
||||
if (ctx) {
|
||||
this.diagnostics?.("dispatcher.session_start.rolled_back", {
|
||||
rpc_id: req.id,
|
||||
session_id: startedSession,
|
||||
agent_window_id: ctx.agentWindowId,
|
||||
reason: "response_send_failed",
|
||||
});
|
||||
console.warn(
|
||||
"[bsk dispatcher] rolled back orphan session after send failure",
|
||||
startedSession,
|
||||
);
|
||||
}
|
||||
} catch (rollbackErr) {
|
||||
this.diagnostics?.("dispatcher.session_start.rollback_failed", {
|
||||
rpc_id: req.id,
|
||||
session_id: startedSession,
|
||||
error: rollbackErr,
|
||||
});
|
||||
console.warn("[bsk dispatcher] session rollback after send failure failed", rollbackErr);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -106,6 +106,14 @@ function isAbortLikeError(err: unknown): boolean {
|
||||
return err instanceof DOMException && err.name === "AbortError";
|
||||
}
|
||||
|
||||
function isInvalidCssSelectorError(err: unknown): boolean {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
return (
|
||||
message.includes("DOM Error while querying") ||
|
||||
/invalid selector|not a valid selector|querySelector.*failed/i.test(message)
|
||||
);
|
||||
}
|
||||
|
||||
async function wait(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
if (signal?.aborted) throw new DOMException("aborted", "AbortError");
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
@@ -208,6 +216,14 @@ async function resolveBackendNode(
|
||||
}
|
||||
return { backendNodeId, cdpTarget: { tabId: target.tabId }, usedSelector: params.selector };
|
||||
} catch (err) {
|
||||
if (isInvalidCssSelectorError(err)) {
|
||||
return rpcError(
|
||||
"invalid_params",
|
||||
"invalid_selector",
|
||||
`selector is not valid CSS: ${params.selector}`,
|
||||
{ selector: params.selector },
|
||||
);
|
||||
}
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
|
||||
@@ -24,6 +24,7 @@ export type RpcErrorReason =
|
||||
| "agent_window_scope"
|
||||
| "element_not_visible"
|
||||
| "ref_not_found"
|
||||
| "invalid_selector"
|
||||
| "selector_not_found"
|
||||
| "target_not_fillable"
|
||||
| "target_not_select"
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { DiagnosticSink } from "@/lib/diagnostics";
|
||||
import type { ConnectionStateHandler, Disposable, FrameHandler, Transport } from "./transport";
|
||||
import type { ConnectionState, ProtocolFrame } from "./types";
|
||||
|
||||
@@ -18,6 +19,7 @@ export interface WSTransportOptions {
|
||||
* Inject a fake WebSocket constructor in tests; defaults to `globalThis.WebSocket`.
|
||||
*/
|
||||
webSocketFactory?: WebSocketFactory;
|
||||
diagnostics?: DiagnosticSink;
|
||||
}
|
||||
|
||||
interface CloseLikeEvent {
|
||||
@@ -44,6 +46,7 @@ export class WSTransport implements Transport {
|
||||
private readonly factory: WebSocketFactory;
|
||||
private readonly initialDelayMs: number;
|
||||
private readonly maxDelayMs: number;
|
||||
private readonly diagnostics?: DiagnosticSink;
|
||||
|
||||
private socket: WebSocket | null = null;
|
||||
private currentState: ConnectionState = "disconnected";
|
||||
@@ -63,6 +66,7 @@ export class WSTransport implements Transport {
|
||||
this.factory = options.webSocketFactory ?? ((url: string) => new WebSocket(url));
|
||||
this.initialDelayMs = options.reconnect?.initialDelayMs ?? DEFAULT_INITIAL_DELAY_MS;
|
||||
this.maxDelayMs = options.reconnect?.maxDelayMs ?? DEFAULT_MAX_DELAY_MS;
|
||||
this.diagnostics = options.diagnostics;
|
||||
}
|
||||
|
||||
get state(): ConnectionState {
|
||||
@@ -70,6 +74,13 @@ export class WSTransport implements Transport {
|
||||
}
|
||||
|
||||
connect(): Promise<void> {
|
||||
this.diagnostics?.("transport.connect.requested", {
|
||||
state: this.currentState,
|
||||
socket_generation: this.socketGeneration,
|
||||
has_socket: this.socket !== null,
|
||||
has_connecting_promise: this.connectingPromise !== null,
|
||||
has_reconnect_timer: this.reconnectTimer !== null,
|
||||
});
|
||||
if (this.currentState === "connected") return Promise.resolve();
|
||||
|
||||
this.explicitlyClosed = false;
|
||||
@@ -94,6 +105,11 @@ export class WSTransport implements Transport {
|
||||
}
|
||||
|
||||
async disconnect(): Promise<void> {
|
||||
this.diagnostics?.("transport.disconnect.requested", {
|
||||
state: this.currentState,
|
||||
socket_generation: this.socketGeneration,
|
||||
has_socket: this.socket !== null,
|
||||
});
|
||||
this.explicitlyClosed = true;
|
||||
if (this.reconnectTimer) {
|
||||
clearTimeout(this.reconnectTimer);
|
||||
@@ -124,6 +140,12 @@ export class WSTransport implements Transport {
|
||||
|
||||
send(msg: ProtocolFrame): void {
|
||||
if (!this.socket || this.socket.readyState !== 1 /* OPEN */) {
|
||||
this.diagnostics?.("transport.send.rejected", {
|
||||
state: this.currentState,
|
||||
socket_generation: this.socketGeneration,
|
||||
socket_ready_state: this.socket?.readyState ?? null,
|
||||
frame_kind: "method" in msg ? msg.method : "event" in msg ? msg.event : "response",
|
||||
});
|
||||
throw new Error("[WSTransport] cannot send while not connected");
|
||||
}
|
||||
this.socket.send(JSON.stringify(msg));
|
||||
@@ -152,10 +174,16 @@ export class WSTransport implements Transport {
|
||||
const generation = ++this.socketGeneration;
|
||||
const socket = this.factory(this.url);
|
||||
this.socket = socket;
|
||||
this.diagnostics?.("transport.socket.created", {
|
||||
socket_generation: generation,
|
||||
reconnect_attempt: this.reconnectAttempt,
|
||||
url: this.url,
|
||||
});
|
||||
|
||||
socket.addEventListener("open", () => {
|
||||
if (!this.isCurrentSocket(socket, generation)) return;
|
||||
this.reconnectAttempt = 0;
|
||||
this.diagnostics?.("transport.socket.open", { socket_generation: generation });
|
||||
this.setState("connected");
|
||||
const resolve = this.resolveConnect;
|
||||
this.resolveConnect = null;
|
||||
@@ -174,6 +202,7 @@ export class WSTransport implements Transport {
|
||||
});
|
||||
|
||||
socket.addEventListener("error", () => {
|
||||
this.diagnostics?.("transport.socket.error", { socket_generation: generation });
|
||||
// The 'close' handler always fires after 'error' in browsers, so we
|
||||
// just record the error and let close drive reconnect logic.
|
||||
});
|
||||
@@ -197,7 +226,16 @@ export class WSTransport implements Transport {
|
||||
}
|
||||
|
||||
private handleClose(socket: WebSocket, generation: number, _ev: CloseLikeEvent): void {
|
||||
if (!this.isCurrentSocket(socket, generation)) return;
|
||||
const isCurrent = this.isCurrentSocket(socket, generation);
|
||||
this.diagnostics?.("transport.socket.close", {
|
||||
socket_generation: generation,
|
||||
current_socket_generation: this.socketGeneration,
|
||||
is_current: isCurrent,
|
||||
explicitly_closed: this.explicitlyClosed,
|
||||
close_code: _ev.code ?? null,
|
||||
close_reason: _ev.reason ?? "",
|
||||
});
|
||||
if (!isCurrent) return;
|
||||
this.socket = null;
|
||||
if (this.explicitlyClosed) {
|
||||
this.setState("disconnected");
|
||||
@@ -211,9 +249,18 @@ export class WSTransport implements Transport {
|
||||
if (this.reconnectTimer) return;
|
||||
const delay = Math.min(this.initialDelayMs * 2 ** this.reconnectAttempt, this.maxDelayMs);
|
||||
this.reconnectAttempt += 1;
|
||||
this.diagnostics?.("transport.reconnect.scheduled", {
|
||||
delay_ms: delay,
|
||||
reconnect_attempt: this.reconnectAttempt,
|
||||
socket_generation: this.socketGeneration,
|
||||
});
|
||||
this.reconnectTimer = setTimeout(() => {
|
||||
this.reconnectTimer = null;
|
||||
if (this.explicitlyClosed) return;
|
||||
this.diagnostics?.("transport.reconnect.timer_fired", {
|
||||
reconnect_attempt: this.reconnectAttempt,
|
||||
socket_generation: this.socketGeneration,
|
||||
});
|
||||
void this.connect().catch((err) => {
|
||||
console.debug("[WSTransport] reconnect attempt failed", err);
|
||||
});
|
||||
@@ -226,7 +273,13 @@ export class WSTransport implements Transport {
|
||||
|
||||
private setState(next: ConnectionState): void {
|
||||
if (this.currentState === next) return;
|
||||
const previous = this.currentState;
|
||||
this.currentState = next;
|
||||
this.diagnostics?.("transport.state.changed", {
|
||||
previous,
|
||||
next,
|
||||
socket_generation: this.socketGeneration,
|
||||
});
|
||||
for (const h of this.stateHandlers) {
|
||||
try {
|
||||
h(next);
|
||||
|
||||
@@ -22,7 +22,8 @@ export default defineConfig({
|
||||
outDir: "dist",
|
||||
modules: ["@wxt-dev/module-react"],
|
||||
manifest: {
|
||||
name: "BrowserSkill",
|
||||
name: "BrowserSkill (Lumi Diagnostics)",
|
||||
version_name: `${EXTENSION_VERSION}-lumi-session-loss-v2`,
|
||||
description:
|
||||
"Let AI agents use your logged-in browser in a separate Agent Window—without interrupting your work. Powered by the bsk CLI.",
|
||||
// Flat debugger sessions are required to address out-of-process iframes.
|
||||
@@ -45,7 +46,7 @@ export default defineConfig({
|
||||
128: "icon/logo.png",
|
||||
},
|
||||
action: {
|
||||
default_title: "BrowserSkill",
|
||||
default_title: "BrowserSkill (Lumi Diagnostics)",
|
||||
default_icon: {
|
||||
16: "icon/logo.png",
|
||||
32: "icon/logo.png",
|
||||
|
||||
+283
-98
@@ -1,142 +1,327 @@
|
||||
---
|
||||
name: browser-skill
|
||||
description: |
|
||||
Use when the user asks to automate their logged-in Chromium browser: visit
|
||||
and read pages, fill forms, scrape data, click through flows, regression-test
|
||||
a PR's UI, validate a deployed page, or operate a tab they identify. Requires
|
||||
the bsk CLI and browser extension.
|
||||
Use when the user asks to perform browser automation tasks against their
|
||||
logged-in browser: visit and read pages, fill forms, scrape data, click
|
||||
through a flow, regression-test a PR's UI, validate a deployed page.
|
||||
Requires the bsk CLI installed and the browser-skill extension loaded.
|
||||
---
|
||||
|
||||
# browser-skill
|
||||
|
||||
Drive the user's real Chromium browser through `bsk`. Automation runs in an isolated **Agent
|
||||
Window** with the user's existing logins and cookies. User-window tabs remain protected unless they
|
||||
are explicitly borrowed.
|
||||
Drive the user's **real Chromium browser** (with their logins and cookies) through the `bsk` CLI. The extension opens an isolated **Agent Window** for automation; the user's normal windows stay protected unless you explicitly borrow a tab.
|
||||
|
||||
Do not use this skill for tasks with no browser, for extension installation, or when the user only
|
||||
wants instructions. Never extract credentials, cookies, tokens, or other secrets from pages.
|
||||
## When to use
|
||||
|
||||
## Required lifecycle
|
||||
- Open pages, read titles/text, scrape structured data from sites the user can already access
|
||||
- Fill forms, click through multi-step flows, smoke-test a UI change
|
||||
- Understand pages with `bsk snapshot` first; use `bsk get-html` or `bsk screenshot` only when the snapshot is insufficient
|
||||
- Operate on a specific user tab they point you at (after `bsk tab borrow`)
|
||||
|
||||
Every browser task owns a bounded session:
|
||||
## When NOT to use
|
||||
|
||||
```text
|
||||
1. bsk session start # retain the printed 4-letter session id
|
||||
2. bsk ... --session <id> # pass it to every session-scoped command
|
||||
3. bsk session stop <id> # always run on success and error paths
|
||||
- Tasks with **no browser** involved (files, APIs, databases only)
|
||||
- Installing or configuring the extension (point the user to setup docs instead)
|
||||
- **Credential harvesting** — never run `bsk evaluate` on banking, SSO, or password-manager pages to extract tokens, cookies, or secrets
|
||||
- Long-lived control of a user's personal login window — borrow only for the immediate step, then `bsk tab return` or end the session
|
||||
- Replacing the user's manual browsing when they only wanted an explanation
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. `bsk` on `PATH` (Rust CLI from browser-skill)
|
||||
2. browser-skill **extension** loaded in Chromium and connected (popup shows green, or an upgrade reminder — still connected; continue)
|
||||
3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails
|
||||
|
||||
## Mandatory workflow
|
||||
|
||||
Every automation task **must** follow this lifecycle. Do **not** rely on idle timeouts (default session idle is 5 minutes).
|
||||
|
||||
```
|
||||
1. bsk session start → capture the 4-letter session id printed on stdout
|
||||
2. … every tool command … → always pass --session <id>
|
||||
3. bsk session stop <id> → REQUIRED when done (even on error paths)
|
||||
```
|
||||
|
||||
Do not rely on the idle timeout for cleanup. Stop the session as soon as the goal is met unless the
|
||||
user explicitly asks to keep it open. Stopping also returns borrowed tabs.
|
||||
Optional: `bsk session start --browser <instance-id-or-label>` when multiple browsers are connected (`bsk browsers` / error output lists them). Add `--no-focus` to open the Agent Window in the background without stealing focus from the user's current window.
|
||||
|
||||
Any `bsk` command auto-starts the background services it needs; never manage the daemon by hand.
|
||||
When multiple browsers are connected, use `bsk browsers` and start with
|
||||
`bsk session start --browser <id-or-label>`. Add `--no-focus` to that same start command when the
|
||||
Agent Window does not need to interrupt the user's current work; it is not a flag on other commands.
|
||||
Run `bsk doctor` when startup or transport problems persist after one retry.
|
||||
Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**.
|
||||
|
||||
## Work toward one observable goal
|
||||
## Stop when the goal is met
|
||||
|
||||
- Derive a concrete success condition from the user's request or a supplied trace.
|
||||
- Take the shortest purposeful path: observe, act, then make at most one observation to confirm an
|
||||
ambiguous result.
|
||||
- Once success is visible, do not click, refresh, navigate, switch tabs, or perform extra checks.
|
||||
- If a human-only step appears or two attempts make no progress, request help instead of
|
||||
brute-forcing.
|
||||
Every task is a **bounded goal**, not open-ended browsing. The goal may come from the user's request, a recorded `trace.json`, or both.
|
||||
|
||||
With a trace, follow its semantic target information and values in order, but treat its refs as
|
||||
record-local hints. Stop when its purpose or last meaningful effect is satisfied. A trace guides the
|
||||
task; it does not expand the user's goal or authorize additional actions.
|
||||
1. **Define success first** — one concrete, observable condition derived from the user's words, `purpose`, or the last meaningful step in a trace (e.g. "form submitted", "item added to cart", "playback started").
|
||||
2. **Take the shortest path** — snapshot → act → at most one check. Do not wander, re-try unrelated actions, or stack exploratory steps.
|
||||
3. **Stop as soon as success is reached** — run `bsk session stop <id>` immediately unless the user explicitly asked to keep the session open (e.g. "don't close yet", "keep browsing").
|
||||
4. **No post-success work** — once the goal is met, do not click, refresh, navigate, re-search, switch tabs, or "double-check" that it worked. Further verification is a new task.
|
||||
5. **When blocked, pause — do not brute-force** — if the page requires human input (login, captcha, OTP, payment confirmation) or an action fails twice with no progress, call `bsk request-help` instead of retrying blindly. See **Ask the human for help** below.
|
||||
6. **When unsure** — at most one extra `bsk snapshot`. If success looks met, stop. If not, ask the user; do not keep clicking.
|
||||
|
||||
## Observe, act, observe
|
||||
**With a trace:** replay steps in order using `target` role/name/tag and raw `value`/`selection` fields. After the last step (or when its `effect.navigated_to` / success hint is satisfied), apply rules 3–4 immediately. The trace guides execution; it does not extend control beyond the goal.
|
||||
|
||||
Use this default loop:
|
||||
**Without a trace:** the user's request *is* the success condition. Satisfying it ends the task — same stop rules apply.
|
||||
|
||||
```text
|
||||
## Core interaction loop
|
||||
|
||||
Write operations only affect tabs in the **Agent Window** (or tabs you **borrowed** into it).
|
||||
|
||||
```
|
||||
bsk navigate <url> --session <id>
|
||||
bsk observe --session <id>
|
||||
bsk click|hover|fill|select|press ... --session <id>
|
||||
bsk observe --session <id> # after navigation or a meaningful DOM change
|
||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk observe --session <id> → again after navigation / DOM change
|
||||
```
|
||||
|
||||
Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also
|
||||
make them stale. Observe again before the next interaction.
|
||||
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
||||
|
||||
An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`.
|
||||
The listed items are labels, not usable refs: hover the trigger, observe again, then act on the
|
||||
revealed item's own ref. Do not click the trigger itself unless the user wants the trigger's action.
|
||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
||||
|
||||
Escalate page reading only as needed:
|
||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
||||
|
||||
1. `bsk observe` for normal semantic understanding, text, controls, and refs.
|
||||
2. `bsk snapshot` when a stricter static accessibility tree is more useful.
|
||||
3. `bsk get-html` for exact markup or hidden metadata that semantic views cannot provide.
|
||||
4. `bsk screenshot` for layout, styling, canvas, images, or requested visual evidence.
|
||||
## Observation priority
|
||||
|
||||
Do not start with raw HTML or screenshots merely to discover ordinary controls. When interaction is
|
||||
needed, obtain a fresh observation before acting on screenshot or HTML findings.
|
||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
|
||||
## Respect the Agent Window boundary
|
||||
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
||||
2. `bsk snapshot` — strict static accessibility tree fallback
|
||||
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
||||
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
||||
|
||||
Normal page writes affect only Agent Window tabs. To operate a user tab, first list it with
|
||||
`bsk tab list --scope user --session <id>`, then `bsk tab borrow <tab-id>`. Return it immediately
|
||||
after the relevant step with `bsk tab return <tab-id>`; never invent a tab id or keep a personal tab
|
||||
borrowed across unrelated work.
|
||||
Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page.
|
||||
|
||||
## Ask the human when needed
|
||||
## Sandbox rules
|
||||
|
||||
Use `bsk request-help` for login, captcha, OTP, payment confirmation, consent, or another step the
|
||||
user must complete. Give a precise prompt and pass fresh `--target` refs/selectors when concrete
|
||||
controls can be highlighted. Use completion criteria only when the page has a clear stable success
|
||||
signal.
|
||||
| Rule | Detail |
|
||||
|------|--------|
|
||||
| Agent Window | `bsk tab create`, `bsk navigate`, `bsk click`, etc. work on agent tabs by default |
|
||||
| User tabs | Read-only until borrowed: `bsk tab list --session <id> --scope user` then `bsk tab borrow <tab-id> --session <id>` |
|
||||
| Return borrowed tabs | Call `bsk tab return <tab-id> --session <id>` when finished; unreturned tabs are **auto-returned** on `bsk session stop` |
|
||||
| Writes off-agent | Commands that mutate the page fail if the tab is not in the Agent Window — borrow or create a tab first |
|
||||
|
||||
The result `outcome` is one of `continued`, `completed`, `cancelled`, `timed_out`, or `disabled`
|
||||
(`navigated` is deprecated — never treat navigation as a completion signal). Resume only after
|
||||
`continued` or `completed`. Treat `cancelled` as rejection, and `timed_out` or `disabled` as a
|
||||
blocker rather than a reason to retry. After control returns, run a fresh `bsk observe` before
|
||||
reasoning about the page or using refs.
|
||||
## Global flags
|
||||
|
||||
## Command inventory
|
||||
| Flag | Purpose |
|
||||
|------|---------|
|
||||
| `--json` | Machine-readable JSON on stdout (errors too) |
|
||||
| `--quiet` | Suppress informational stderr |
|
||||
| `-v` / `-vv` | More verbose logging |
|
||||
|
||||
This list of names is complete. Never invent a command outside it; read
|
||||
`bsk <command...> --help` for flags instead of guessing them.
|
||||
Auto-update is **on by default** — the background daemon upgrades `bsk` itself when a new release is available (postponed while a session is active). Set `BSK_AUTO_UPDATE=off` to disable it and upgrade manually with `bsk update`.
|
||||
|
||||
```text
|
||||
session start|stop|list browsers status doctor update logs
|
||||
navigate navigate-back navigate-forward reload wait-for-navigation wait-ms
|
||||
observe snapshot get-html screenshot console network
|
||||
click hover fill select press evaluate
|
||||
tab list|create|close|select|borrow|return window resize emulate
|
||||
request-help record start|stop
|
||||
Command-specific flags (timeouts, `--tab-id`, `--wait-until`, …): **`bsk <cmd> --help`**
|
||||
|
||||
## CLI command reference (one line each)
|
||||
|
||||
Details and flags: **`bsk <cmd> --help`**
|
||||
|
||||
### Diagnostics
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk status` | Connection health, connected browsers, active sessions |
|
||||
| `bsk doctor` | Deep diagnostics and repair hints |
|
||||
| `bsk browsers` | List connected browser instances (ids, labels, versions) |
|
||||
|
||||
### Session
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk session start` | Open Agent Window (`--width`/`--height` for initial size); prints **4-letter session id** |
|
||||
| `bsk session start --no-focus` | Open Agent Window in the background without stealing focus |
|
||||
| `bsk session stop <id>` | End session, close Agent Window, auto-return borrowed tabs |
|
||||
| `bsk session stop --all` | Stop every active session |
|
||||
| `bsk session list` | List active sessions |
|
||||
|
||||
### Window (require `--session <id>`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk window resize` | Resize the Agent Window (`--width`, `--height`; 100..=7680 CSS px) |
|
||||
|
||||
### Device emulation — `bsk emulate` (requires `--session <id>`)
|
||||
|
||||
Emulate a mobile device environment on the agent tab via CDP — viewport, User-Agent, and touch — to debug a page's mobile layout and behaviour:
|
||||
|
||||
```bash
|
||||
bsk emulate --session <id> --device iphone-14
|
||||
bsk emulate --session <id> --width 390 --height 844 --dpr 3 --mobile --ua "Mozilla/5.0 (iPhone…" --touch
|
||||
bsk emulate --session <id> --off
|
||||
```
|
||||
|
||||
Required flags that are easy to get wrong:
|
||||
- Presets (`--device`): `iphone-14`, `iphone-14-pro-max`, `iphone-se`, `pixel-7`, `galaxy-s23`, `ipad-mini`, `galaxy-tab-s8`. Manual flags (`--width`/`--height`/`--dpr`/`--mobile`/`--ua`/`--accept-language`/`--touch`/`--max-touch-points`) also work without a preset, or override individual preset fields; `--no-mobile`/`--no-touch` turn a preset's mobile viewport / touch emulation back off.
|
||||
- Repeated runs merge field by field onto the tab's current emulation state — only the flags you pass change. E.g. after `--device iphone-14`, `bsk emulate --session <id> --width 390 --height 844` keeps the preset's dpr (`3`) and mobile viewport. (The extension remembers the state per tab; after an extension reload the next run applies only the fields it carries.)
|
||||
- `--off` clears every override (viewport, UA, touch) and the remembered state, restoring the tab's real environment.
|
||||
- Scope: overrides apply to **one tab only** (CDP per-target) and are **not inherited by new tabs** — re-run `bsk emulate` after opening or switching to another tab (default target is the session's active tab; `--tab-id` overrides).
|
||||
- Emulation covers viewport/UA/touch only; it does not throttle the network, fake geolocation, or synthesise real touch-event streams.
|
||||
|
||||
```text
|
||||
bsk fill <ref> --value <text> bsk select <ref> --value <option-value>
|
||||
bsk screenshot --out <path> bsk emulate --device <preset-id>
|
||||
### Tabs (require `--session <id>`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk tab list` | List tabs (`--scope user\|agent\|all`, default `all`) |
|
||||
| `bsk tab create` | New tab in Agent Window (`--url`, `--no-active`, `--index`) |
|
||||
| `bsk tab close <tab-id>` | Close an agent tab |
|
||||
| `bsk tab select <tab-id>` | Focus an agent tab |
|
||||
| `bsk tab borrow <tab-id>` | Move a user tab into the Agent Window |
|
||||
| `bsk tab return <tab-id>` | Return a borrowed tab to its original window |
|
||||
|
||||
### Observation (require `--session` unless noted)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
|
||||
| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
|
||||
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
|
||||
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
|
||||
|
||||
### Console & network debugging (read-only; require `--session`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk console` | Buffered page console messages, JS exceptions, and browser log entries (`--include-stack` for stack traces) |
|
||||
| `bsk network` | Buffered network responses (status, method, URL, MIME/resource type) and failures (`net::ERR_*` reason) |
|
||||
|
||||
Both capture from the moment the tab is attached and read a bounded per-tab buffer: `--since <seq>` pages from a cursor (`next_since` in the result), `--limit` (default 50, max 200), `--max-text-chars` (default 1000, max 4096), `--tab-id` to target a non-active tab. Both are strictly read-only — they never intercept or modify traffic, and request/response headers, bodies, and timings are not captured.
|
||||
|
||||
### Navigation
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk navigate <url>` | Go to URL in agent tab (`--wait-until`, `--timeout`) |
|
||||
| `bsk navigate-back` | History back one step |
|
||||
| `bsk navigate-forward` | History forward one step |
|
||||
| `bsk reload` | Reload current tab (`--hard` bypass cache) |
|
||||
|
||||
(`bsk navigate back` / `bsk navigate forward` are equivalent subcommands.)
|
||||
|
||||
### Interaction
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||
|
||||
### Scripting & timing
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk evaluate <expression>` | Run JS in agent tab (see red lines). JS throw → stderr, **exit 0** (RPC success); use `--json` and check `.ok` to detect JS errors |
|
||||
| `bsk wait-for-navigation` | Block until load/DOM idle/etc. (`--wait-until`, `--timeout`) |
|
||||
| `bsk wait-ms <duration>` | Sleep (`500ms`, `2s`, `1m`; **no** `--session`) |
|
||||
|
||||
### Ask the human for help — `bsk request-help`
|
||||
|
||||
When a step needs a human (captcha, login, OTP) or you want the user to
|
||||
confirm an important action, pause and ask:
|
||||
|
||||
bsk request-help --session <id> --prompt "Solve the captcha, then click Done only after the site accepts it" \
|
||||
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
|
||||
|
||||
- `--prompt` (required): what the user should do.
|
||||
- `--title` (optional): custom title for the overlay panel. When omitted,
|
||||
the extension shows its default localized title.
|
||||
- `--target` (repeatable): a snapshot ref (`@e7`) or CSS selector
|
||||
(`#submit`) to scroll to and flash-highlight. **Strongly recommended** —
|
||||
whenever the prompt refers to a concrete element (a button to click, a
|
||||
field to fill, a checkbox to toggle), pass its `@eN` ref / selector so the
|
||||
user is guided straight to the right spot instead of hunting for it. For
|
||||
interaction scenarios, always include the relevant target(s); reserve a
|
||||
prompt with no `--target` for cases where there is genuinely no specific
|
||||
element to point at (e.g. "wait for the page to finish loading").
|
||||
- `--timeout` (default `5m`): how long to wait.
|
||||
- `--completion-criteria` (optional): JSON success detector. Use it only
|
||||
when there is a concrete post-help success signal, e.g.
|
||||
`{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`.
|
||||
|
||||
The target tab is brought to the foreground; the page stays interactive
|
||||
while the agent control mask is hidden. The call blocks until the user
|
||||
explicitly acts, the timeout expires, cancellation arrives, or explicit
|
||||
completion criteria match. Page reloads, SPA route changes, and captcha
|
||||
refreshes do not return control by themselves. The result `outcome` is one of:
|
||||
|
||||
- `continued` — the user finished and clicked Done / return control (treat as confirm).
|
||||
- `cancelled` — the user clicked Cancel (treat as reject/abort).
|
||||
- `timed_out` — nobody acted within the timeout.
|
||||
- `completed` — the explicit `--completion-criteria` matched while the user had control.
|
||||
- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal.
|
||||
|
||||
`note` carries any text the user typed back. `resolved_targets` reports
|
||||
which refs/selectors matched a live element.
|
||||
|
||||
`request-help` does not refresh the page model after the user returns
|
||||
control. After a `continued` or `completed` result, issue a separate
|
||||
observation tool call (usually `bsk snapshot --session <id>`) before using
|
||||
new refs or reasoning about the post-help page state.
|
||||
#### Disabling request-help (unattended mode)
|
||||
Set `BSK_REQUEST_HELP=off` on unattended servers: `bsk request-help` then
|
||||
returns immediately with `outcome="disabled"` (no overlay, no waiting,
|
||||
exit 0). Any other value keeps it enabled. If you get `disabled`, do not
|
||||
retry — complete the task autonomously or stop gracefully.
|
||||
|
||||
### Recording — `bsk record`
|
||||
|
||||
Capture the user's own actions in the Agent Window to a **trace bundle**, for later LLM-driven automation. New CLI builds request **trace v3** (page observations + action chain); older extensions may still return **trace v2** (actions only), which the CLI exports as a single `trace.json` without `states/`.
|
||||
|
||||
```bash
|
||||
bsk record start --browser <instance-id-or-label> \
|
||||
[--url https://…] [--purpose "publish a wiki doc"] \
|
||||
[--max-page-tokens 3000] [--redact-values] \
|
||||
[--output trace]
|
||||
# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
|
||||
# Blocks until the user clicks Finish in the recording panel, then writes:
|
||||
# trace/trace.json — action chain (+ state index when v3)
|
||||
# trace/states/ — v3 only: one `sN.txt` observe snapshot per settled page state
|
||||
|
||||
bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable
|
||||
```
|
||||
|
||||
`select` matches an option's `value` attribute, not its visible label. Device preset ids are
|
||||
lowercase and hyphenated, such as `iphone-14`.
|
||||
- **v3 bundle (preferred):** `--output` is a directory (default `./trace`) containing `trace.json` and `states/`. `trace.json` lists `states[]` (page observation ids) and `steps[]` (each step binds `state` = observe snapshot *before* the action and `result.state` = snapshot *after* settle). Page bodies live in `states/sN.txt` using the same VOM format as `bsk observe`.
|
||||
- **v2 fallback:** when the connected extension is older, export may contain only legacy `trace.json` with `pages[]` action context and no `states/` observation files. Update the extension for full v3 bundles.
|
||||
- Each `states[]` entry is one **settled page observation** (captured after recording start, navigation landing, or action settle — not on a timer).
|
||||
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
|
||||
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
||||
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
|
||||
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
||||
|
||||
- `console` and `network` provide bounded, read-only debugging evidence.
|
||||
- `emulate` applies viewport, user-agent, and touch overrides to one tab; new tabs do not inherit
|
||||
them. Use `--off` to restore the real environment.
|
||||
- `evaluate` is a last resort when observe plus normal interactions cannot complete the task. With
|
||||
`--json`, inspect `.ok`: a JavaScript exception may still have CLI exit code 0 because the RPC
|
||||
succeeded. Never evaluate credential surfaces to read storage, cookies, or auth data.
|
||||
- `record` captures a user's actions for later replay. Read `bsk record start --help` before use,
|
||||
and never record banking, SSO, password-manager, or other sensitive pages.
|
||||
## Error handling
|
||||
|
||||
## Recover without wandering
|
||||
### Exit codes (`echo $?` after `bsk …`)
|
||||
|
||||
- Stale ref: observe again and retry the intended action once.
|
||||
- Unknown tab or session: list current tabs/sessions; never guess identifiers.
|
||||
- Timeout: inspect current page state before deciding whether one longer purposeful wait is useful.
|
||||
- Unsupported command: continue with available capabilities; suggest updating only when the missing
|
||||
command is necessary.
|
||||
- Unrecoverable failure: report the blocker and stop the session in a finally-style path.
|
||||
| Code | Meaning | What to do |
|
||||
|------|---------|------------|
|
||||
| `0` | Success (including `evaluate` where JS threw but RPC succeeded) | Continue; for `evaluate`, check `--json` `.ok` to distinguish JS success from RPC success |
|
||||
| `1` | User error — bad args, unknown session, tab not in Agent Window, stale ref | Fix args; `bsk session list`; re-snapshot |
|
||||
| `2` | Protocol / transport — service unreachable, IPC failure | `bsk doctor`; check extension connected; retry the command |
|
||||
| `3` | Browser / CDP execution failed | Retry; simplify selector; check tab still open |
|
||||
| `4` | Timeout | Increase `--timeout`; try `--wait-until domcontentloaded` |
|
||||
| `5` | Unknown RPC method | This command is not implemented in the current build. Continue the task with other commands. If a newly added command is missing, suggest `bsk update`. |
|
||||
|
||||
The CLI's current help and error hints are authoritative for flags, parameters, and recovery
|
||||
details.
|
||||
Human errors print `error:` + `hint:` on stderr; `--json` includes `code`, `message`, `hint`, `exit_code`.
|
||||
|
||||
### When to run diagnostics
|
||||
|
||||
| Situation | Command |
|
||||
|-----------|---------|
|
||||
| Before first task in a session | `bsk status` — extension connected? |
|
||||
| Any failure you cannot fix in one retry | `bsk doctor` |
|
||||
| Multiple browsers / wrong target | `bsk browsers` then `bsk session start --browser <id>` |
|
||||
|
||||
Always **`bsk session stop <id>`** in a `finally`-style path so the Agent Window closes and borrowed tabs return.
|
||||
|
||||
## Red lines
|
||||
|
||||
1. **No token theft** — do not `bsk evaluate` on sensitive sites to read `localStorage`, cookies, or auth headers for exfiltration.
|
||||
2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
|
||||
3. **No skip stop** — always `bsk session stop <id>`; never assume idle timeout will clean up.
|
||||
4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
|
||||
5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics or conditional surfaces help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual.
|
||||
6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
|
||||
|
||||
---
|
||||
|
||||
**More detail for any command:** `bsk <cmd> --help`
|
||||
|
||||
@@ -497,4 +497,21 @@ mod tests {
|
||||
.contains("CSS selector")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_selector_explains_css_contract_in_json() {
|
||||
let cli = CliError::from_rpc(RpcError {
|
||||
code: ErrorCode::InvalidParams,
|
||||
message: "selector is not valid CSS: text=submit".into(),
|
||||
data: Some(serde_json::json!({ "reason": "invalid_selector" })),
|
||||
});
|
||||
let exit = cli.exit_code();
|
||||
let hint = hint_for(&cli, render_info_for(&cli).as_ref());
|
||||
let json = json_error_string(&cli, exit, hint);
|
||||
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
|
||||
let hint = parsed.get("hint").and_then(|v| v.as_str()).unwrap();
|
||||
assert!(hint.contains("CSS selector"));
|
||||
assert!(hint.contains("text="));
|
||||
assert!(hint.contains("@eN"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,6 +41,7 @@ pub mod reason {
|
||||
pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope";
|
||||
pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible";
|
||||
pub const REF_NOT_FOUND: &str = "ref_not_found";
|
||||
pub const INVALID_SELECTOR: &str = "invalid_selector";
|
||||
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
|
||||
pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
|
||||
pub const TARGET_NOT_SELECT: &str = "target_not_select";
|
||||
@@ -221,6 +222,13 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
||||
hint: Some("verify the CSS selector or wait for the element to appear before retrying"),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::InvalidParams, reason::INVALID_SELECTOR) => RenderInfo {
|
||||
summary: "selector is not valid CSS",
|
||||
hint: Some(
|
||||
"use a standard CSS selector or a fresh @eN ref; Playwright text= selectors and XPath are not supported",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::InvalidParams, reason::TARGET_NOT_FILLABLE) => RenderInfo {
|
||||
summary: "target element is not fillable",
|
||||
hint: Some(
|
||||
|
||||
@@ -223,6 +223,14 @@ impl BrowserRegistry {
|
||||
"browser reconnect: replacing previous registration"
|
||||
);
|
||||
}
|
||||
tracing::info!(
|
||||
diagnostic = true,
|
||||
event = "browser.registry.insert",
|
||||
id = %client.id,
|
||||
generation = client.generation,
|
||||
previous_generation = guard.get(&client.id).map(|previous| previous.generation),
|
||||
"diagnostic browser registry mutation"
|
||||
);
|
||||
guard.insert(client.id.clone(), client);
|
||||
}
|
||||
|
||||
@@ -239,16 +247,31 @@ impl BrowserRegistry {
|
||||
/// took over the same `id` while the old socket was tearing down
|
||||
/// (review M4/M5 round 2 Important #1). Returns the removed entry
|
||||
/// when a removal actually happened.
|
||||
#[track_caller]
|
||||
pub fn remove_if_generation_matches(
|
||||
&self,
|
||||
id: &BrowserId,
|
||||
generation: u64,
|
||||
) -> Option<std::sync::Arc<BrowserClient>> {
|
||||
let caller = std::panic::Location::caller();
|
||||
let mut guard = self.inner.lock().expect("browser registry poisoned");
|
||||
match guard.get(id) {
|
||||
let current_generation = guard.get(id).map(|current| current.generation);
|
||||
let removed = match guard.get(id) {
|
||||
Some(current) if current.generation == generation => guard.remove(id),
|
||||
_ => None,
|
||||
}
|
||||
};
|
||||
tracing::info!(
|
||||
diagnostic = true,
|
||||
event = "browser.registry.remove_if_generation_matches",
|
||||
id = %id,
|
||||
requested_generation = generation,
|
||||
?current_generation,
|
||||
removed = removed.is_some(),
|
||||
caller_file = caller.file(),
|
||||
caller_line = caller.line(),
|
||||
"diagnostic browser registry mutation"
|
||||
);
|
||||
removed
|
||||
}
|
||||
|
||||
pub fn get(&self, id: &BrowserId) -> Option<std::sync::Arc<BrowserClient>> {
|
||||
|
||||
@@ -37,7 +37,7 @@ use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use tokio::sync::Notify;
|
||||
use tokio::task::JoinHandle;
|
||||
use tracing::{debug, warn};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use super::abort::AbortRegistry;
|
||||
use super::queue::{DEFAULT_TOOL_TIMEOUT, DispatchError};
|
||||
@@ -197,7 +197,22 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
||||
let status = status.clone();
|
||||
let state = Arc::clone(&state);
|
||||
Box::pin(async move {
|
||||
match method {
|
||||
let method_name = format!("{method:?}");
|
||||
let diagnostic_rpc_id = rpc_id.clone();
|
||||
let session_id = params
|
||||
.get("session_id")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_owned);
|
||||
let started = Instant::now();
|
||||
info!(
|
||||
diagnostic = true,
|
||||
event = "ipc.rpc.started",
|
||||
rpc_id = %diagnostic_rpc_id,
|
||||
method = %method_name,
|
||||
session = session_id.as_deref().unwrap_or(""),
|
||||
"diagnostic IPC RPC"
|
||||
);
|
||||
let body = match method {
|
||||
Method::SystemPing => {
|
||||
let result = PingResult { pong: true };
|
||||
ResponseBody::Ok(serde_json::to_value(result).unwrap_or(Value::Null))
|
||||
@@ -261,7 +276,23 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
||||
message: format!("method not implemented yet: {other:?}"),
|
||||
data: None,
|
||||
}),
|
||||
}
|
||||
};
|
||||
let (outcome, error_code) = match &body {
|
||||
ResponseBody::Ok(_) => ("ok", String::new()),
|
||||
ResponseBody::Err(error) => ("error", format!("{:?}", error.code)),
|
||||
};
|
||||
info!(
|
||||
diagnostic = true,
|
||||
event = "ipc.rpc.completed",
|
||||
rpc_id = %diagnostic_rpc_id,
|
||||
method = %method_name,
|
||||
session = session_id.as_deref().unwrap_or(""),
|
||||
outcome,
|
||||
error_code = %error_code,
|
||||
elapsed_ms = started.elapsed().as_millis() as u64,
|
||||
"diagnostic IPC RPC"
|
||||
);
|
||||
body
|
||||
})
|
||||
})
|
||||
}
|
||||
@@ -465,8 +496,14 @@ fn handle_cancel(state: &Arc<DaemonState>, params: Value) -> ResponseBody {
|
||||
};
|
||||
let local = state.abort_registry.cancel(¶ms.rpc_id);
|
||||
let mut cancelled = local;
|
||||
let mut surface = if local { "abort_registry" } else { "none" };
|
||||
if !local && let Some(snap) = state.tool_inflight.cancel(¶ms.rpc_id) {
|
||||
cancelled = true;
|
||||
surface = if snap.ws_rpc_id.is_some() {
|
||||
"tool_inflight_forwarded"
|
||||
} else {
|
||||
"tool_inflight_queued"
|
||||
};
|
||||
// Forward the WS-side cancel only when the worker has
|
||||
// already promoted the entry to "forwarded"; queued entries
|
||||
// short-circuit on their own pre-flight without ever
|
||||
@@ -484,6 +521,14 @@ fn handle_cancel(state: &Arc<DaemonState>, params: Value) -> ResponseBody {
|
||||
);
|
||||
}
|
||||
}
|
||||
info!(
|
||||
diagnostic = true,
|
||||
event = "ipc.cancel.resolved",
|
||||
target_rpc_id = %params.rpc_id,
|
||||
cancelled,
|
||||
surface,
|
||||
"diagnostic IPC cancellation"
|
||||
);
|
||||
ResponseBody::Ok(serde_json::to_value(CancelResult { cancelled }).unwrap_or(Value::Null))
|
||||
}
|
||||
|
||||
|
||||
@@ -41,7 +41,7 @@ use bsk_protocol::{ErrorCode, Frame, Method, RequestFrame, ResponseBody, RpcErro
|
||||
use rand::Rng;
|
||||
use serde_json::Value;
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
use tracing::{debug, warn};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use super::abort::AbortToken;
|
||||
use super::browsers::BrowserRegistry;
|
||||
@@ -515,6 +515,16 @@ async fn forward_one(
|
||||
sessions: &Arc<SessionRegistry>,
|
||||
job: &ToolJob,
|
||||
) -> Result<Value, RpcError> {
|
||||
let started = std::time::Instant::now();
|
||||
info!(
|
||||
diagnostic = true,
|
||||
event = "tool.dispatch.started",
|
||||
session = %sid,
|
||||
method = ?job.method,
|
||||
has_inflight = job.inflight.is_some(),
|
||||
has_lifecycle_cancel = job.lifecycle_cancel.is_some(),
|
||||
"diagnostic tool dispatch"
|
||||
);
|
||||
// Pre-flight: a cancel that landed while this job was still in
|
||||
// the per-session channel must short-circuit before any session
|
||||
// / browser resolution work, and before any WS frame leaves the
|
||||
@@ -535,6 +545,13 @@ async fn forward_one(
|
||||
));
|
||||
}
|
||||
let Some(session) = sessions.get(sid) else {
|
||||
warn!(
|
||||
diagnostic = true,
|
||||
event = "tool.dispatch.session_missing",
|
||||
session = %sid,
|
||||
method = ?job.method,
|
||||
"diagnostic tool dispatch"
|
||||
);
|
||||
return Err(RpcError {
|
||||
code: ErrorCode::NotFound,
|
||||
message: format!("session {sid} no longer exists"),
|
||||
@@ -542,6 +559,14 @@ async fn forward_one(
|
||||
});
|
||||
};
|
||||
let Some(client) = browsers.get(&session.browser_id) else {
|
||||
warn!(
|
||||
diagnostic = true,
|
||||
event = "tool.dispatch.browser_missing",
|
||||
session = %sid,
|
||||
method = ?job.method,
|
||||
browser = %session.browser_id,
|
||||
"diagnostic tool dispatch"
|
||||
);
|
||||
return Err(RpcError {
|
||||
code: ErrorCode::NotFound,
|
||||
message: "owning browser is no longer connected".into(),
|
||||
@@ -558,6 +583,16 @@ async fn forward_one(
|
||||
method: job.method.clone(),
|
||||
params: Some(job.params.clone()),
|
||||
});
|
||||
info!(
|
||||
diagnostic = true,
|
||||
event = "tool.dispatch.forwarding",
|
||||
session = %sid,
|
||||
method = ?job.method,
|
||||
browser = %session.browser_id,
|
||||
browser_generation = client.generation,
|
||||
ws_rpc_id = %rpc_id,
|
||||
"diagnostic tool dispatch"
|
||||
);
|
||||
// Promote the inflight entry to "forwarded" AND push the WS
|
||||
// request frame to the sink inside the same critical section
|
||||
// (review round 2 C1). The closure runs while the entry's inner
|
||||
@@ -717,6 +752,18 @@ async fn forward_one(
|
||||
});
|
||||
}
|
||||
};
|
||||
info!(
|
||||
diagnostic = true,
|
||||
event = "tool.dispatch.response",
|
||||
session = %sid,
|
||||
method = ?job.method,
|
||||
browser = %session.browser_id,
|
||||
browser_generation = client.generation,
|
||||
ws_rpc_id = %rpc_id,
|
||||
elapsed_ms = started.elapsed().as_millis() as u64,
|
||||
response_is_error = matches!(&response.body, ResponseBody::Err(_)),
|
||||
"diagnostic tool dispatch"
|
||||
);
|
||||
match response.body {
|
||||
ResponseBody::Ok(v) => Ok(v),
|
||||
ResponseBody::Err(err) => Err(err),
|
||||
|
||||
@@ -105,12 +105,22 @@ impl SessionRegistry {
|
||||
|
||||
pub fn insert(&self, session: Session) {
|
||||
let session_id = session.id.clone();
|
||||
let browser_id = session.browser_id.clone();
|
||||
let agent_window_id = session.agent_window_id;
|
||||
let mut sessions = self.inner.lock().expect("session registry poisoned");
|
||||
sessions.insert(session_id.clone(), session);
|
||||
self.last_activity
|
||||
.lock()
|
||||
.expect("session activity registry poisoned")
|
||||
.insert(session_id, Instant::now());
|
||||
.insert(session_id.clone(), Instant::now());
|
||||
tracing::info!(
|
||||
diagnostic = true,
|
||||
event = "session.registry.insert",
|
||||
session = %session_id,
|
||||
browser = %browser_id,
|
||||
?agent_window_id,
|
||||
"diagnostic session registry mutation"
|
||||
);
|
||||
}
|
||||
|
||||
/// Reserve a fresh, collision-free [`SessionId`] under the registry
|
||||
@@ -152,6 +162,13 @@ impl SessionRegistry {
|
||||
.lock()
|
||||
.expect("session activity registry poisoned")
|
||||
.insert(candidate.clone(), Instant::now());
|
||||
tracing::info!(
|
||||
diagnostic = true,
|
||||
event = "session.registry.reserve",
|
||||
session = %candidate,
|
||||
browser = %browser_id,
|
||||
"diagnostic session registry mutation"
|
||||
);
|
||||
return Some(candidate);
|
||||
}
|
||||
None
|
||||
@@ -169,13 +186,24 @@ impl SessionRegistry {
|
||||
let mut guard = self.inner.lock().expect("session registry poisoned");
|
||||
let session = guard.get_mut(session_id)?;
|
||||
session.agent_window_id = agent_window_id;
|
||||
tracing::info!(
|
||||
diagnostic = true,
|
||||
event = "session.registry.commit",
|
||||
session = %session_id,
|
||||
browser = %session.browser_id,
|
||||
?agent_window_id,
|
||||
"diagnostic session registry mutation"
|
||||
);
|
||||
Some(session.clone())
|
||||
}
|
||||
|
||||
/// Drop a placeholder reservation, used on extension error/timeout
|
||||
/// paths so failed reservations do not accumulate.
|
||||
#[track_caller]
|
||||
pub fn cancel_reservation(&self, session_id: &SessionId) {
|
||||
self.inner
|
||||
let caller = std::panic::Location::caller();
|
||||
let removed = self
|
||||
.inner
|
||||
.lock()
|
||||
.expect("session registry poisoned")
|
||||
.remove(session_id);
|
||||
@@ -183,9 +211,20 @@ impl SessionRegistry {
|
||||
.lock()
|
||||
.expect("session activity registry poisoned")
|
||||
.remove(session_id);
|
||||
tracing::info!(
|
||||
diagnostic = true,
|
||||
event = "session.registry.cancel_reservation",
|
||||
session = %session_id,
|
||||
existed = removed.is_some(),
|
||||
caller_file = caller.file(),
|
||||
caller_line = caller.line(),
|
||||
"diagnostic session registry mutation"
|
||||
);
|
||||
}
|
||||
|
||||
#[track_caller]
|
||||
pub fn remove(&self, id: &SessionId) -> Option<Session> {
|
||||
let caller = std::panic::Location::caller();
|
||||
let removed = self
|
||||
.inner
|
||||
.lock()
|
||||
@@ -195,6 +234,17 @@ impl SessionRegistry {
|
||||
.lock()
|
||||
.expect("session activity registry poisoned")
|
||||
.remove(id);
|
||||
tracing::info!(
|
||||
diagnostic = true,
|
||||
event = "session.registry.remove",
|
||||
session = %id,
|
||||
existed = removed.is_some(),
|
||||
browser = removed.as_ref().map(|session| session.browser_id.0.as_str()),
|
||||
agent_window_id = removed.as_ref().and_then(|session| session.agent_window_id),
|
||||
caller_file = caller.file(),
|
||||
caller_line = caller.line(),
|
||||
"diagnostic session registry mutation"
|
||||
);
|
||||
removed
|
||||
}
|
||||
|
||||
@@ -243,7 +293,9 @@ impl SessionRegistry {
|
||||
}
|
||||
|
||||
/// Drop all sessions owned by `browser_id` (e.g. on disconnect).
|
||||
#[track_caller]
|
||||
pub fn purge_browser(&self, browser_id: &BrowserId) -> Vec<Session> {
|
||||
let caller = std::panic::Location::caller();
|
||||
let mut guard = self.inner.lock().expect("session registry poisoned");
|
||||
let drained: Vec<Session> = guard
|
||||
.values()
|
||||
@@ -260,6 +312,20 @@ impl SessionRegistry {
|
||||
for session in &drained {
|
||||
activity.remove(&session.id);
|
||||
}
|
||||
let session_ids: Vec<&str> = drained
|
||||
.iter()
|
||||
.map(|session| session.id.0.as_str())
|
||||
.collect();
|
||||
tracing::info!(
|
||||
diagnostic = true,
|
||||
event = "session.registry.purge_browser",
|
||||
browser = %browser_id,
|
||||
sessions = ?session_ids,
|
||||
count = drained.len(),
|
||||
caller_file = caller.file(),
|
||||
caller_line = caller.line(),
|
||||
"diagnostic session registry mutation"
|
||||
);
|
||||
drained
|
||||
}
|
||||
|
||||
|
||||
@@ -40,6 +40,8 @@ pub(crate) const DAEMONIZED_ENV: &str = "BSK_DAEMONIZED";
|
||||
/// daemon lock, IPC socket, and WS port — before taking over.
|
||||
pub(crate) const DAEMON_REPLACEMENT_WAIT_ENV: &str = "BSK_DAEMON_REPLACES_PID";
|
||||
|
||||
const LUMI_DIAGNOSTIC_BUILD: &str = "lumi-session-loss-v2";
|
||||
|
||||
/// Concrete daemon configuration resolved from CLI flags / defaults.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DaemonConfig {
|
||||
@@ -251,6 +253,7 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
|
||||
pid = info.pid,
|
||||
ws_port = info.ws_port,
|
||||
sock = %sock_path.display(),
|
||||
diagnostic_build = LUMI_DIAGNOSTIC_BUILD,
|
||||
"daemon ready"
|
||||
);
|
||||
|
||||
@@ -588,7 +591,9 @@ pub(crate) fn spawn_update_check_task(
|
||||
// The session gate is read after the fetch, as late
|
||||
// as possible before the binary gets replaced.
|
||||
let active_sessions = state.sessions.len();
|
||||
let auto_update = update::auto_update_enabled() && exe_path.is_some();
|
||||
// Keep this local diagnostic build in place until a
|
||||
// reproduction has uploaded its extension-side evidence.
|
||||
let auto_update = false;
|
||||
update::auto_update_step(
|
||||
candidate.as_ref(),
|
||||
auto_update,
|
||||
@@ -732,7 +737,11 @@ fn init_tracing() -> Option<tracing_appender::non_blocking::WorkerGuard> {
|
||||
use tracing_subscriber::prelude::*;
|
||||
|
||||
let log_dir = paths::log_dir().ok();
|
||||
let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
|
||||
let env_filter = EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new("info"))
|
||||
// Lumi commonly runs with RUST_LOG=warn. This local build must retain
|
||||
// BrowserSkill's INFO diagnostics even under that host-wide setting.
|
||||
.add_directive("bsk=info".parse().expect("valid diagnostic log directive"));
|
||||
|
||||
let (file_layer, guard) = if let Some(dir) = log_dir {
|
||||
let appender = tracing_appender::rolling::daily(&dir, "daemon.log");
|
||||
|
||||
@@ -233,6 +233,14 @@ async fn drive_connection(
|
||||
.ok_or_else(|| anyhow!("handshake missing params"))?;
|
||||
let params: HandshakeParams = serde_json::from_value(params_raw)
|
||||
.map_err(|err| anyhow!("invalid HandshakeParams: {err}"))?;
|
||||
info!(
|
||||
diagnostic = true,
|
||||
event = "browser.handshake.received",
|
||||
browser = %params.instance_id,
|
||||
extension_version = %params.version,
|
||||
extension_protocol = %params.protocol_version,
|
||||
"diagnostic browser handshake"
|
||||
);
|
||||
|
||||
// Version compatibility (design §10, M10.4): protocol_version only.
|
||||
let our_app_version: Version = env!("CARGO_PKG_VERSION")
|
||||
@@ -350,7 +358,7 @@ async fn drive_connection(
|
||||
// Pump loop: outbound frames from sink → ws; inbound from ws → resolve.
|
||||
let pump_state = Arc::clone(&state);
|
||||
let pump_browser = Arc::clone(&client);
|
||||
let result_outcome: anyhow::Result<()> = async {
|
||||
let result_outcome: anyhow::Result<String> = async {
|
||||
loop {
|
||||
tokio::select! {
|
||||
outbound = rx.recv() => {
|
||||
@@ -359,7 +367,7 @@ async fn drive_connection(
|
||||
let json = serde_json::to_string(&frame)?;
|
||||
writer.send(Message::Text(json)).await?;
|
||||
}
|
||||
None => break,
|
||||
None => break Ok("outbound_channel_closed".to_string()),
|
||||
}
|
||||
}
|
||||
msg = reader.next() => {
|
||||
@@ -381,18 +389,37 @@ async fn drive_connection(
|
||||
pump_browser.touch();
|
||||
}
|
||||
Some(Ok(Message::Frame(_))) => {}
|
||||
Some(Ok(Message::Close(_))) | None => break,
|
||||
Some(Ok(Message::Close(frame))) => {
|
||||
let reason = frame
|
||||
.map(|frame| format!("peer_close code={:?} reason={}", frame.code, frame.reason))
|
||||
.unwrap_or_else(|| "peer_close_without_frame".to_string());
|
||||
break Ok(reason);
|
||||
}
|
||||
None => break Ok("peer_eof".to_string()),
|
||||
Some(Err(err)) => {
|
||||
warn!(%err, "ws read error");
|
||||
break;
|
||||
break Ok(format!("read_error: {err}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
let pump_exit_reason = match &result_outcome {
|
||||
Ok(reason) => reason.clone(),
|
||||
Err(err) => format!("pump_error: {err:#}"),
|
||||
};
|
||||
info!(
|
||||
diagnostic = true,
|
||||
event = "browser.websocket.pump_exit",
|
||||
id = %browser_id,
|
||||
generation,
|
||||
reason = %pump_exit_reason,
|
||||
last_seen_idle_ms = client.idle_for().as_millis() as u64,
|
||||
session_ids = ?state.sessions.ids_for_browser(&browser_id),
|
||||
"diagnostic browser websocket exit"
|
||||
);
|
||||
|
||||
// Cleanup: drop browser + purge its sessions, but only if the
|
||||
// registry still holds *this* generation. If a reconnect already
|
||||
@@ -416,7 +443,7 @@ async fn drive_connection(
|
||||
"browser disconnected; newer reconnect already owns this id, leaving sessions intact"
|
||||
);
|
||||
}
|
||||
result_outcome
|
||||
result_outcome.map(|_| ())
|
||||
}
|
||||
|
||||
async fn handle_inbound_text(state: &Arc<DaemonState>, client: &Arc<BrowserClient>, text: &str) {
|
||||
@@ -442,6 +469,38 @@ async fn handle_inbound_text(state: &Arc<DaemonState>, client: &Arc<BrowserClien
|
||||
client.mark_heartbeat_seen();
|
||||
debug!(id = %client.id, "heartbeat");
|
||||
}
|
||||
bsk_protocol::EventKind::BrowserDiagnostic => {
|
||||
let entries = ev.payload.get("entries").and_then(|value| value.as_array());
|
||||
info!(
|
||||
diagnostic = true,
|
||||
event = "extension.diagnostic.batch",
|
||||
browser = %client.id,
|
||||
generation = client.generation,
|
||||
worker_boot_id = ev.payload.get("worker_boot_id").and_then(|value| value.as_str()).unwrap_or("unknown"),
|
||||
count = entries.map_or(0, Vec::len),
|
||||
"received extension diagnostic batch"
|
||||
);
|
||||
if let Some(entries) = entries {
|
||||
for entry in entries {
|
||||
info!(
|
||||
diagnostic = true,
|
||||
event = "extension.diagnostic.entry",
|
||||
browser = %client.id,
|
||||
generation = client.generation,
|
||||
entry = %entry,
|
||||
"extension diagnostic"
|
||||
);
|
||||
}
|
||||
} else {
|
||||
warn!(
|
||||
diagnostic = true,
|
||||
event = "extension.diagnostic.invalid",
|
||||
browser = %client.id,
|
||||
payload = %ev.payload,
|
||||
"extension diagnostic payload missing entries"
|
||||
);
|
||||
}
|
||||
}
|
||||
bsk_protocol::EventKind::SessionWindowClosed => {
|
||||
handle_session_window_closed(state, &client.id, &ev.payload);
|
||||
}
|
||||
|
||||
@@ -58,6 +58,10 @@ pub enum EventKind {
|
||||
BrowserDisconnected,
|
||||
#[serde(rename = "browser.connected")]
|
||||
BrowserConnected,
|
||||
/// Structured diagnostics emitted by locally instrumented extension
|
||||
/// builds. Production peers may ignore this event safely.
|
||||
#[serde(rename = "browser.diagnostic")]
|
||||
BrowserDiagnostic,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
@@ -256,6 +260,27 @@ mod tests {
|
||||
assert_eq!(frame.event, EventKind::SystemHeartbeat);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn browser_diagnostic_event_frame_round_trips_from_extension_shape() {
|
||||
let wire = serde_json::json!({
|
||||
"event": "browser.diagnostic",
|
||||
"payload": {
|
||||
"schema_version": 1,
|
||||
"worker_boot_id": "worker-1",
|
||||
"entries": [{ "event": "extension.worker.started" }]
|
||||
}
|
||||
});
|
||||
let frame: EventFrame = serde_json::from_value(wire).unwrap();
|
||||
assert_eq!(frame.event, EventKind::BrowserDiagnostic);
|
||||
assert_eq!(
|
||||
frame
|
||||
.payload
|
||||
.get("worker_boot_id")
|
||||
.and_then(|value| value.as_str()),
|
||||
Some("worker-1")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_user_interrupt_serialises_as_snake_case() {
|
||||
let v = serde_json::to_value(EventKind::SessionUserInterrupt).unwrap();
|
||||
|
||||
@@ -5,99 +5,142 @@ description: Browser automation through six injected domain tools.
|
||||
|
||||
# browser-skill for DeepSeek Harness
|
||||
|
||||
Drive the user's logged-in Chromium through this plugin's structured browser tools. Automation is
|
||||
isolated in an Agent Window; user-window tabs remain protected unless explicitly borrowed.
|
||||
Drive the user's real Chromium browser through the six structured domain tools provided by this
|
||||
plugin. The browser keeps the user's existing logins and cookies while isolating automation in an
|
||||
Agent Window. Every call must include the tool's `action` field; other fields depend on that action.
|
||||
|
||||
Loading this skill reveals six tools for the rest of the conversation:
|
||||
## Tool availability
|
||||
|
||||
- `browser_session` owns session lifecycle.
|
||||
- `browser_page` handles navigation and lifecycle waits.
|
||||
- `browser_inspect` reads semantic, visual, console, and network state.
|
||||
- `browser_interact` performs normal page interactions.
|
||||
- `browser_tabs` manages Agent Window tabs and temporary user-tab borrowing.
|
||||
- `browser_assist` handles human help, window size, and device emulation.
|
||||
Loading this skill reveals the browser tools for the rest of the conversation. All browser work
|
||||
must use the injected tools directly. Do not invoke another process to control the browser. Direct
|
||||
tool calls preserve session ownership, the live observation window, cancellation, attachments, and
|
||||
cleanup.
|
||||
|
||||
Every call includes an `action`. Treat each loaded tool schema as authoritative for its actions and
|
||||
parameters; do not guess fields. All browser work must use the injected tools directly so session
|
||||
ownership, cancellation, attachments, observation UI, and cleanup remain intact. Do not invoke
|
||||
another process to control the browser.
|
||||
The six tools are:
|
||||
|
||||
- `browser_session`: start, stop, or list plugin-owned sessions.
|
||||
- `browser_page`: navigate, back, forward, reload, or wait for navigation.
|
||||
- `browser_inspect`: observe, snapshot, html, screenshot, console, or network.
|
||||
- `browser_interact`: click, hover, fill, select, or press.
|
||||
- `browser_tabs`: list, create, select, close, borrow, or return tabs.
|
||||
- `browser_assist`: resize the Agent Window, emulate a device, or request help from the user.
|
||||
|
||||
## Mandatory workflow
|
||||
|
||||
Every task owns a bounded plugin session:
|
||||
Every browser task follows this lifecycle:
|
||||
|
||||
1. Call `browser_session` with `action: "start"` and retain its returned `sessionId`.
|
||||
2. Use that session for every later call. Pass `session` explicitly when more than one exists.
|
||||
3. Call `browser_session` with `action: "stop"` when the goal is reached or an error ends the task.
|
||||
Treat cleanup like a finally path unless the user explicitly asks to keep the session open.
|
||||
|
||||
Conceptual sequence:
|
||||
|
||||
```text
|
||||
browser_session({ action: "start", ... })
|
||||
... use the returned sessionId for browser work ...
|
||||
browser_session({ action: "start", url: "https://example.com" })
|
||||
browser_inspect({ action: "observe", session: sessionId })
|
||||
browser_interact({ action: "click", session: sessionId, target: "@e4" })
|
||||
browser_inspect({ action: "observe", session: sessionId })
|
||||
browser_session({ action: "stop", session: sessionId })
|
||||
```
|
||||
|
||||
Pass the session explicitly when more than one exists. Never guess or reuse an id owned by another
|
||||
program. Stop in a finally-style path on success and failure unless the user explicitly asks to keep
|
||||
the session open. Stopping also returns borrowed tabs.
|
||||
The plugin owns only sessions it starts. Never guess or reuse a session id from another program.
|
||||
When `session` is omitted, most actions use the current owned session, but explicit ids are safer.
|
||||
|
||||
## Work toward one observable goal
|
||||
## Stop when the goal is met
|
||||
|
||||
- Derive a concrete success condition from the user's request.
|
||||
- Take the shortest purposeful path: observe, act, then make at most one observation to confirm an
|
||||
ambiguous result.
|
||||
- Once success is visible, do not click, refresh, navigate, switch tabs, or perform extra checks.
|
||||
- If a human-only step appears or two attempts make no progress, request help instead of
|
||||
brute-forcing.
|
||||
Browser work is bounded. Define an observable success condition, take the shortest path, and stop
|
||||
as soon as success is visible. Do not refresh or click after success. If an action fails twice
|
||||
without progress, pause instead of brute-forcing. If state is ambiguous, make at most one fresh
|
||||
observation and then ask the user.
|
||||
|
||||
## Observe, act, observe
|
||||
## Observe and interact
|
||||
|
||||
Use `browser_inspect` action `observe` as the primary semantic page view. It returns roles, states,
|
||||
text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may
|
||||
also become stale after large DOM changes, so observe again before the next interaction.
|
||||
Use `browser_inspect` with `action: "observe"` as the primary semantic page view. It returns roles,
|
||||
states, text, and `@eN` refs. Use `action: "snapshot"` when a static accessibility tree is more useful.
|
||||
Prefer a fresh ref over a raw CSS selector for interactions.
|
||||
|
||||
Use `browser_interact` for click, hover, fill, select, and key actions. An observation marks a
|
||||
hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are
|
||||
labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref.
|
||||
Do not click the trigger itself unless the user wants the trigger's action.
|
||||
Refs invalidate after navigation and may become stale after large DOM changes. Observe again after
|
||||
navigation or a meaningful DOM update before using another ref. If an observation identifies a
|
||||
hover-triggered surface, hover the trigger, observe again, then interact with the revealed control.
|
||||
|
||||
Escalate reading only as needed:
|
||||
Use `browser_interact` as follows:
|
||||
|
||||
1. `observe` for normal understanding and interaction refs.
|
||||
2. `snapshot` when a stricter static accessibility tree is more useful.
|
||||
3. `html` for exact markup or hidden metadata that semantic views cannot provide.
|
||||
4. `screenshot` for layout, styling, canvas, images, or requested visual evidence.
|
||||
- `action: "click"`: requires `target`; optionally accepts `button` and `clickCount`.
|
||||
- `action: "hover"`: requires `target`; optionally accepts `modifiers`, `settleMs`, and `timeoutMs`.
|
||||
- `action: "fill"`: requires `target` and `value`; `noClear` appends instead of replacing.
|
||||
- `action: "select"`: requires `target` and a non-empty `values` array. Values are option value
|
||||
attributes, not necessarily the visible labels.
|
||||
- `action: "press"`: requires `key`; optionally focus `target` first and set `holdMs`.
|
||||
|
||||
Do not start with raw HTML or screenshots merely to discover ordinary controls. When interaction is
|
||||
needed, obtain a fresh observation before acting on screenshot or HTML findings.
|
||||
## Navigate and wait
|
||||
|
||||
Use `browser_page` for purposeful navigation, history, reload, or a lifecycle wait. Avoid speculative
|
||||
waits when no navigation is expected. After any page change, discard old refs and observe again.
|
||||
Use `browser_page` with these actions:
|
||||
|
||||
## Respect the Agent Window boundary
|
||||
- `navigate` requires `url` and optionally accepts `waitUntil` and `timeoutMs`.
|
||||
- `back` and `forward` move one history entry and optionally accept lifecycle wait fields.
|
||||
- `reload` refreshes the current tab; set `hard: true` only when bypassing cache is necessary.
|
||||
- `wait` waits for an expected navigation after an interaction that did not already wait.
|
||||
|
||||
Use `browser_tabs` to list returned tab ids before selecting, closing, borrowing, or returning tabs.
|
||||
Borrow a user tab only for the immediate task, and return it as soon as that step is complete. Never
|
||||
invent a tab id or keep a personal tab borrowed across unrelated work.
|
||||
Avoid speculative waits when no navigation is expected. After any page change, discard old refs and
|
||||
observe again.
|
||||
|
||||
## Ask the human when needed
|
||||
## Reading priority
|
||||
|
||||
Use `browser_assist` action `request-help` for login, captcha, OTP, payment confirmation, consent, or
|
||||
another step the user must complete. Give a precise prompt and highlight fresh targets when concrete
|
||||
controls are involved. Use completion criteria only for a clear stable success signal.
|
||||
Escalate page reading only as needed:
|
||||
|
||||
Resume only after the user continues or the criteria complete. Treat cancellation as rejection and
|
||||
timeout as a blocker rather than retrying. Observe again after control returns before reasoning about
|
||||
the new state or using refs.
|
||||
1. `browser_inspect` with `action: "observe"` for normal understanding and refs.
|
||||
2. The same tool with `action: "snapshot"` for a stricter static tree.
|
||||
3. `action: "html"` for exact markup that semantic views cannot provide; keep `maxBytes` bounded and
|
||||
use only a fresh ref when scoping a subtree.
|
||||
4. `action: "screenshot"` for layout, styling, canvas, or genuinely visual evidence. A fresh `ref`
|
||||
crops to an element; omit it for the visible tab.
|
||||
|
||||
The same tool can resize the Agent Window or emulate a device when the task requires visual or
|
||||
responsive testing. Emulation is scoped to one tab.
|
||||
Do not start with raw HTML or a screenshot merely to discover ordinary controls.
|
||||
|
||||
## Debug and recover without wandering
|
||||
## Tabs and the Agent Window boundary
|
||||
|
||||
Use `browser_inspect` console or network actions only for relevant, bounded, read-only diagnostics.
|
||||
Continue from returned sequence cursors instead of rereading the same buffer.
|
||||
Use `browser_tabs` with these actions:
|
||||
|
||||
- Stale ref: observe again and retry the intended action once.
|
||||
- Unknown tab: list tabs instead of guessing.
|
||||
- Unknown session: list owned sessions or start one; never try foreign ids.
|
||||
- Timeout: inspect current state before deciding whether one longer purposeful wait is useful.
|
||||
- Unrecoverable failure: report the blocker and stop the owned session.
|
||||
- `list` returns visible tabs; `scope: "user"` finds a user tab before borrowing.
|
||||
- `create` makes an Agent Window tab and optionally accepts `url`, `active`, and `index`.
|
||||
- `select` focuses an Agent Window tab and requires a returned `tabId`.
|
||||
- `close` closes an Agent Window tab and requires a returned `tabId`.
|
||||
- `borrow` moves a user tab into the Agent Window and requires a listed user `tabId`.
|
||||
- `return` restores a borrowed tab and requires that borrowed `tabId`.
|
||||
|
||||
Arbitrary page-script evaluation and interaction recording are intentionally unsupported. Do not
|
||||
invent tools or route around those limits.
|
||||
Return borrowed tabs as soon as the immediate task finishes. Stopping the session is only fallback
|
||||
cleanup. Never invent a tab id.
|
||||
|
||||
## Human help and display controls
|
||||
|
||||
Use `browser_assist` with `action: "request-help"` for login, captcha, OTP, payment confirmation,
|
||||
consent, or another human-only step. `prompt` is required. Optional `targets` highlight fresh refs or
|
||||
selectors. `completionCriteria` can detect explicit stable success through URL, selector, or text
|
||||
conditions. After the user continues or criteria complete, observe again before reasoning about the
|
||||
new state.
|
||||
|
||||
Use `action: "resize"` with `width` and `height` to resize the outer Agent Window. Use
|
||||
`action: "emulate"` with a device preset or explicit width and height; `mobile: true` also requires
|
||||
dimensions. Use `off: true` alone to clear emulation. Emulation is per tab.
|
||||
|
||||
## Read-only diagnostics
|
||||
|
||||
Use `browser_inspect` with `action: "console"` for buffered logs and exceptions, and
|
||||
`action: "network"` for response and failure metadata. Both accept `tabId`, `since`, `limit`, and
|
||||
`maxTextChars`; continue with the returned sequence cursor rather than rereading the same entries.
|
||||
Console additionally accepts `includeStack`.
|
||||
|
||||
Arbitrary page script evaluation and interaction recording are intentionally unsupported. Do not
|
||||
invent tool names or route around that limitation.
|
||||
|
||||
## Error handling and final checklist
|
||||
|
||||
On a stale ref, observe again and retry once. On an unknown tab, list tabs instead of guessing. On
|
||||
an unknown session, list owned sessions or start one; never try foreign ids. On timeout, decide
|
||||
whether one longer purposeful wait is justified, and never immediately repeat a failed history
|
||||
action without first checking page state.
|
||||
|
||||
Before finishing, verify that the observable goal is met or the blocker is clear, borrowed tabs are
|
||||
returned when practical, the owned session is stopped, and no extra page action occurs after
|
||||
success.
|
||||
|
||||
@@ -47,10 +47,7 @@ describe("registerBskSkill", () => {
|
||||
expect(content).toContain("Mandatory workflow");
|
||||
expect(content).toContain("Refs invalidate after navigation");
|
||||
expect(content).toContain("evaluation and interaction recording are intentionally unsupported");
|
||||
// Keep the lazily injected instructions inside a bounded prompt budget,
|
||||
// while the lower bound catches accidental truncation of the guidance.
|
||||
expect(content.length).toBeGreaterThan(3_000);
|
||||
expect(content.length).toBeLessThan(6_000);
|
||||
expect(content.length).toBeGreaterThan(6_000);
|
||||
expect(skill.source).toBe("bundled");
|
||||
// Source frontmatter is registration metadata and must not leak into the body.
|
||||
expect(content.startsWith("---")).toBe(false);
|
||||
|
||||
+283
-98
@@ -1,142 +1,327 @@
|
||||
---
|
||||
name: browser-skill
|
||||
description: |
|
||||
Use when the user asks to automate their logged-in Chromium browser: visit
|
||||
and read pages, fill forms, scrape data, click through flows, regression-test
|
||||
a PR's UI, validate a deployed page, or operate a tab they identify. Requires
|
||||
the bsk CLI and browser extension.
|
||||
Use when the user asks to perform browser automation tasks against their
|
||||
logged-in browser: visit and read pages, fill forms, scrape data, click
|
||||
through a flow, regression-test a PR's UI, validate a deployed page.
|
||||
Requires the bsk CLI installed and the browser-skill extension loaded.
|
||||
---
|
||||
|
||||
# browser-skill
|
||||
|
||||
Drive the user's real Chromium browser through `bsk`. Automation runs in an isolated **Agent
|
||||
Window** with the user's existing logins and cookies. User-window tabs remain protected unless they
|
||||
are explicitly borrowed.
|
||||
Drive the user's **real Chromium browser** (with their logins and cookies) through the `bsk` CLI. The extension opens an isolated **Agent Window** for automation; the user's normal windows stay protected unless you explicitly borrow a tab.
|
||||
|
||||
Do not use this skill for tasks with no browser, for extension installation, or when the user only
|
||||
wants instructions. Never extract credentials, cookies, tokens, or other secrets from pages.
|
||||
## When to use
|
||||
|
||||
## Required lifecycle
|
||||
- Open pages, read titles/text, scrape structured data from sites the user can already access
|
||||
- Fill forms, click through multi-step flows, smoke-test a UI change
|
||||
- Understand pages with `bsk snapshot` first; use `bsk get-html` or `bsk screenshot` only when the snapshot is insufficient
|
||||
- Operate on a specific user tab they point you at (after `bsk tab borrow`)
|
||||
|
||||
Every browser task owns a bounded session:
|
||||
## When NOT to use
|
||||
|
||||
```text
|
||||
1. bsk session start # retain the printed 4-letter session id
|
||||
2. bsk ... --session <id> # pass it to every session-scoped command
|
||||
3. bsk session stop <id> # always run on success and error paths
|
||||
- Tasks with **no browser** involved (files, APIs, databases only)
|
||||
- Installing or configuring the extension (point the user to setup docs instead)
|
||||
- **Credential harvesting** — never run `bsk evaluate` on banking, SSO, or password-manager pages to extract tokens, cookies, or secrets
|
||||
- Long-lived control of a user's personal login window — borrow only for the immediate step, then `bsk tab return` or end the session
|
||||
- Replacing the user's manual browsing when they only wanted an explanation
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. `bsk` on `PATH` (Rust CLI from browser-skill)
|
||||
2. browser-skill **extension** loaded in Chromium and connected (popup shows green, or an upgrade reminder — still connected; continue)
|
||||
3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails
|
||||
|
||||
## Mandatory workflow
|
||||
|
||||
Every automation task **must** follow this lifecycle. Do **not** rely on idle timeouts (default session idle is 5 minutes).
|
||||
|
||||
```
|
||||
1. bsk session start → capture the 4-letter session id printed on stdout
|
||||
2. … every tool command … → always pass --session <id>
|
||||
3. bsk session stop <id> → REQUIRED when done (even on error paths)
|
||||
```
|
||||
|
||||
Do not rely on the idle timeout for cleanup. Stop the session as soon as the goal is met unless the
|
||||
user explicitly asks to keep it open. Stopping also returns borrowed tabs.
|
||||
Optional: `bsk session start --browser <instance-id-or-label>` when multiple browsers are connected (`bsk browsers` / error output lists them). Add `--no-focus` to open the Agent Window in the background without stealing focus from the user's current window.
|
||||
|
||||
Any `bsk` command auto-starts the background services it needs; never manage the daemon by hand.
|
||||
When multiple browsers are connected, use `bsk browsers` and start with
|
||||
`bsk session start --browser <id-or-label>`. Add `--no-focus` to that same start command when the
|
||||
Agent Window does not need to interrupt the user's current work; it is not a flag on other commands.
|
||||
Run `bsk doctor` when startup or transport problems persist after one retry.
|
||||
Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**.
|
||||
|
||||
## Work toward one observable goal
|
||||
## Stop when the goal is met
|
||||
|
||||
- Derive a concrete success condition from the user's request or a supplied trace.
|
||||
- Take the shortest purposeful path: observe, act, then make at most one observation to confirm an
|
||||
ambiguous result.
|
||||
- Once success is visible, do not click, refresh, navigate, switch tabs, or perform extra checks.
|
||||
- If a human-only step appears or two attempts make no progress, request help instead of
|
||||
brute-forcing.
|
||||
Every task is a **bounded goal**, not open-ended browsing. The goal may come from the user's request, a recorded `trace.json`, or both.
|
||||
|
||||
With a trace, follow its semantic target information and values in order, but treat its refs as
|
||||
record-local hints. Stop when its purpose or last meaningful effect is satisfied. A trace guides the
|
||||
task; it does not expand the user's goal or authorize additional actions.
|
||||
1. **Define success first** — one concrete, observable condition derived from the user's words, `purpose`, or the last meaningful step in a trace (e.g. "form submitted", "item added to cart", "playback started").
|
||||
2. **Take the shortest path** — snapshot → act → at most one check. Do not wander, re-try unrelated actions, or stack exploratory steps.
|
||||
3. **Stop as soon as success is reached** — run `bsk session stop <id>` immediately unless the user explicitly asked to keep the session open (e.g. "don't close yet", "keep browsing").
|
||||
4. **No post-success work** — once the goal is met, do not click, refresh, navigate, re-search, switch tabs, or "double-check" that it worked. Further verification is a new task.
|
||||
5. **When blocked, pause — do not brute-force** — if the page requires human input (login, captcha, OTP, payment confirmation) or an action fails twice with no progress, call `bsk request-help` instead of retrying blindly. See **Ask the human for help** below.
|
||||
6. **When unsure** — at most one extra `bsk snapshot`. If success looks met, stop. If not, ask the user; do not keep clicking.
|
||||
|
||||
## Observe, act, observe
|
||||
**With a trace:** replay steps in order using `target` role/name/tag and raw `value`/`selection` fields. After the last step (or when its `effect.navigated_to` / success hint is satisfied), apply rules 3–4 immediately. The trace guides execution; it does not extend control beyond the goal.
|
||||
|
||||
Use this default loop:
|
||||
**Without a trace:** the user's request *is* the success condition. Satisfying it ends the task — same stop rules apply.
|
||||
|
||||
```text
|
||||
## Core interaction loop
|
||||
|
||||
Write operations only affect tabs in the **Agent Window** (or tabs you **borrowed** into it).
|
||||
|
||||
```
|
||||
bsk navigate <url> --session <id>
|
||||
bsk observe --session <id>
|
||||
bsk click|hover|fill|select|press ... --session <id>
|
||||
bsk observe --session <id> # after navigation or a meaningful DOM change
|
||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk observe --session <id> → again after navigation / DOM change
|
||||
```
|
||||
|
||||
Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also
|
||||
make them stale. Observe again before the next interaction.
|
||||
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
||||
|
||||
An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`.
|
||||
The listed items are labels, not usable refs: hover the trigger, observe again, then act on the
|
||||
revealed item's own ref. Do not click the trigger itself unless the user wants the trigger's action.
|
||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
||||
|
||||
Escalate page reading only as needed:
|
||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
||||
|
||||
1. `bsk observe` for normal semantic understanding, text, controls, and refs.
|
||||
2. `bsk snapshot` when a stricter static accessibility tree is more useful.
|
||||
3. `bsk get-html` for exact markup or hidden metadata that semantic views cannot provide.
|
||||
4. `bsk screenshot` for layout, styling, canvas, images, or requested visual evidence.
|
||||
## Observation priority
|
||||
|
||||
Do not start with raw HTML or screenshots merely to discover ordinary controls. When interaction is
|
||||
needed, obtain a fresh observation before acting on screenshot or HTML findings.
|
||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
|
||||
## Respect the Agent Window boundary
|
||||
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
||||
2. `bsk snapshot` — strict static accessibility tree fallback
|
||||
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
||||
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
||||
|
||||
Normal page writes affect only Agent Window tabs. To operate a user tab, first list it with
|
||||
`bsk tab list --scope user --session <id>`, then `bsk tab borrow <tab-id>`. Return it immediately
|
||||
after the relevant step with `bsk tab return <tab-id>`; never invent a tab id or keep a personal tab
|
||||
borrowed across unrelated work.
|
||||
Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page.
|
||||
|
||||
## Ask the human when needed
|
||||
## Sandbox rules
|
||||
|
||||
Use `bsk request-help` for login, captcha, OTP, payment confirmation, consent, or another step the
|
||||
user must complete. Give a precise prompt and pass fresh `--target` refs/selectors when concrete
|
||||
controls can be highlighted. Use completion criteria only when the page has a clear stable success
|
||||
signal.
|
||||
| Rule | Detail |
|
||||
|------|--------|
|
||||
| Agent Window | `bsk tab create`, `bsk navigate`, `bsk click`, etc. work on agent tabs by default |
|
||||
| User tabs | Read-only until borrowed: `bsk tab list --session <id> --scope user` then `bsk tab borrow <tab-id> --session <id>` |
|
||||
| Return borrowed tabs | Call `bsk tab return <tab-id> --session <id>` when finished; unreturned tabs are **auto-returned** on `bsk session stop` |
|
||||
| Writes off-agent | Commands that mutate the page fail if the tab is not in the Agent Window — borrow or create a tab first |
|
||||
|
||||
The result `outcome` is one of `continued`, `completed`, `cancelled`, `timed_out`, or `disabled`
|
||||
(`navigated` is deprecated — never treat navigation as a completion signal). Resume only after
|
||||
`continued` or `completed`. Treat `cancelled` as rejection, and `timed_out` or `disabled` as a
|
||||
blocker rather than a reason to retry. After control returns, run a fresh `bsk observe` before
|
||||
reasoning about the page or using refs.
|
||||
## Global flags
|
||||
|
||||
## Command inventory
|
||||
| Flag | Purpose |
|
||||
|------|---------|
|
||||
| `--json` | Machine-readable JSON on stdout (errors too) |
|
||||
| `--quiet` | Suppress informational stderr |
|
||||
| `-v` / `-vv` | More verbose logging |
|
||||
|
||||
This list of names is complete. Never invent a command outside it; read
|
||||
`bsk <command...> --help` for flags instead of guessing them.
|
||||
Auto-update is **on by default** — the background daemon upgrades `bsk` itself when a new release is available (postponed while a session is active). Set `BSK_AUTO_UPDATE=off` to disable it and upgrade manually with `bsk update`.
|
||||
|
||||
```text
|
||||
session start|stop|list browsers status doctor update logs
|
||||
navigate navigate-back navigate-forward reload wait-for-navigation wait-ms
|
||||
observe snapshot get-html screenshot console network
|
||||
click hover fill select press evaluate
|
||||
tab list|create|close|select|borrow|return window resize emulate
|
||||
request-help record start|stop
|
||||
Command-specific flags (timeouts, `--tab-id`, `--wait-until`, …): **`bsk <cmd> --help`**
|
||||
|
||||
## CLI command reference (one line each)
|
||||
|
||||
Details and flags: **`bsk <cmd> --help`**
|
||||
|
||||
### Diagnostics
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk status` | Connection health, connected browsers, active sessions |
|
||||
| `bsk doctor` | Deep diagnostics and repair hints |
|
||||
| `bsk browsers` | List connected browser instances (ids, labels, versions) |
|
||||
|
||||
### Session
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk session start` | Open Agent Window (`--width`/`--height` for initial size); prints **4-letter session id** |
|
||||
| `bsk session start --no-focus` | Open Agent Window in the background without stealing focus |
|
||||
| `bsk session stop <id>` | End session, close Agent Window, auto-return borrowed tabs |
|
||||
| `bsk session stop --all` | Stop every active session |
|
||||
| `bsk session list` | List active sessions |
|
||||
|
||||
### Window (require `--session <id>`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk window resize` | Resize the Agent Window (`--width`, `--height`; 100..=7680 CSS px) |
|
||||
|
||||
### Device emulation — `bsk emulate` (requires `--session <id>`)
|
||||
|
||||
Emulate a mobile device environment on the agent tab via CDP — viewport, User-Agent, and touch — to debug a page's mobile layout and behaviour:
|
||||
|
||||
```bash
|
||||
bsk emulate --session <id> --device iphone-14
|
||||
bsk emulate --session <id> --width 390 --height 844 --dpr 3 --mobile --ua "Mozilla/5.0 (iPhone…" --touch
|
||||
bsk emulate --session <id> --off
|
||||
```
|
||||
|
||||
Required flags that are easy to get wrong:
|
||||
- Presets (`--device`): `iphone-14`, `iphone-14-pro-max`, `iphone-se`, `pixel-7`, `galaxy-s23`, `ipad-mini`, `galaxy-tab-s8`. Manual flags (`--width`/`--height`/`--dpr`/`--mobile`/`--ua`/`--accept-language`/`--touch`/`--max-touch-points`) also work without a preset, or override individual preset fields; `--no-mobile`/`--no-touch` turn a preset's mobile viewport / touch emulation back off.
|
||||
- Repeated runs merge field by field onto the tab's current emulation state — only the flags you pass change. E.g. after `--device iphone-14`, `bsk emulate --session <id> --width 390 --height 844` keeps the preset's dpr (`3`) and mobile viewport. (The extension remembers the state per tab; after an extension reload the next run applies only the fields it carries.)
|
||||
- `--off` clears every override (viewport, UA, touch) and the remembered state, restoring the tab's real environment.
|
||||
- Scope: overrides apply to **one tab only** (CDP per-target) and are **not inherited by new tabs** — re-run `bsk emulate` after opening or switching to another tab (default target is the session's active tab; `--tab-id` overrides).
|
||||
- Emulation covers viewport/UA/touch only; it does not throttle the network, fake geolocation, or synthesise real touch-event streams.
|
||||
|
||||
```text
|
||||
bsk fill <ref> --value <text> bsk select <ref> --value <option-value>
|
||||
bsk screenshot --out <path> bsk emulate --device <preset-id>
|
||||
### Tabs (require `--session <id>`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk tab list` | List tabs (`--scope user\|agent\|all`, default `all`) |
|
||||
| `bsk tab create` | New tab in Agent Window (`--url`, `--no-active`, `--index`) |
|
||||
| `bsk tab close <tab-id>` | Close an agent tab |
|
||||
| `bsk tab select <tab-id>` | Focus an agent tab |
|
||||
| `bsk tab borrow <tab-id>` | Move a user tab into the Agent Window |
|
||||
| `bsk tab return <tab-id>` | Return a borrowed tab to its original window |
|
||||
|
||||
### Observation (require `--session` unless noted)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
|
||||
| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
|
||||
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
|
||||
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
|
||||
|
||||
### Console & network debugging (read-only; require `--session`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk console` | Buffered page console messages, JS exceptions, and browser log entries (`--include-stack` for stack traces) |
|
||||
| `bsk network` | Buffered network responses (status, method, URL, MIME/resource type) and failures (`net::ERR_*` reason) |
|
||||
|
||||
Both capture from the moment the tab is attached and read a bounded per-tab buffer: `--since <seq>` pages from a cursor (`next_since` in the result), `--limit` (default 50, max 200), `--max-text-chars` (default 1000, max 4096), `--tab-id` to target a non-active tab. Both are strictly read-only — they never intercept or modify traffic, and request/response headers, bodies, and timings are not captured.
|
||||
|
||||
### Navigation
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk navigate <url>` | Go to URL in agent tab (`--wait-until`, `--timeout`) |
|
||||
| `bsk navigate-back` | History back one step |
|
||||
| `bsk navigate-forward` | History forward one step |
|
||||
| `bsk reload` | Reload current tab (`--hard` bypass cache) |
|
||||
|
||||
(`bsk navigate back` / `bsk navigate forward` are equivalent subcommands.)
|
||||
|
||||
### Interaction
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||
|
||||
### Scripting & timing
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk evaluate <expression>` | Run JS in agent tab (see red lines). JS throw → stderr, **exit 0** (RPC success); use `--json` and check `.ok` to detect JS errors |
|
||||
| `bsk wait-for-navigation` | Block until load/DOM idle/etc. (`--wait-until`, `--timeout`) |
|
||||
| `bsk wait-ms <duration>` | Sleep (`500ms`, `2s`, `1m`; **no** `--session`) |
|
||||
|
||||
### Ask the human for help — `bsk request-help`
|
||||
|
||||
When a step needs a human (captcha, login, OTP) or you want the user to
|
||||
confirm an important action, pause and ask:
|
||||
|
||||
bsk request-help --session <id> --prompt "Solve the captcha, then click Done only after the site accepts it" \
|
||||
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
|
||||
|
||||
- `--prompt` (required): what the user should do.
|
||||
- `--title` (optional): custom title for the overlay panel. When omitted,
|
||||
the extension shows its default localized title.
|
||||
- `--target` (repeatable): a snapshot ref (`@e7`) or CSS selector
|
||||
(`#submit`) to scroll to and flash-highlight. **Strongly recommended** —
|
||||
whenever the prompt refers to a concrete element (a button to click, a
|
||||
field to fill, a checkbox to toggle), pass its `@eN` ref / selector so the
|
||||
user is guided straight to the right spot instead of hunting for it. For
|
||||
interaction scenarios, always include the relevant target(s); reserve a
|
||||
prompt with no `--target` for cases where there is genuinely no specific
|
||||
element to point at (e.g. "wait for the page to finish loading").
|
||||
- `--timeout` (default `5m`): how long to wait.
|
||||
- `--completion-criteria` (optional): JSON success detector. Use it only
|
||||
when there is a concrete post-help success signal, e.g.
|
||||
`{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`.
|
||||
|
||||
The target tab is brought to the foreground; the page stays interactive
|
||||
while the agent control mask is hidden. The call blocks until the user
|
||||
explicitly acts, the timeout expires, cancellation arrives, or explicit
|
||||
completion criteria match. Page reloads, SPA route changes, and captcha
|
||||
refreshes do not return control by themselves. The result `outcome` is one of:
|
||||
|
||||
- `continued` — the user finished and clicked Done / return control (treat as confirm).
|
||||
- `cancelled` — the user clicked Cancel (treat as reject/abort).
|
||||
- `timed_out` — nobody acted within the timeout.
|
||||
- `completed` — the explicit `--completion-criteria` matched while the user had control.
|
||||
- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal.
|
||||
|
||||
`note` carries any text the user typed back. `resolved_targets` reports
|
||||
which refs/selectors matched a live element.
|
||||
|
||||
`request-help` does not refresh the page model after the user returns
|
||||
control. After a `continued` or `completed` result, issue a separate
|
||||
observation tool call (usually `bsk snapshot --session <id>`) before using
|
||||
new refs or reasoning about the post-help page state.
|
||||
#### Disabling request-help (unattended mode)
|
||||
Set `BSK_REQUEST_HELP=off` on unattended servers: `bsk request-help` then
|
||||
returns immediately with `outcome="disabled"` (no overlay, no waiting,
|
||||
exit 0). Any other value keeps it enabled. If you get `disabled`, do not
|
||||
retry — complete the task autonomously or stop gracefully.
|
||||
|
||||
### Recording — `bsk record`
|
||||
|
||||
Capture the user's own actions in the Agent Window to a **trace bundle**, for later LLM-driven automation. New CLI builds request **trace v3** (page observations + action chain); older extensions may still return **trace v2** (actions only), which the CLI exports as a single `trace.json` without `states/`.
|
||||
|
||||
```bash
|
||||
bsk record start --browser <instance-id-or-label> \
|
||||
[--url https://…] [--purpose "publish a wiki doc"] \
|
||||
[--max-page-tokens 3000] [--redact-values] \
|
||||
[--output trace]
|
||||
# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
|
||||
# Blocks until the user clicks Finish in the recording panel, then writes:
|
||||
# trace/trace.json — action chain (+ state index when v3)
|
||||
# trace/states/ — v3 only: one `sN.txt` observe snapshot per settled page state
|
||||
|
||||
bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable
|
||||
```
|
||||
|
||||
`select` matches an option's `value` attribute, not its visible label. Device preset ids are
|
||||
lowercase and hyphenated, such as `iphone-14`.
|
||||
- **v3 bundle (preferred):** `--output` is a directory (default `./trace`) containing `trace.json` and `states/`. `trace.json` lists `states[]` (page observation ids) and `steps[]` (each step binds `state` = observe snapshot *before* the action and `result.state` = snapshot *after* settle). Page bodies live in `states/sN.txt` using the same VOM format as `bsk observe`.
|
||||
- **v2 fallback:** when the connected extension is older, export may contain only legacy `trace.json` with `pages[]` action context and no `states/` observation files. Update the extension for full v3 bundles.
|
||||
- Each `states[]` entry is one **settled page observation** (captured after recording start, navigation landing, or action settle — not on a timer).
|
||||
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
|
||||
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
||||
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
|
||||
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
||||
|
||||
- `console` and `network` provide bounded, read-only debugging evidence.
|
||||
- `emulate` applies viewport, user-agent, and touch overrides to one tab; new tabs do not inherit
|
||||
them. Use `--off` to restore the real environment.
|
||||
- `evaluate` is a last resort when observe plus normal interactions cannot complete the task. With
|
||||
`--json`, inspect `.ok`: a JavaScript exception may still have CLI exit code 0 because the RPC
|
||||
succeeded. Never evaluate credential surfaces to read storage, cookies, or auth data.
|
||||
- `record` captures a user's actions for later replay. Read `bsk record start --help` before use,
|
||||
and never record banking, SSO, password-manager, or other sensitive pages.
|
||||
## Error handling
|
||||
|
||||
## Recover without wandering
|
||||
### Exit codes (`echo $?` after `bsk …`)
|
||||
|
||||
- Stale ref: observe again and retry the intended action once.
|
||||
- Unknown tab or session: list current tabs/sessions; never guess identifiers.
|
||||
- Timeout: inspect current page state before deciding whether one longer purposeful wait is useful.
|
||||
- Unsupported command: continue with available capabilities; suggest updating only when the missing
|
||||
command is necessary.
|
||||
- Unrecoverable failure: report the blocker and stop the session in a finally-style path.
|
||||
| Code | Meaning | What to do |
|
||||
|------|---------|------------|
|
||||
| `0` | Success (including `evaluate` where JS threw but RPC succeeded) | Continue; for `evaluate`, check `--json` `.ok` to distinguish JS success from RPC success |
|
||||
| `1` | User error — bad args, unknown session, tab not in Agent Window, stale ref | Fix args; `bsk session list`; re-snapshot |
|
||||
| `2` | Protocol / transport — service unreachable, IPC failure | `bsk doctor`; check extension connected; retry the command |
|
||||
| `3` | Browser / CDP execution failed | Retry; simplify selector; check tab still open |
|
||||
| `4` | Timeout | Increase `--timeout`; try `--wait-until domcontentloaded` |
|
||||
| `5` | Unknown RPC method | This command is not implemented in the current build. Continue the task with other commands. If a newly added command is missing, suggest `bsk update`. |
|
||||
|
||||
The CLI's current help and error hints are authoritative for flags, parameters, and recovery
|
||||
details.
|
||||
Human errors print `error:` + `hint:` on stderr; `--json` includes `code`, `message`, `hint`, `exit_code`.
|
||||
|
||||
### When to run diagnostics
|
||||
|
||||
| Situation | Command |
|
||||
|-----------|---------|
|
||||
| Before first task in a session | `bsk status` — extension connected? |
|
||||
| Any failure you cannot fix in one retry | `bsk doctor` |
|
||||
| Multiple browsers / wrong target | `bsk browsers` then `bsk session start --browser <id>` |
|
||||
|
||||
Always **`bsk session stop <id>`** in a `finally`-style path so the Agent Window closes and borrowed tabs return.
|
||||
|
||||
## Red lines
|
||||
|
||||
1. **No token theft** — do not `bsk evaluate` on sensitive sites to read `localStorage`, cookies, or auth headers for exfiltration.
|
||||
2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
|
||||
3. **No skip stop** — always `bsk session stop <id>`; never assume idle timeout will clean up.
|
||||
4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
|
||||
5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics or conditional surfaces help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual.
|
||||
6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
|
||||
|
||||
---
|
||||
|
||||
**More detail for any command:** `bsk <cmd> --help`
|
||||
|
||||
Reference in New Issue
Block a user