Compare commits

..
Author SHA1 Message Date
drakezhang 388c1283bc fix: clarify invalid selectors and dedupe diagnostics 2026-09-01 21:03:22 +08:00
drakezhang b449be1544 chore: add session-loss diagnostics 2026-09-01 20:16:32 +08:00
73 changed files with 1093 additions and 3388 deletions

No files matched your search

+61 -6
View File
@@ -1,6 +1,7 @@
import { i18n } from "@browser-skill/i18n";
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
import { ConnectionController } from "@/lib/connection-controller";
import { DiagnosticLogger } from "@/lib/diagnostics";
import { startHeartbeat } from "@/lib/heartbeat";
import {
getConnectionEnabled,
@@ -46,9 +47,20 @@ import type { Transport } from "@/transport/transport";
import { WSTransport } from "@/transport/ws-transport";
export default defineBackground(() => {
const controller = new ConnectionController();
const transport = new WSTransport({ url: __BSK_DAEMON_WS_URL__ });
const sessions = new SessionManager();
const diagnostics = new DiagnosticLogger();
diagnostics.record("extension.worker.started", {
diagnostic_build: "lumi-session-loss-v2",
extension_version: __BSK_EXT_VERSION__,
extension_runtime_id: chrome.runtime.id,
daemon_ws_url: __BSK_DAEMON_WS_URL__,
});
const controller = new ConnectionController(diagnostics.sink);
const transport = new WSTransport({
url: __BSK_DAEMON_WS_URL__,
diagnostics: diagnostics.sink,
});
diagnostics.bindTransport((frame) => transport.send(frame));
const sessions = new SessionManager({ diagnostics: diagnostics.sink });
const cdp = new ChromiumCdp();
const sessionsLive = attachSessionsLiveFlag({ manager: sessions });
let overlayGeneration = 0;
@@ -109,6 +121,9 @@ export default defineBackground(() => {
}
function onOverlaySessionStateChanged(): void {
diagnostics.record("background.sessions.changed", {
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
});
void sessionsLive.syncFromManager();
const liveSessionIds = new Set(sessions.list().map((ctx) => ctx.sessionId));
for (const sessionId of controlModes.keys()) {
@@ -146,6 +161,7 @@ export default defineBackground(() => {
onSessionsChanged: () => {
void sessionsLive.syncFromManager();
},
diagnostics: diagnostics.sink,
});
const recordDeps = {
tabsApi: chrome.tabs,
@@ -193,6 +209,7 @@ export default defineBackground(() => {
title: i18n.t("helpRequest.notificationTitle", { ns: "extension" }),
body: "",
}),
diagnostics: diagnostics.sink,
});
dispatcher.start();
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
@@ -225,6 +242,7 @@ export default defineBackground(() => {
transport,
cdp,
onSessionsChanged: onOverlaySessionStateChanged,
diagnostics: diagnostics.sink,
});
// MV3 service worker keepalive + reconnect supervisor (review M4/M5
@@ -236,6 +254,7 @@ export default defineBackground(() => {
startKeepalive({
transport,
shouldConnect: () => controller.isConnectionEnabled,
diagnostics: diagnostics.sink,
});
// Application-level heartbeat (Chrome 116+): while the post-handshake
@@ -249,6 +268,19 @@ export default defineBackground(() => {
controller.subscribe((snap) =>
cb(snap.state === "connected" || snap.state === "version_skew"),
),
diagnostics: diagnostics.sink,
});
controller.subscribe((snapshot) => {
const ready = snapshot.state === "connected" || snapshot.state === "version_skew";
diagnostics.setTransportReady(ready);
diagnostics.record("background.controller.snapshot", {
state: snapshot.state,
connection_enabled: snapshot.connectionEnabled,
instance_id: snapshot.instanceId,
last_error: snapshot.lastError,
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
});
});
// Wake-driven reconnect (best effort). An MV3 service worker is killed
@@ -257,7 +289,12 @@ export default defineBackground(() => {
// the worker and let us reconnect immediately instead of waiting for
// the next 30s alarm tick. They only help when the daemon is actually
// running; a cold daemon is (re)spawned by the next `bsk` command.
const reconnectIfNeeded = () => {
const reconnectIfNeeded = (source: string) => {
diagnostics.record("background.wake_reconnect.requested", {
source,
controller_enabled: controller.isConnectionEnabled,
transport_state: transport.state,
});
if (!controller.isConnectionEnabled) return;
if (transport.state === "connected") return;
void transport.connect().catch((err) => {
@@ -265,13 +302,14 @@ export default defineBackground(() => {
});
};
if (typeof chrome.runtime?.onStartup?.addListener === "function") {
chrome.runtime.onStartup.addListener(reconnectIfNeeded);
chrome.runtime.onStartup.addListener(() => reconnectIfNeeded("runtime.onStartup"));
}
if (typeof chrome.idle?.onStateChanged?.addListener === "function") {
chrome.idle.onStateChanged.addListener((state) => {
diagnostics.record("chrome.idle.state_changed", { state });
// "active" fires when the user returns from idle/locked — the most
// reliable "machine just woke" signal we get.
if (state === "active") reconnectIfNeeded();
if (state === "active") reconnectIfNeeded("idle.active");
});
// Treat >60s of no input as idle so the active transition is timely.
chrome.idle.setDetectionInterval?.(60);
@@ -279,14 +317,21 @@ export default defineBackground(() => {
void (async () => {
const connectionEnabled = await getConnectionEnabled();
diagnostics.record("background.controller.attach_started", { connectionEnabled });
await controller.attach(transport, detectBrowserMeta(), connectionEnabled, {
beforeDisconnect: async () => {
diagnostics.record("background.before_disconnect_cleanup", {
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
});
const report = await cleanupAfterDisconnect();
if (report.failures.length > 0) {
console.warn("[browser-skill] session cleanup before disconnect was incomplete", report);
}
},
onDisconnected: async () => {
diagnostics.record("background.after_disconnect_cleanup", {
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
});
const report = await cleanupAfterDisconnect();
if (report.failures.length > 0) {
console.warn("[browser-skill] session cleanup after disconnect was incomplete", report);
@@ -294,6 +339,7 @@ export default defineBackground(() => {
},
});
})().catch((err) => {
diagnostics.record("background.controller.attach_failed", { error: err });
console.error("[browser-skill] controller failed to attach", err);
});
@@ -374,6 +420,15 @@ export default defineBackground(() => {
dbg.__bhDispatcher = dispatcher;
}
if (typeof chrome.runtime?.onSuspend?.addListener === "function") {
chrome.runtime.onSuspend.addListener(() => {
diagnostics.record("extension.worker.suspending", {
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
transport_state: transport.state,
});
});
}
console.info("[browser-skill] background worker initialised");
});
@@ -0,0 +1,85 @@
import { describe, expect, it, vi } from "vitest";
import { DiagnosticLogger } from "../diagnostics";
class MemoryStorage {
value: Record<string, unknown> = {};
async get(key: string): Promise<Record<string, unknown>> {
return { [key]: this.value[key] };
}
async set(items: Record<string, unknown>): Promise<void> {
this.value = { ...this.value, ...items };
}
}
describe("DiagnosticLogger", () => {
it("persists evidence before transport readiness and flushes it after reconnect", async () => {
const storage = new MemoryStorage();
const send = vi.fn();
const logger = new DiagnosticLogger(storage);
logger.bindTransport(send);
logger.record("transport.socket.closed", { code: 1006 });
await logger.flush();
expect(send).not.toHaveBeenCalled();
logger.setTransportReady(true);
await logger.flush();
expect(send).toHaveBeenCalledOnce();
expect(send.mock.calls[0]?.[0]).toMatchObject({
event: "browser.diagnostic",
payload: {
schema_version: 1,
entries: [
{
event: "transport.socket.closed",
fields: { code: 1006 },
},
],
},
});
expect(storage.value.bsk_diagnostic_events_v1).toEqual([]);
});
it("keeps persisted evidence when the transport rejects the flush", async () => {
const storage = new MemoryStorage();
const logger = new DiagnosticLogger(storage);
logger.bindTransport(() => {
throw new Error("socket closed");
});
logger.record("session.stop.requested", { session_id: "abcd" });
logger.setTransportReady(true);
await logger.flush();
expect(storage.value.bsk_diagnostic_events_v1).toMatchObject([
{ event: "session.stop.requested", fields: { session_id: "abcd" } },
]);
});
it("does not resend an acknowledged event when send records a new event", async () => {
const storage = new MemoryStorage();
const batches: Array<{ payload?: { entries?: Array<{ event: string }> } }> = [];
const logger = new DiagnosticLogger(storage);
logger.record("dispatcher.rpc.received", { rpc_id: "one" });
await logger.flush();
logger.bindTransport((frame) => {
batches.push(frame as { payload?: { entries?: Array<{ event: string }> } });
if (batches.length === 1) {
logger.record("dispatcher.rpc.response_sent", { rpc_id: "one" });
}
});
logger.setTransportReady(true);
await logger.flush();
await logger.flush();
expect(batches.map((batch) => batch.payload?.entries?.map((entry) => entry.event))).toEqual([
["dispatcher.rpc.received"],
["dispatcher.rpc.response_sent"],
]);
expect(storage.value.bsk_diagnostic_events_v1).toEqual([]);
});
});
@@ -6,6 +6,7 @@ import {
} from "../transport/handshake";
import type { Transport } from "../transport/transport";
import type { ConnectionState, HandshakeResult } from "../transport/types";
import type { DiagnosticSink } from "./diagnostics";
import { getLabel, getOrCreateInstanceId } from "./instance-id";
import { compareProtocol, parseProtocolMajor } from "./semver";
@@ -51,6 +52,11 @@ export class ConnectionController {
private handshakeAbort: AbortController | null = null;
private handshakeRetryTimer: ReturnType<typeof setTimeout> | null = null;
private suppressDisconnectRecovery = false;
private readonly diagnostics?: DiagnosticSink;
constructor(diagnostics?: DiagnosticSink) {
this.diagnostics = diagnostics;
}
get isConnectionEnabled(): boolean {
return this.connectionEnabled;
@@ -89,8 +95,20 @@ export class ConnectionController {
this.lifecycleHooks = lifecycleHooks;
this.instanceId = await getOrCreateInstanceId();
this.label = await getLabel();
this.diagnostics?.("controller.attach", {
instance_id: this.instanceId,
connection_enabled: connectionEnabled,
transport_state: transport.state,
});
transport.onConnectionStateChange((s) => {
this.diagnostics?.("controller.transport_state", {
transport_state: s,
controller_state: this.currentState,
connection_generation: this.connectionGeneration,
suppress_disconnect_recovery: this.suppressDisconnectRecovery,
recovery_inflight: this.disconnectRecovery !== null,
});
if (s === "disconnected") {
// An in-flight handshake belongs to the dead connection — cancel it so
// a late resolution can never clobber the next attempt.
@@ -103,6 +121,9 @@ export class ConnectionController {
// handshake bounce); the caller owns the reconnect policy there.
this.suppressDisconnectRecovery = false;
} else {
this.diagnostics?.("controller.disconnect_recovery.requested", {
connection_generation: this.connectionGeneration,
});
void this.recoverFromDisconnect();
}
}
@@ -161,6 +182,12 @@ export class ConnectionController {
const generation = ++this.connectionGeneration;
const abort = new AbortController();
this.handshakeAbort = abort;
this.diagnostics?.("controller.handshake.started", {
connection_generation: generation,
browser_name: browser.name,
browser_version: browser.version,
instance_id: this.instanceId,
});
void this.runHandshake(browser, generation, abort.signal);
}
@@ -197,11 +224,21 @@ export class ConnectionController {
}
this.clearHandshakeRetry();
this.lastError = null;
this.diagnostics?.("controller.handshake.completed", {
connection_generation: generation,
verdict: verdict.kind,
daemon_version: outcome.result.version,
daemon_protocol: outcome.result.protocol_version,
});
this.setState(verdict.kind);
} catch (err) {
if (generation !== this.connectionGeneration || signal.aborted || isAbortError(err)) return;
this.handshake = null;
this.lastError = err instanceof Error ? err.message : String(err);
this.diagnostics?.("controller.handshake.failed", {
connection_generation: generation,
error: err,
});
// Bounce the half-open socket deliberately; the retry timer below owns
// the reconnect, so skip unexpected-loss recovery for this disconnect.
this.suppressDisconnectRecovery = true;
@@ -228,23 +265,57 @@ export class ConnectionController {
}
private recoverFromDisconnect(): Promise<void> {
if (this.disconnectRecovery) return this.disconnectRecovery;
if (this.disconnectRecovery) {
this.diagnostics?.("controller.disconnect_recovery.coalesced", {
connection_generation: this.connectionGeneration,
});
return this.disconnectRecovery;
}
const recovery = (async () => {
const recoveryGeneration = this.connectionGeneration;
this.diagnostics?.("controller.disconnect_recovery.started", {
recovery_generation: recoveryGeneration,
});
// WSTransport schedules its reconnect timer immediately after notifying
// state listeners. Yield once, then disconnect explicitly so that timer
// is cancelled before local session teardown begins.
await Promise.resolve();
if (!this.connectionEnabled || !this.transport) return;
if (!this.connectionEnabled || !this.transport) {
this.diagnostics?.("controller.disconnect_recovery.skipped", {
recovery_generation: recoveryGeneration,
reason: "disabled_or_missing_transport",
});
return;
}
// Another path already reconnected while we yielded — nothing to do.
if (this.currentState !== "disconnected") return;
if (this.currentState !== "disconnected") {
this.diagnostics?.("controller.disconnect_recovery.skipped", {
recovery_generation: recoveryGeneration,
reason: "state_already_changed",
current_state: this.currentState,
});
return;
}
await this.transport.disconnect().catch(() => {});
try {
this.diagnostics?.("controller.disconnect_cleanup.started", {
recovery_generation: recoveryGeneration,
current_generation: this.connectionGeneration,
});
await this.lifecycleHooks.onDisconnected?.();
this.diagnostics?.("controller.disconnect_cleanup.completed", {
recovery_generation: recoveryGeneration,
current_generation: this.connectionGeneration,
});
} catch (err) {
console.warn("[browser-skill] session cleanup after disconnect failed", err);
}
if (!this.connectionEnabled) return;
try {
this.diagnostics?.("controller.disconnect_reconnect.started", {
recovery_generation: recoveryGeneration,
current_generation: this.connectionGeneration,
});
await this.transport.connect();
} catch (err) {
this.lastError = err instanceof Error ? err.message : String(err);
@@ -252,22 +323,39 @@ export class ConnectionController {
}
})();
this.disconnectRecovery = recovery.finally(() => {
this.diagnostics?.("controller.disconnect_recovery.completed", {
connection_generation: this.connectionGeneration,
current_state: this.currentState,
});
this.disconnectRecovery = null;
});
return this.disconnectRecovery;
}
private cancelHandshake(): void {
const previousGeneration = this.connectionGeneration;
this.connectionGeneration += 1;
this.diagnostics?.("controller.handshake.cancelled", {
previous_generation: previousGeneration,
next_generation: this.connectionGeneration,
had_abort_controller: this.handshakeAbort !== null,
});
this.handshakeAbort?.abort();
this.handshakeAbort = null;
}
private scheduleHandshakeRetry(browser: { name: string; version: string }): void {
if (this.handshakeRetryTimer || !this.connectionEnabled) return;
this.diagnostics?.("controller.handshake_retry.scheduled", {
delay_ms: HANDSHAKE_RETRY_DELAY_MS,
connection_generation: this.connectionGeneration,
});
this.handshakeRetryTimer = setTimeout(() => {
this.handshakeRetryTimer = null;
if (!this.connectionEnabled || !this.transport) return;
this.diagnostics?.("controller.handshake_retry.timer_fired", {
connection_generation: this.connectionGeneration,
});
void this.transport.connect().catch((err) => {
this.lastError = err instanceof Error ? err.message : String(err);
this.setState("disconnected");
+178
View File
@@ -0,0 +1,178 @@
import type { ProtocolFrame } from "@/transport/types";
const STORAGE_KEY = "bsk_diagnostic_events_v1";
const MAX_ENTRIES = 500;
export type DiagnosticFields = Record<string, unknown>;
export type DiagnosticSink = (event: string, fields?: DiagnosticFields) => void;
export interface DiagnosticEntry {
id: string;
sequence: number;
timestamp: string;
monotonic_ms: number;
worker_boot_id: string;
event: string;
fields: DiagnosticFields;
}
interface DiagnosticStorage {
get(key: string): Promise<Record<string, unknown>>;
set(items: Record<string, unknown>): Promise<void>;
}
function defaultStorage(): DiagnosticStorage | null {
if (typeof chrome === "undefined" || !chrome.storage?.local) return null;
return {
get: (key) => chrome.storage.local.get(key),
set: (items) => chrome.storage.local.set(items),
};
}
function makeBootId(): string {
try {
return crypto.randomUUID();
} catch {
return `worker-${Date.now()}-${Math.random().toString(16).slice(2)}`;
}
}
function normalise(value: unknown): unknown {
if (value instanceof Error) {
return { name: value.name, message: value.message, stack: value.stack };
}
if (typeof value === "bigint") return value.toString();
if (Array.isArray(value)) return value.map(normalise);
if (value && typeof value === "object") {
return Object.fromEntries(
Object.entries(value as Record<string, unknown>).map(([key, child]) => [
key,
normalise(child),
]),
);
}
return value;
}
/**
* Persistent diagnostics for the MV3 background worker.
*
* Events are first written to chrome.storage.local so a worker termination or
* dead WebSocket does not erase the evidence. Once the post-handshake link is
* ready, pending entries are forwarded to the daemon as one
* `browser.diagnostic` event and removed from the local queue.
*/
export class DiagnosticLogger {
readonly workerBootId: string;
readonly sink: DiagnosticSink;
private sequence = 0;
private storageQueue: Promise<void> = Promise.resolve();
private flushPromise: Promise<void> | null = null;
private flushRequested = false;
private send: ((frame: ProtocolFrame) => void) | null = null;
private transportReady = false;
private readonly storage: DiagnosticStorage | null;
constructor(storage: DiagnosticStorage | null = defaultStorage()) {
this.workerBootId = makeBootId();
this.storage = storage;
this.sink = (event, fields = {}) => this.record(event, fields);
}
bindTransport(send: (frame: ProtocolFrame) => void): void {
this.send = send;
}
setTransportReady(ready: boolean): void {
this.transportReady = ready;
if (ready) void this.flush();
}
record(event: string, fields: DiagnosticFields = {}): void {
const sequence = ++this.sequence;
const entry: DiagnosticEntry = {
id: `${this.workerBootId}:${sequence}`,
sequence,
timestamp: new Date().toISOString(),
monotonic_ms: Math.round(performance.now()),
worker_boot_id: this.workerBootId,
event,
fields: normalise(fields) as DiagnosticFields,
};
console.info("[bsk diagnostic]", entry);
this.storageQueue = this.storageQueue
.then(async () => {
if (!this.storage) return;
const stored = await this.storage.get(STORAGE_KEY);
const previous = Array.isArray(stored[STORAGE_KEY])
? (stored[STORAGE_KEY] as DiagnosticEntry[])
: [];
await this.storage.set({
[STORAGE_KEY]: [...previous, entry].slice(-MAX_ENTRIES),
});
})
.catch((error) => {
console.warn("[bsk diagnostic] failed to persist event", error);
});
if (this.transportReady) void this.flush();
}
flush(): Promise<void> {
if (this.flushPromise) {
this.flushRequested = true;
return this.flushPromise;
}
const running = (async () => {
await this.storageQueue;
if (!this.transportReady || !this.send || !this.storage) return;
const stored = await this.storage.get(STORAGE_KEY);
const entries = Array.isArray(stored[STORAGE_KEY])
? (stored[STORAGE_KEY] as DiagnosticEntry[])
: [];
if (entries.length === 0) return;
try {
this.send({
event: "browser.diagnostic",
payload: {
schema_version: 1,
worker_boot_id: this.workerBootId,
entries,
},
});
} catch (error) {
console.warn("[bsk diagnostic] daemon flush failed", error);
return;
}
const sentIds = new Set(entries.map((entry) => entry.id));
// Serialize acknowledgement behind every record write that may have
// started while send() was in progress. Without this, a concurrent
// record can read the pre-ack queue and write an already-sent entry
// back into storage, causing the next batch to duplicate it.
this.storageQueue = this.storageQueue
.then(async () => {
const current = await this.storage?.get(STORAGE_KEY);
const currentEntries = Array.isArray(current?.[STORAGE_KEY])
? (current[STORAGE_KEY] as DiagnosticEntry[])
: [];
await this.storage?.set({
[STORAGE_KEY]: currentEntries.filter((entry) => !sentIds.has(entry.id)),
});
})
.catch((error) => {
console.warn("[bsk diagnostic] failed to acknowledge event batch", error);
});
await this.storageQueue;
})().catch((error) => {
console.warn("[bsk diagnostic] flush failed", error);
});
this.flushPromise = running.finally(() => {
this.flushPromise = null;
if (this.flushRequested && this.transportReady) {
this.flushRequested = false;
void this.flush();
}
});
return this.flushPromise;
}
}
+6
View File
@@ -21,6 +21,7 @@
* after the handshake has landed.
*/
import type { DiagnosticSink } from "@/lib/diagnostics";
import type { ProtocolFrame } from "@/transport/types";
export const HEARTBEAT_EVENT = "system.heartbeat";
@@ -46,6 +47,7 @@ export interface HeartbeatOptions {
intervalMs?: number;
/** Inject fake timers (tests). */
timers?: HeartbeatTimers;
diagnostics?: DiagnosticSink;
}
export interface HeartbeatHandle {
@@ -74,7 +76,9 @@ export function startHeartbeat(options: HeartbeatOptions): HeartbeatHandle {
const beat = () => {
try {
options.send({ event: HEARTBEAT_EVENT, payload: {} });
options.diagnostics?.("heartbeat.sent");
} catch (err) {
options.diagnostics?.("heartbeat.send_failed", { error: err });
// The socket raced a close between the state change and this
// tick. The transport's own reconnect path recovers; swallow so
// the interval keeps running (it is cleared on the disconnect
@@ -87,12 +91,14 @@ export function startHeartbeat(options: HeartbeatOptions): HeartbeatHandle {
if (timer !== null) {
timers.clearInterval(timer);
timer = null;
options.diagnostics?.("heartbeat.timer.stopped");
}
};
const start = () => {
if (timer !== null) return;
timer = timers.setInterval(beat, intervalMs);
options.diagnostics?.("heartbeat.timer.started", { interval_ms: intervalMs });
};
const unsubscribe = options.onActiveChange((active) => {
+17 -2
View File
@@ -20,6 +20,7 @@
* SW idle threshold.
*/
import type { DiagnosticSink } from "@/lib/diagnostics";
import type { Transport } from "@/transport/transport";
export const KEEPALIVE_ALARM_NAME = "bh-keepalive";
@@ -59,6 +60,7 @@ export interface KeepaliveOptions {
alarmName?: string;
/** Override the alarm period (tests). */
periodMin?: number;
diagnostics?: DiagnosticSink;
}
export interface KeepaliveHandle {
@@ -79,11 +81,24 @@ export function startKeepalive(options: KeepaliveOptions): KeepaliveHandle {
const period = options.periodMin ?? KEEPALIVE_PERIOD_MIN;
const tick = async () => {
if (options.shouldConnect && !options.shouldConnect()) return;
if (options.transport.state === "connected") return;
if (options.shouldConnect && !options.shouldConnect()) {
options.diagnostics?.("keepalive.tick.skipped", { reason: "connection_disabled" });
return;
}
if (options.transport.state === "connected") {
options.diagnostics?.("keepalive.tick.healthy", { transport_state: "connected" });
return;
}
options.diagnostics?.("keepalive.reconnect.started", {
transport_state: options.transport.state,
});
try {
await options.transport.connect();
options.diagnostics?.("keepalive.reconnect.completed", {
transport_state: options.transport.state,
});
} catch (err) {
options.diagnostics?.("keepalive.reconnect.failed", { error: err });
console.debug("[bsk keepalive] connect attempt failed", err);
}
};
@@ -13,8 +13,6 @@ describe("RefStore", () => {
backendNodeId: 42,
tabId: 7,
generation: 0,
kind: "dom",
capabilities: ["interact", "screenshot"],
});
expect(s.size()).toBe(1);
expect(s.isEmpty()).toBe(false);
@@ -50,31 +48,4 @@ describe("RefStore", () => {
cdpSessionId: "child-session",
});
});
it("preserves explicit capabilities for visual surface refs", () => {
const s = new RefStore();
s.set("e1", 42, {
tabId: 7,
kind: "surface",
capabilities: ["screenshot"],
});
expect(s.resolveEntry("e1")).toMatchObject({
kind: "surface",
capabilities: ["screenshot"],
});
});
it("never grants ordinary interaction to a surface through defaults or caller input", () => {
const s = new RefStore();
s.set("e1", 42, { tabId: 7, kind: "surface" });
s.set("e2", 43, {
tabId: 7,
kind: "surface",
capabilities: ["interact", "screenshot"],
});
expect(s.resolveEntry("e1")?.capabilities).toEqual(["screenshot"]);
expect(s.resolveEntry("e2")?.capabilities).toEqual(["screenshot"]);
});
});
@@ -1,67 +0,0 @@
import { describe, expect, it } from "vitest";
import { type SurfaceCaptureInput, SurfaceCaptureStore } from "../surface-capture-store";
function input(): SurfaceCaptureInput {
return {
sessionId: "aa11",
tabId: 7,
navigationIdentity: "navigation",
surface: {
ref: "e3",
frameId: "main",
backendNodeId: 99,
observationGeneration: 2,
},
topViewportRect: { x: 10, y: 20, w: 100, h: 50 },
imageWidth: 200,
imageHeight: 100,
viewportSignature: "viewport",
frameProjectionSignature: "frame",
};
}
describe("SurfaceCaptureStore", () => {
it("creates metadata-only, short-lived captures and consumes them once", () => {
let now = 1_000;
const store = new SurfaceCaptureStore({
now: () => now,
ttlMs: 500,
createId: () => "sc_test",
});
const capture = store.create(input());
expect(capture).toMatchObject({
id: "sc_test",
createdAt: 1_000,
expiresAt: 1_500,
consumed: false,
});
expect(capture).not.toHaveProperty("imageBase64");
expect(store.consume("sc_test")).toMatchObject({ ok: true });
expect(store.consume("sc_test")).toEqual({ ok: false, reason: "consumed" });
now = 1_600;
expect(store.size()).toBe(0);
});
it("rejects expired and unknown captures", () => {
let now = 1_000;
const store = new SurfaceCaptureStore({
now: () => now,
ttlMs: 10,
createId: () => "sc_expired",
});
store.create(input());
now = 1_010;
expect(store.consume("sc_expired")).toEqual({ ok: false, reason: "expired" });
expect(store.consume("sc_missing")).toEqual({ ok: false, reason: "not_found" });
});
it("clears every capture on session cleanup", () => {
const store = new SurfaceCaptureStore({ createId: () => "sc_clear" });
store.create(input());
store.clear();
expect(store.size()).toBe(0);
});
});
@@ -1,3 +1,4 @@
import type { DiagnosticSink } from "@/lib/diagnostics";
import { handleSessionStop, type SessionStopDeps } from "@/tools/session";
import type { SessionManager } from "./manager";
@@ -15,6 +16,7 @@ export interface DisconnectCleanupOptions {
manager: SessionManager;
sessionStopDeps?: SessionStopDeps;
onSessionsChanged?: () => void;
diagnostics?: DiagnosticSink;
}
/**
@@ -29,7 +31,15 @@ export function createDisconnectCleanup(options: DisconnectCleanupOptions) {
let inFlight: Promise<DisconnectCleanupReport> | null = null;
return (): Promise<DisconnectCleanupReport> => {
if (inFlight) return inFlight;
if (inFlight) {
options.diagnostics?.("disconnect_cleanup.coalesced", {
live_session_ids: options.manager.list().map((ctx) => ctx.sessionId),
});
return inFlight;
}
options.diagnostics?.("disconnect_cleanup.scheduled", {
live_session_ids: options.manager.list().map((ctx) => ctx.sessionId),
});
inFlight = cleanupSessions(options).finally(() => {
inFlight = null;
});
@@ -42,8 +52,17 @@ async function cleanupSessions(
): Promise<DisconnectCleanupReport> {
const stoppedSessionIds: string[] = [];
const failures: DisconnectCleanupFailure[] = [];
const capturedSessions = options.manager.list();
options.diagnostics?.("disconnect_cleanup.started", {
captured_session_ids: capturedSessions.map((ctx) => ctx.sessionId),
});
for (const ctx of options.manager.list()) {
for (const ctx of capturedSessions) {
options.diagnostics?.("disconnect_cleanup.session_stop.started", {
session_id: ctx.sessionId,
agent_window_id: ctx.agentWindowId,
still_registered: options.manager.has(ctx.sessionId),
});
try {
const result = await handleSessionStop(
options.manager,
@@ -62,14 +81,26 @@ async function cleanupSessions(
continue;
}
stoppedSessionIds.push(ctx.sessionId);
options.diagnostics?.("disconnect_cleanup.session_stop.completed", {
session_id: ctx.sessionId,
});
} catch (err) {
failures.push({
sessionId: ctx.sessionId,
message: err instanceof Error ? err.message : String(err),
});
options.diagnostics?.("disconnect_cleanup.session_stop.failed", {
session_id: ctx.sessionId,
error: err,
});
}
}
options.onSessionsChanged?.();
options.diagnostics?.("disconnect_cleanup.completed", {
stopped_session_ids: stoppedSessionIds,
failures,
remaining_session_ids: options.manager.list().map((ctx) => ctx.sessionId),
});
return { stoppedSessionIds, failures };
}
@@ -1,3 +1,4 @@
import type { DiagnosticSink } from "@/lib/diagnostics";
import type { Transport } from "@/transport/transport";
import type { EventFrame } from "@/transport/types";
import type { SessionManager } from "./manager";
@@ -24,6 +25,7 @@ export interface SessionEventHandlerOptions {
* `chrome.storage.session` "sessions live" flag (review M4/M5 I3).
*/
onSessionsChanged?: () => void;
diagnostics?: DiagnosticSink;
}
function chromeWindowEvents(): WindowRemovedListener {
@@ -50,6 +52,11 @@ export function attachSessionEventHandler(options: SessionEventHandlerOptions):
const onRemoved = (windowId: number): void => {
const ctx = manager.findByWindowId(windowId);
options.diagnostics?.("chrome.window.removed", {
window_id: windowId,
matched_session_id: ctx?.sessionId ?? null,
live_session_ids: manager.list().map((entry) => entry.sessionId),
});
if (!ctx) return;
const returnFailures = Array.from(ctx.borrowedTabs.keys()).map((tabId) => ({
tab_id: tabId,
@@ -81,7 +88,16 @@ export function attachSessionEventHandler(options: SessionEventHandlerOptions):
};
try {
transport.send(event);
options.diagnostics?.("session.window_closed.sent", {
session_id: ctx.sessionId,
window_id: windowId,
});
} catch (err) {
options.diagnostics?.("session.window_closed.send_failed", {
session_id: ctx.sessionId,
window_id: windowId,
error: err,
});
console.warn("[bh] could not push session.window_closed event", err);
}
})
+41 -3
View File
@@ -1,12 +1,11 @@
import type { DiagnosticSink } from "@/lib/diagnostics";
import { AGENT_WINDOW_HOME, type AgentWindowApi, chromeAgentWindowApi } from "./agent-window";
import { RefStore } from "./ref-store";
import { SurfaceCaptureStore } from "./surface-capture-store";
export interface SessionContext {
sessionId: string;
agentWindowId: number;
refStore: RefStore;
surfaceCaptures: SurfaceCaptureStore;
borrowedTabs: Map<number, BorrowedTab>;
createdAtMs: number;
}
@@ -25,6 +24,7 @@ export interface BorrowReservation {
export interface SessionManagerOptions {
agentWindow?: AgentWindowApi;
now?: () => number;
diagnostics?: DiagnosticSink;
}
/** Options for starting a session's Agent Window. */
@@ -85,10 +85,12 @@ export class SessionManager {
private readonly borrowReservations = new Map<number, string>();
private readonly agentWindow: AgentWindowApi;
private readonly now: () => number;
private readonly diagnostics?: DiagnosticSink;
constructor(options: SessionManagerOptions = {}) {
this.agentWindow = options.agentWindow ?? chromeAgentWindowApi;
this.now = options.now ?? Date.now;
this.diagnostics = options.diagnostics;
}
has(sessionId: string): boolean {
@@ -171,6 +173,13 @@ export class SessionManager {
* daemon in the `tool.session_start` reply.
*/
async start(sessionId: string, opts: SessionStartOptions = {}): Promise<SessionContext> {
this.diagnostics?.("session_manager.start.requested", {
session_id: sessionId,
existing_session_ids: this.list().map((ctx) => ctx.sessionId),
focused: opts.focused ?? null,
size: opts.size ?? null,
signal_aborted: opts.signal?.aborted ?? false,
});
if (this.sessions.has(sessionId)) {
throw new Error(`[bh] session ${sessionId} already exists`);
}
@@ -180,6 +189,10 @@ export class SessionManager {
try {
const { signal: _signal, ...createOptions } = opts;
windowId = await this.agentWindow.create(AGENT_WINDOW_HOME, createOptions);
this.diagnostics?.("session_manager.window.created", {
session_id: sessionId,
agent_window_id: windowId,
});
throwIfSessionStartAborted(opts.signal);
await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
throwIfSessionStartAborted(opts.signal);
@@ -188,14 +201,23 @@ export class SessionManager {
sessionId,
agentWindowId: windowId,
refStore: new RefStore(),
surfaceCaptures: new SurfaceCaptureStore({ now: this.now }),
borrowedTabs: new Map(),
createdAtMs: this.now(),
};
this.sessions.set(sessionId, ctx);
this.windowIndex.set(windowId, sessionId);
this.diagnostics?.("session_manager.start.committed", {
session_id: sessionId,
agent_window_id: windowId,
live_session_ids: this.list().map((entry) => entry.sessionId),
});
return ctx;
} catch (startupError) {
this.diagnostics?.("session_manager.start.failed", {
session_id: sessionId,
agent_window_id: windowId,
error: startupError,
});
if (windowId !== null) {
try {
await this.agentWindow.remove(windowId);
@@ -219,12 +241,28 @@ export class SessionManager {
options: { dropOnly?: boolean } = {},
): Promise<SessionContext | null> {
const ctx = this.sessions.get(sessionId);
this.diagnostics?.("session_manager.stop.requested", {
session_id: sessionId,
drop_only: options.dropOnly ?? false,
existed: ctx !== undefined,
live_session_ids: this.list().map((entry) => entry.sessionId),
});
if (!ctx) return null;
if (!options.dropOnly) {
await this.agentWindow.remove(ctx.agentWindowId);
this.diagnostics?.("session_manager.window.removed", {
session_id: sessionId,
agent_window_id: ctx.agentWindowId,
});
}
this.sessions.delete(sessionId);
this.windowIndex.delete(ctx.agentWindowId);
this.diagnostics?.("session_manager.stop.committed", {
session_id: sessionId,
agent_window_id: ctx.agentWindowId,
drop_only: options.dropOnly ?? false,
live_session_ids: this.list().map((entry) => entry.sessionId),
});
return ctx;
}
@@ -1,5 +1,3 @@
import type { Rect } from "@browser-skill/vom";
/**
* Per-session map from `@e<N>` snapshot refs to a CDP node address.
*
@@ -14,17 +12,12 @@ import type { Rect } from "@browser-skill/vom";
* inside the `SessionContext`.
*/
export type BackendNodeId = number;
export type RefCapability = "interact" | "screenshot";
export type RefTargetKind = "dom" | "surface";
export interface RefEntry {
backendNodeId: BackendNodeId;
tabId: number | null;
frameId?: string;
cdpSessionId?: string;
visibleRect?: Rect;
kind: RefTargetKind;
capabilities: RefCapability[];
generation: number;
}
@@ -35,19 +28,8 @@ export type RefInput =
tabId: number;
frameId?: string;
cdpSessionId?: string;
visibleRect?: Rect;
kind?: RefTargetKind;
capabilities?: RefCapability[];
};
function capabilitiesFor(
kind: RefTargetKind,
capabilities: RefCapability[] | undefined,
): RefCapability[] {
if (kind === "surface") return ["screenshot"];
return capabilities ?? ["interact", "screenshot"];
}
export class RefStore {
private readonly map = new Map<string, RefEntry>();
private generation = 0;
@@ -88,20 +70,13 @@ export class RefStore {
tabId?: number;
frameId?: string;
cdpSessionId?: string;
visibleRect?: Rect;
kind?: RefTargetKind;
capabilities?: RefCapability[];
} = {},
): void {
const kind = opts.kind ?? "dom";
this.map.set(normaliseRef(ref), {
backendNodeId: id,
tabId: opts.tabId ?? null,
...(opts.frameId ? { frameId: opts.frameId } : {}),
...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
...(opts.visibleRect ? { visibleRect: opts.visibleRect } : {}),
kind,
capabilities: capabilitiesFor(kind, opts.capabilities),
generation: this.generation,
});
}
@@ -119,20 +94,14 @@ export class RefStore {
return {
backendNodeId: input,
tabId: null,
kind: "dom",
capabilities: ["interact", "screenshot"],
generation: this.generation,
};
}
const kind = input.kind ?? "dom";
return {
backendNodeId: input.backendNodeId,
tabId: input.tabId,
...(input.frameId ? { frameId: input.frameId } : {}),
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
...(input.visibleRect ? { visibleRect: input.visibleRect } : {}),
kind,
capabilities: capabilitiesFor(kind, input.capabilities),
generation: this.generation,
};
}
@@ -1,100 +0,0 @@
import type { Rect } from "@browser-skill/vom";
export const SURFACE_CAPTURE_TTL_MS = 30_000;
export interface SurfaceCapture {
id: string;
sessionId: string;
tabId: number;
navigationIdentity: string;
surface: {
ref: string;
frameId?: string;
backendNodeId: number;
observationGeneration: number;
};
topViewportRect: Rect;
imageWidth: number;
imageHeight: number;
viewportSignature: string;
frameProjectionSignature: string;
createdAt: number;
expiresAt: number;
consumed: boolean;
}
export type SurfaceCaptureInput = Omit<
SurfaceCapture,
"id" | "createdAt" | "expiresAt" | "consumed"
>;
export type SurfaceCaptureConsumeResult =
| { ok: true; capture: SurfaceCapture }
| { ok: false; reason: "not_found" | "expired" | "consumed" };
export interface SurfaceCaptureStoreOptions {
now?: () => number;
ttlMs?: number;
createId?: () => string;
}
function randomCaptureId(): string {
const bytes = crypto.getRandomValues(new Uint8Array(12));
return `sc_${Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("")}`;
}
/** Per-session, metadata-only store for short-lived screenshot coordinate transactions. */
export class SurfaceCaptureStore {
private readonly captures = new Map<string, SurfaceCapture>();
private readonly now: () => number;
private readonly ttlMs: number;
private readonly createId: () => string;
constructor(options: SurfaceCaptureStoreOptions = {}) {
this.now = options.now ?? Date.now;
this.ttlMs = options.ttlMs ?? SURFACE_CAPTURE_TTL_MS;
this.createId = options.createId ?? randomCaptureId;
}
create(input: SurfaceCaptureInput): SurfaceCapture {
this.purgeExpired();
const createdAt = this.now();
const capture: SurfaceCapture = {
...input,
id: this.createId(),
createdAt,
expiresAt: createdAt + this.ttlMs,
consumed: false,
};
this.captures.set(capture.id, capture);
return capture;
}
consume(id: string): SurfaceCaptureConsumeResult {
const capture = this.captures.get(id);
if (!capture) return { ok: false, reason: "not_found" };
if (capture.expiresAt <= this.now()) {
this.captures.delete(id);
return { ok: false, reason: "expired" };
}
if (capture.consumed) return { ok: false, reason: "consumed" };
capture.consumed = true;
return { ok: true, capture };
}
clear(): void {
this.captures.clear();
}
size(): number {
this.purgeExpired();
return this.captures.size;
}
private purgeExpired(): void {
const now = this.now();
for (const [id, capture] of this.captures) {
if (capture.expiresAt <= now) this.captures.delete(id);
}
}
}
@@ -11,7 +11,6 @@ import {
parseKeySpec,
resolveKeyDescriptor,
} from "../interaction";
import { captureSurfaceEnvironment } from "../surface-target";
function fakeAgentWindow(ids: number[]) {
let i = 0;
@@ -55,61 +54,6 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
return { cdp, tabsApi, sent };
}
async function makeSurfaceClickFixture(onMouse?: (params: Record<string, unknown>) => void) {
const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const context = await manager.start("aa11");
context.refStore.set("e3", 99, {
tabId: 4,
kind: "surface",
visibleRect: { x: 10, y: 20, w: 100, h: 40 },
});
const fake = makeFakeCdp({
"Page.getFrameTree": () => ({
frameTree: {
frame: { id: "main", loaderId: "loader-1", url: "https://fixture.test/" },
},
}),
"Page.getLayoutMetrics": () => ({
cssLayoutViewport: { clientWidth: 800, clientHeight: 600, pageX: 0, pageY: 0 },
cssVisualViewport: {
clientWidth: 800,
clientHeight: 600,
pageX: 0,
pageY: 0,
scale: 1,
},
}),
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
"Runtime.evaluate": () => ({
result: { value: { overlayHostPresent: false, overlayHostConnected: false } },
}),
"Input.dispatchMouseEvent": (params) => {
onMouse?.(params as Record<string, unknown>);
return {};
},
});
const environment = await captureSurfaceEnvironment(fake.cdp, 4, undefined);
if ("code" in environment) throw new Error(environment.message);
const entry = context.refStore.resolveEntry("e3");
if (!entry) throw new Error("missing Surface ref");
const capture = context.surfaceCaptures.create({
sessionId: "aa11",
tabId: 4,
navigationIdentity: environment.navigationIdentity,
surface: {
ref: "e3",
backendNodeId: 99,
observationGeneration: entry.generation,
},
topViewportRect: { x: 10, y: 20, w: 100, h: 40 },
imageWidth: 200,
imageHeight: 80,
viewportSignature: environment.viewportSignature,
frameProjectionSignature: environment.frameProjectionSignature,
});
return { manager, context, fake, capture };
}
describe("modifiersBitfield", () => {
it("matches CDP's expected bit layout", () => {
expect(modifiersBitfield([])).toBe(0);
@@ -179,29 +123,6 @@ describe("handleClick", () => {
expect(fake.cdp.send).not.toHaveBeenCalled();
});
it("rejects screenshot-only visual surface refs before issuing CDP calls", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, {
tabId: 4,
kind: "surface",
capabilities: ["screenshot"],
});
const fake = makeFakeCdp({});
const res = await handleClick(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
expect(res).toMatchObject({
code: "permission_denied",
data: { reason: "ref_capability_denied", required_capability: "interact" },
});
expect(fake.cdp.send).not.toHaveBeenCalled();
});
it("clicks by ref, computes the quad centre, dispatches three mouse events", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
@@ -246,74 +167,6 @@ describe("handleClick", () => {
});
});
it.each([
"left",
"middle",
"right",
] as const)("uses the common click executor for a %s-button Surface click", async (button) => {
const { manager, fake, capture } = await makeSurfaceClickFixture();
const result = await handleClick(
manager,
{
session_id: "aa11",
ref: "@e3",
capture_id: capture.id,
image_x: 50,
image_y: 20,
button,
click_count: 2,
modifiers: ["ctrl", "shift"],
},
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in result) throw new Error(JSON.stringify(result));
expect(result).toMatchObject({
used_ref: "e3",
capture_id: capture.id,
image_x: 50,
image_y: 20,
x: 35,
y: 30,
});
const mouse = fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent");
expect(mouse).toHaveLength(3);
expect(mouse[1].params).toMatchObject({
type: "mousePressed",
button,
clickCount: 2,
modifiers: 2 | 8,
});
expect(mouse[2].params).toMatchObject({
type: "mouseReleased",
button,
clickCount: 2,
});
});
it("consumes a Surface capture when the common click executor fails", async () => {
let mouseCalls = 0;
const { manager, fake, capture } = await makeSurfaceClickFixture((params) => {
mouseCalls += 1;
if (params.type === "mousePressed") throw new Error("input failed");
});
const params = {
session_id: "aa11",
ref: "@e3",
capture_id: capture.id,
image_x: 50,
image_y: 20,
};
expect(
await handleClick(manager, params, { cdp: fake.cdp, tabsApi: fake.tabsApi }),
).toMatchObject({ code: "cdp_failed" });
expect(mouseCalls).toBe(2);
expect(
await handleClick(manager, params, { cdp: fake.cdp, tabsApi: fake.tabsApi }),
).toMatchObject({ data: { reason: "surface_capture_consumed" } });
});
it("resolves frame refs in their CDP session and dispatches input in top coordinates", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
@@ -603,6 +456,30 @@ describe("handleClick", () => {
expect(res).toMatchObject({ code: "not_found", data: { reason: "selector_not_found" } });
});
it("returns invalid_selector when Chrome rejects non-CSS selector syntax", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
const fake = makeFakeCdp({
"DOM.getDocument": () => ({ root: { nodeId: 1 } }),
"DOM.querySelector": () => {
throw new Error('{"code":-32000,"message":"DOM Error while querying"}');
},
});
const res = await handleClick(
sm,
{ session_id: "aa11", selector: "text=下午 12~18点" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
expect(res).toMatchObject({
code: "invalid_params",
message: "selector is not valid CSS: text=下午 12~18点",
data: {
reason: "invalid_selector",
selector: "text=下午 12~18点",
},
});
});
it("respects the AbortSignal", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
@@ -1,119 +0,0 @@
import { describe, expect, it, vi } from "vitest";
import { dispatchMouseClick } from "../mouse-input";
import type { CdpRunner } from "../shared";
function fakeCdp(onSend?: (method: string, params: Record<string, unknown>) => void): {
cdp: CdpRunner;
events: Record<string, unknown>[];
} {
const events: Record<string, unknown>[] = [];
return {
events,
cdp: {
send: vi.fn(async (_tabId, method, params) => {
const event = params as Record<string, unknown>;
events.push(event);
onSend?.(method, event);
return {} as never;
}),
},
};
}
describe("dispatchMouseClick", () => {
it("emits one move, press, and release sequence", async () => {
const { cdp, events } = fakeCdp();
expect(
await dispatchMouseClick(
cdp,
4,
{ x: 12, y: 34 },
{
button: "left",
clickCount: 1,
modifiers: 0,
moveSettleMs: 0,
},
),
).toBe("completed");
expect(events).toEqual([
{
type: "mouseMoved",
x: 12,
y: 34,
modifiers: 0,
},
{
type: "mousePressed",
x: 12,
y: 34,
button: "left",
clickCount: 1,
modifiers: 0,
},
{
type: "mouseReleased",
x: 12,
y: 34,
button: "left",
clickCount: 1,
modifiers: 0,
},
]);
});
it("releases a pressed button when cancellation arrives", async () => {
const controller = new AbortController();
const { cdp, events } = fakeCdp((_method, event) => {
if (event.type === "mousePressed") controller.abort();
});
expect(
await dispatchMouseClick(
cdp,
4,
{ x: 1, y: 2 },
{
button: "left",
clickCount: 1,
modifiers: 0,
signal: controller.signal,
moveSettleMs: 0,
},
),
).toBe("cancelled");
expect(events.at(-1)).toMatchObject({ type: "mouseReleased" });
});
it("settles pointer-move hit testing before pressing", async () => {
vi.useFakeTimers();
try {
const { cdp, events } = fakeCdp();
const click = dispatchMouseClick(
cdp,
4,
{ x: 1, y: 2 },
{
button: "left",
clickCount: 1,
modifiers: 0,
moveSettleMs: 32,
},
);
await vi.advanceTimersByTimeAsync(31);
expect(events.map((event) => event.type)).toEqual(["mouseMoved"]);
await vi.advanceTimersByTimeAsync(1);
await click;
expect(events.map((event) => event.type)).toEqual([
"mouseMoved",
"mousePressed",
"mouseReleased",
]);
} finally {
vi.useRealTimers();
}
});
});
@@ -67,9 +67,6 @@ function makeFakeCdp(handlers: Record<string, (params?: object) => unknown>) {
if (!handler && method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
}
if (!handler && method === "Page.getFrameTree") {
return { frameTree: { frame: { id: "main", loaderId: "loader-1", url: "https://test/" } } };
}
if (!handler) throw new Error(`unexpected CDP call ${method}`);
return handler(params);
});
@@ -303,82 +300,6 @@ describe("handleScreenshot", () => {
expect(clip).toMatchObject({ x: 10, y: 20, width: 100, height: 40 });
});
it("captures a bounded visible crop for a visual surface without scrolling it", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e5", 999, {
tabId: 7,
kind: "surface",
capabilities: ["screenshot"],
});
const { cdp, sent } = makeFakeCdp({
"Page.getLayoutMetrics": () => ({
cssLayoutViewport: { clientWidth: 4096, clientHeight: 4096 },
}),
"DOM.getContentQuads": () => ({ quads: [[0, 0, 4096, 0, 4096, 4096, 0, 4096]] }),
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
});
const res = await handleScreenshot(
sm,
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
makeScreenshotDeps({
cdp,
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
query: vi.fn(),
captureVisibleTab: vi.fn(),
}),
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(sent.some((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toBe(false);
const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
clip?: { width: number; height: number; scale: number };
};
expect(clip.clip).toMatchObject({ width: 4096, height: 4096 });
expect(clip.clip?.scale).toBeCloseTo(Math.sqrt(4_000_000 / (4096 * 4096)));
expect(res.capture).toMatchObject({
surface_ref: "@e5",
coordinate_space: "capture-image-pixel",
});
expect(ctx.surfaceCaptures.size()).toBe(1);
});
it("crops a visual surface screenshot to its observation-time visible region", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e5", 999, {
tabId: 7,
kind: "surface",
visibleRect: { x: 25, y: 30, w: 50, h: 40 },
});
const { cdp, sent } = makeFakeCdp({
"Page.getLayoutMetrics": () => ({
cssLayoutViewport: { clientWidth: 800, clientHeight: 600 },
}),
"DOM.getContentQuads": () => ({ quads: [[0, 0, 200, 0, 200, 100, 0, 100]] }),
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
});
const res = await handleScreenshot(
sm,
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
makeScreenshotDeps({
cdp,
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
query: vi.fn(),
captureVisibleTab: vi.fn(),
}),
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
clip?: { x: number; y: number; width: number; height: number };
};
expect(clip.clip).toMatchObject({ x: 25, y: 30, width: 50, height: 40 });
expect(res.capture?.id).toMatch(/^sc_/);
});
it("returns not_found for unknown ref", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
@@ -2772,85 +2693,6 @@ describe("handleSnapshot", () => {
);
});
it("discovers rendered canvases only in observe and stores a screenshot-only ref", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
const root: CdpAxNode = {
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
name: { type: "computedString", value: "Canvas app" },
backendDOMNodeId: 100,
};
const strings = ["body", "canvas", "static", "auto"];
const i = (value: string) => strings.indexOf(value);
const send = vi.fn(async (_tabId: number, method: string) => {
if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [root] };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
}
if (method === "DOMSnapshot.captureSnapshot") {
return {
strings,
documents: [
{
nodes: {
parentIndex: [-1, 0],
nodeName: [i("body"), i("canvas")],
backendNodeId: [100, 200],
attributes: [[], []],
},
layout: {
nodeIndex: [0, 1],
styles: [
[i("static"), i("auto"), i("auto")],
[i("static"), i("auto"), i("auto")],
],
bounds: [
[0, 0, 1000, 800],
[20, 30, 800, 500],
],
paintOrders: [0, 1],
},
},
],
};
}
throw new Error(`unexpected CDP method ${method}`);
});
const deps = {
cdp: {
send: send as unknown as <T = unknown>(
tabId: number,
method: string,
params?: object,
) => Promise<T>,
trackSessionTab: vi.fn(),
},
tabsApi: {
get: vi.fn(
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
},
conditionalSurfaceProbe: false,
};
const observed = await handleObserve(sm, { session_id: "aa11" }, deps);
if ("code" in observed) throw new Error(`unexpected error: ${JSON.stringify(observed)}`);
expect(observed.text).toContain('@e1 surface "canvas visual surface"');
expect(ctx.refStore.resolveEntry("e1")).toMatchObject({
backendNodeId: 200,
kind: "surface",
capabilities: ["screenshot"],
});
const snapshot = await handleSnapshot(sm, { session_id: "aa11" }, deps);
if ("code" in snapshot) throw new Error(`unexpected error: ${JSON.stringify(snapshot)}`);
expect(snapshot.text).not.toContain("visual surface");
expect(snapshot.ref_count).toBe(0);
});
it("allows passive snapshots of explicit user-window tabs", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
@@ -252,23 +252,12 @@ describe("handleSessionStop with auto-return", () => {
expect(aw.remove).not.toHaveBeenCalled();
});
it("clears refs and Surface captures before window teardown", async () => {
it("clears the RefStore before window teardown", async () => {
const aw = fakeAgentWindow([100]);
const sm = new SessionManager({ agentWindow: aw });
const ctx = await sm.start("aa11");
// Insert a fake ref so we can verify clear() ran.
ctx.refStore.set("e1", 123, { tabId: 7 });
ctx.surfaceCaptures.create({
sessionId: "aa11",
tabId: 7,
navigationIdentity: "navigation",
surface: { ref: "e1", backendNodeId: 123, observationGeneration: 0 },
topViewportRect: { x: 0, y: 0, w: 100, h: 50 },
imageWidth: 100,
imageHeight: 50,
viewportSignature: "viewport",
frameProjectionSignature: "frame",
});
const state: FakeState = {
tabs: new Map(),
windowsClosed: new Set(),
@@ -277,6 +266,5 @@ describe("handleSessionStop with auto-return", () => {
const { tabs, windows } = makeApis(state);
await handleSessionStop(sm, { session_id: "aa11" }, { tabManagement: { tabs, windows } });
expect(ctx.refStore.isEmpty()).toBe(true);
expect(ctx.surfaceCaptures.size()).toBe(0);
});
});
@@ -24,16 +24,10 @@ describe("lookupSnapshotRef", () => {
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual({
backendNodeId: 1234,
refKey: "e3",
kind: "dom",
capabilities: ["interact", "screenshot"],
generation: 0,
});
expect(lookupSnapshotRef(ctx, "e3", 4)).toEqual({
backendNodeId: 1234,
refKey: "e3",
kind: "dom",
capabilities: ["interact", "screenshot"],
generation: 0,
});
});
@@ -59,8 +53,6 @@ describe("resolveSnapshotRef", () => {
tabId: 4,
frameId: "child-frame",
cdpSessionId: "child-session",
kind: "dom",
capabilities: ["interact", "screenshot"],
});
const expected = {
@@ -68,9 +60,6 @@ describe("resolveSnapshotRef", () => {
refKey: "e3",
frameId: "child-frame",
cdpSessionId: "child-session",
kind: "dom" as const,
capabilities: ["interact", "screenshot"] as const,
generation: 0,
};
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
@@ -116,29 +105,6 @@ describe("resolveSnapshotRef", () => {
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual({
backendNodeId: 1234,
refKey: "e3",
kind: "dom",
capabilities: ["interact", "screenshot"],
generation: 0,
});
});
it("rejects an operation outside the ref's declared capabilities", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, {
tabId: 4,
kind: "surface",
capabilities: ["screenshot"],
});
expect(resolveSnapshotRef(ctx, "@e3", 4, "interact")).toMatchObject({
code: "permission_denied",
message: "ref @e3 does not support interact",
data: {
reason: "ref_capability_denied",
required_capability: "interact",
kind: "surface",
},
});
});
});
@@ -1,52 +0,0 @@
import { describe, expect, it } from "vitest";
import {
mapImagePointToViewport,
pointInRegion,
sameRect,
surfaceVisibleRect,
} from "../surface-coordinate";
describe("surface coordinate mapping", () => {
it("maps capture image pixels through the actual screenshot dimensions", () => {
expect(mapImagePointToViewport({ x: 100, y: 50, w: 400, h: 200 }, 800, 400, 200, 100)).toEqual({
x: 200,
y: 100,
});
});
it("rejects invalid image coordinates and dimensions", () => {
const rect = { x: 0, y: 0, w: 100, h: 50 };
expect(mapImagePointToViewport(rect, 100, 50, -1, 0)).toBeNull();
expect(mapImagePointToViewport(rect, 100, 50, 100, 0)).toBeNull();
expect(mapImagePointToViewport(rect, 100, 50, 0, 50)).toBeNull();
expect(mapImagePointToViewport(rect, 100, 50, Number.NaN, 0)).toBeNull();
expect(mapImagePointToViewport(rect, 100, 50, Number.POSITIVE_INFINITY, 0)).toBeNull();
expect(mapImagePointToViewport(rect, 0, 50, 0, 0)).toBeNull();
});
it("intersects live and observed visible regions and compares them strictly", () => {
const clipped = surfaceVisibleRect(
{ x: 0, y: 0, width: 200, height: 100 },
{ x: 25, y: 30, w: 50, h: 40 },
);
expect(clipped).toEqual({ x: 25, y: 30, w: 50, h: 40 });
expect(sameRect(clipped as NonNullable<typeof clipped>, { x: 25, y: 30, w: 50, h: 40 })).toBe(
true,
);
expect(sameRect(clipped as NonNullable<typeof clipped>, { x: 26, y: 30, w: 50, h: 40 })).toBe(
false,
);
});
it("checks the projected visible region rather than only its bounding box", () => {
const triangle = [
[
{ x: 0, y: 0 },
{ x: 100, y: 0 },
{ x: 0, y: 100 },
],
];
expect(pointInRegion({ x: 20, y: 20 }, triangle)).toBe(true);
expect(pointInRegion({ x: 90, y: 90 }, triangle)).toBe(false);
});
});
@@ -1,306 +0,0 @@
import { describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import type { CdpRunner } from "../shared";
import {
captureSurfaceEnvironment,
resolveSurfacePointerTarget,
SURFACE_POINTER_MOVE_SETTLE_MS,
} from "../surface-target";
function fakeAgentWindow() {
return {
create: vi.fn(async () => 100),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
};
}
function fakeBrowser() {
let loaderId = "loader-1";
let pageY = 0;
let quad = [10, 20, 110, 20, 110, 60, 10, 60];
const sent: Array<{ method: string; params?: object }> = [];
const cdp: CdpRunner = {
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
sent.push({ method, params });
if (method === "Page.getFrameTree") {
return {
frameTree: { frame: { id: "main", loaderId, url: "https://fixture.test/" } },
} as never;
}
if (method === "Page.getLayoutMetrics") {
return {
cssLayoutViewport: { clientWidth: 800, clientHeight: 600, pageX: 0, pageY },
cssVisualViewport: {
clientWidth: 800,
clientHeight: 600,
pageX: 0,
pageY,
scale: 1,
},
} as never;
}
if (method === "DOM.getContentQuads") return { quads: [quad] } as never;
throw new Error(`unexpected CDP call ${method}`);
}) as CdpRunner["send"],
trackSessionTab: vi.fn(),
};
return {
cdp,
sent,
setLoaderId: (value: string) => {
loaderId = value;
},
setPageY: (value: number) => {
pageY = value;
},
setQuad: (value: number[]) => {
quad = value;
},
};
}
async function setupCapture() {
const manager = new SessionManager({ agentWindow: fakeAgentWindow() });
const context = await manager.start("aa11");
context.refStore.set("e3", 99, {
tabId: 4,
kind: "surface",
visibleRect: { x: 10, y: 20, w: 100, h: 40 },
});
const browser = fakeBrowser();
const environment = await captureSurfaceEnvironment(browser.cdp, 4, undefined);
if ("code" in environment) throw new Error(environment.message);
const entry = context.refStore.resolveEntry("e3");
if (!entry) throw new Error("missing ref");
const capture = context.surfaceCaptures.create({
sessionId: "aa11",
tabId: 4,
navigationIdentity: environment.navigationIdentity,
surface: {
ref: "e3",
backendNodeId: 99,
observationGeneration: entry.generation,
},
topViewportRect: { x: 10, y: 20, w: 100, h: 40 },
imageWidth: 200,
imageHeight: 80,
viewportSignature: environment.viewportSignature,
frameProjectionSignature: environment.frameProjectionSignature,
});
return { manager, context, browser, capture };
}
function pointParams(captureId: string) {
return {
ref: "@e3",
captureId,
imageX: 50,
imageY: 20,
};
}
describe("Surface screenshot-bound pointer target", () => {
it("maps one fresh capture coordinate without executing an action", async () => {
const { context, browser, capture } = await setupCapture();
const result = await resolveSurfacePointerTarget(
context,
{ tabId: 4 },
pointParams(capture.id),
{ cdp: browser.cdp },
);
if ("code" in result) throw new Error(JSON.stringify(result));
expect(result).toMatchObject({
usedRef: "e3",
point: { x: 35, y: 30 },
moveSettleMs: SURFACE_POINTER_MOVE_SETTLE_MS,
capture: { id: capture.id, imageX: 50, imageY: 20 },
});
expect(browser.sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
const repeated = await resolveSurfacePointerTarget(
context,
{ tabId: 4 },
pointParams(capture.id),
{ cdp: browser.cdp },
);
expect(repeated).toMatchObject({
code: "permission_denied",
data: { reason: "surface_capture_consumed" },
});
});
it("rejects a new observation generation before sending input", async () => {
const { context, browser, capture } = await setupCapture();
context.refStore.replace([
[
"e3",
{
backendNodeId: 99,
tabId: 4,
kind: "surface" as const,
visibleRect: { x: 10, y: 20, w: 100, h: 40 },
},
],
]);
const result = await resolveSurfacePointerTarget(
context,
{ tabId: 4 },
pointParams(capture.id),
{ cdp: browser.cdp },
);
expect(result).toMatchObject({
code: "permission_denied",
data: { reason: "surface_capture_stale" },
});
expect(browser.sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
});
it("rejects navigation, scroll, and visible geometry changes", async () => {
for (const mutate of [
(browser: ReturnType<typeof fakeBrowser>) => browser.setLoaderId("loader-2"),
(browser: ReturnType<typeof fakeBrowser>) => browser.setPageY(20),
(browser: ReturnType<typeof fakeBrowser>) =>
browser.setQuad([11, 20, 111, 20, 111, 60, 11, 60]),
]) {
const { context, browser, capture } = await setupCapture();
mutate(browser);
const result = await resolveSurfacePointerTarget(
context,
{ tabId: 4 },
pointParams(capture.id),
{ cdp: browser.cdp },
);
expect(result).toMatchObject({
code: "permission_denied",
data: { reason: "surface_capture_stale" },
});
expect(browser.sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
}
});
it("consumes the capture when coordinate validation fails", async () => {
const invalid = await setupCapture();
const outside = await resolveSurfacePointerTarget(
invalid.context,
{ tabId: 4 },
{ ...pointParams(invalid.capture.id), imageX: 200 },
{ cdp: invalid.browser.cdp },
);
expect(outside).toMatchObject({
code: "invalid_params",
data: { reason: "surface_coordinate_invalid" },
});
expect(
await resolveSurfacePointerTarget(
invalid.context,
{ tabId: 4 },
pointParams(invalid.capture.id),
{ cdp: invalid.browser.cdp },
),
).toMatchObject({ data: { reason: "surface_capture_consumed" } });
});
it("maps image coordinates through an OOPIF frame projection", async () => {
const manager = new SessionManager({ agentWindow: fakeAgentWindow() });
const context = await manager.start("aa11");
context.refStore.set("e3", 99, {
tabId: 4,
frameId: "child",
cdpSessionId: "child-session",
kind: "surface",
visibleRect: { x: 120, y: 110, w: 100, h: 40 },
});
const graph = {
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 77,
target: { tabId: 4, sessionId: "child-session" },
},
],
};
const sent: Array<{ method: string; params?: object }> = [];
const cdp: CdpRunner = {
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
sent.push({ method, params });
if (method === "Page.getFrameTree") {
return {
frameTree: {
frame: { id: "main", loaderId: "loader-1", url: "https://fixture.test/" },
},
} as never;
}
if (method === "Page.getLayoutMetrics") {
return {
cssLayoutViewport: { clientWidth: 800, clientHeight: 600, pageX: 0, pageY: 0 },
cssVisualViewport: {
clientWidth: 800,
clientHeight: 600,
pageX: 0,
pageY: 0,
scale: 1,
},
} as never;
}
if (method === "DOM.getBoxModel") {
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } } as never;
}
if (method === "Input.dispatchMouseEvent") return {} as never;
throw new Error(`unexpected root CDP call ${method}`);
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "DOM.getContentQuads") {
return { quads: [[20, 10, 120, 10, 120, 50, 20, 50]] } as never;
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } } as never;
}
throw new Error(`unexpected child CDP call ${method}`);
}) as CdpRunner["sendToTarget"],
getFrameGraph: vi.fn(async () => graph),
trackSessionTab: vi.fn(),
};
const environment = await captureSurfaceEnvironment(cdp, 4, "child");
if ("code" in environment) throw new Error(environment.message);
const entry = context.refStore.resolveEntry("e3");
if (!entry) throw new Error("missing ref");
const capture = context.surfaceCaptures.create({
sessionId: "aa11",
tabId: 4,
navigationIdentity: environment.navigationIdentity,
surface: {
ref: "e3",
frameId: "child",
backendNodeId: 99,
observationGeneration: entry.generation,
},
topViewportRect: { x: 120, y: 110, w: 100, h: 40 },
imageWidth: 200,
imageHeight: 80,
viewportSignature: environment.viewportSignature,
frameProjectionSignature: environment.frameProjectionSignature,
});
const result = await resolveSurfacePointerTarget(
context,
{ tabId: 4 },
{
ref: "@e3",
captureId: capture.id,
imageX: 100,
imageY: 40,
},
{ cdp },
);
if ("code" in result) throw new Error(JSON.stringify(result));
expect(result).toMatchObject({ point: { x: 170, y: 130 } });
expect(sent.some((call) => call.method === "Input.dispatchMouseEvent")).toBe(false);
});
});
+67
View File
@@ -1,3 +1,4 @@
import type { DiagnosticSink } from "@/lib/diagnostics";
import { OVERLAY_AUTOMATION_BYPASS } from "@/lib/overlay-bridge";
import type { SessionManager } from "@/session-manager/manager";
import type { Transport } from "@/transport/transport";
@@ -118,6 +119,7 @@ export interface DispatcherDeps {
approveBorrow?: BorrowConfirmationApprover;
/** i18n notification copy for `tool.request_help` (resolved per-call). */
helpNotificationCopy?: () => { title: string; body: string };
diagnostics?: DiagnosticSink;
}
/**
@@ -145,6 +147,7 @@ export class ToolDispatcher {
private readonly onBrowserControlResumed?: (sessionId: string) => void;
private readonly approveBorrow?: BorrowConfirmationApprover;
private readonly helpNotificationCopy?: () => { title: string; body: string };
private readonly diagnostics?: DiagnosticSink;
private subscription: { dispose(): void } | null = null;
private readonly hoverBypassTabs = new Map<number, string>();
private readonly hoverLatches = new Map<number, HoverLatch>();
@@ -165,6 +168,7 @@ export class ToolDispatcher {
this.onBrowserControlResumed = deps.onBrowserControlResumed;
this.approveBorrow = deps.approveBorrow;
this.helpNotificationCopy = deps.helpNotificationCopy;
this.diagnostics = deps.diagnostics;
}
start(): void {
@@ -192,6 +196,16 @@ export class ToolDispatcher {
private async dispatch(msg: ProtocolFrame): Promise<void> {
if (!isRequestFrame(msg)) return;
const req = msg as RequestFrame;
const requestSessionId = (req.params as { session_id?: unknown } | undefined)?.session_id;
const startedAt = performance.now();
this.diagnostics?.("dispatcher.rpc.received", {
rpc_id: req.id,
method: req.method,
session_id: typeof requestSessionId === "string" ? requestSessionId : null,
transport_state: this.transport.state,
live_session_ids: this.sessions.list().map((ctx) => ctx.sessionId),
inflight_rpc_ids: Array.from(this.inflightAbortControllers.keys()),
});
// Cancel frames take a fast path: trip the matching controller
// (if any), reply with `{cancelled}` so the daemon can answer
@@ -200,6 +214,12 @@ export class ToolDispatcher {
const params = (req.params as { rpc_id?: string } | undefined) ?? {};
const target = typeof params.rpc_id === "string" ? params.rpc_id : "";
const ac = target ? this.inflightAbortControllers.get(target) : undefined;
this.diagnostics?.("dispatcher.cancel.received", {
rpc_id: req.id,
target_rpc_id: target,
target_found: ac !== undefined,
inflight_rpc_ids: Array.from(this.inflightAbortControllers.keys()),
});
if (ac) {
try {
ac.abort();
@@ -213,7 +233,17 @@ export class ToolDispatcher {
};
try {
this.transport.send(reply);
this.diagnostics?.("dispatcher.cancel.ack_sent", {
rpc_id: req.id,
target_rpc_id: target,
cancelled: ac !== undefined,
});
} catch (sendErr) {
this.diagnostics?.("dispatcher.cancel.ack_failed", {
rpc_id: req.id,
target_rpc_id: target,
error: sendErr,
});
console.warn("[bsk dispatcher] failed to ack cancel", sendErr);
}
return;
@@ -238,6 +268,13 @@ export class ToolDispatcher {
}
}
} catch (err) {
this.diagnostics?.("dispatcher.rpc.invoke_failed", {
rpc_id: req.id,
method: req.method,
session_id: typeof requestSessionId === "string" ? requestSessionId : null,
abort_signal: ac.signal.aborted,
error: err,
});
if (isAbortLikeError(err)) {
body = {
id: req.id,
@@ -258,8 +295,27 @@ export class ToolDispatcher {
let sent = true;
try {
this.transport.send(body);
this.diagnostics?.("dispatcher.rpc.response_sent", {
rpc_id: req.id,
method: req.method,
session_id: typeof requestSessionId === "string" ? requestSessionId : null,
response_kind: "error" in body ? "error" : "ok",
error_code: "error" in body ? body.error.code : null,
elapsed_ms: Math.round(performance.now() - startedAt),
transport_state: this.transport.state,
live_session_ids: this.sessions.list().map((ctx) => ctx.sessionId),
});
} catch (sendErr) {
sent = false;
this.diagnostics?.("dispatcher.rpc.response_send_failed", {
rpc_id: req.id,
method: req.method,
session_id: typeof requestSessionId === "string" ? requestSessionId : null,
started_session_id: startedSession,
elapsed_ms: Math.round(performance.now() - startedAt),
transport_state: this.transport.state,
error: sendErr,
});
// Transport is dead by the time we want to reply. Drop the link
// proactively so the alarm-driven keepalive reconnects sooner
// and the daemon's pending RPC times out cleanly instead of
@@ -278,12 +334,23 @@ export class ToolDispatcher {
try {
const ctx = await this.sessions.stop(startedSession);
if (ctx) {
this.diagnostics?.("dispatcher.session_start.rolled_back", {
rpc_id: req.id,
session_id: startedSession,
agent_window_id: ctx.agentWindowId,
reason: "response_send_failed",
});
console.warn(
"[bsk dispatcher] rolled back orphan session after send failure",
startedSession,
);
}
} catch (rollbackErr) {
this.diagnostics?.("dispatcher.session_start.rollback_failed", {
rpc_id: req.id,
session_id: startedSession,
error: rollbackErr,
});
console.warn("[bsk dispatcher] session rollback after send failure failed", rollbackErr);
}
}
+78 -82
View File
@@ -32,7 +32,6 @@ import { attachDialogs, markDialogCursor } from "./dialogs";
import { backendNodeToObject } from "./element-geometry";
import { rpcError } from "./errors";
import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
import { dispatchMouseClick } from "./mouse-input";
import {
type CdpRunner,
type ChromeTabsApi,
@@ -44,7 +43,6 @@ import {
resolveTargetTab,
} from "./shared";
import { resolveSnapshotRef } from "./snapshot-ref";
import { resolveSurfacePointerTarget } from "./surface-target";
export interface InteractionDeps {
cdp: CdpRunner;
@@ -108,6 +106,14 @@ function isAbortLikeError(err: unknown): boolean {
return err instanceof DOMException && err.name === "AbortError";
}
function isInvalidCssSelectorError(err: unknown): boolean {
const message = err instanceof Error ? err.message : String(err);
return (
message.includes("DOM Error while querying") ||
/invalid selector|not a valid selector|querySelector.*failed/i.test(message)
);
}
async function wait(ms: number, signal?: AbortSignal): Promise<void> {
if (signal?.aborted) throw new DOMException("aborted", "AbortError");
await new Promise<void>((resolve, reject) => {
@@ -161,7 +167,7 @@ async function resolveBackendNode(
};
}
if (hasRef) {
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId, "interact");
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
if (isRpcError(resolved)) return resolved;
return {
backendNodeId: resolved.backendNodeId,
@@ -210,6 +216,14 @@ async function resolveBackendNode(
}
return { backendNodeId, cdpTarget: { tabId: target.tabId }, usedSelector: params.selector };
} catch (err) {
if (isInvalidCssSelectorError(err)) {
return rpcError(
"invalid_params",
"invalid_selector",
`selector is not valid CSS: ${params.selector}`,
{ selector: params.selector },
);
}
return {
code: "cdp_failed",
message: err instanceof Error ? err.message : String(err),
@@ -221,25 +235,11 @@ async function resolveBackendNode(
// tool.click
// ---------------------------------------------------------------------------
interface ResolvedClickPointerTarget {
point: { x: number; y: number };
usedRef?: string;
usedSelector?: string;
moveSettleMs: number;
capture?: {
id: string;
imageX: number;
imageY: number;
};
}
export async function handleClick(
manager: SessionManager,
params: ClickParams,
deps: InteractionDeps = getDefaultDeps(),
): Promise<ClickResult | RpcError> {
const hasSurfacePointParams =
params.capture_id !== undefined || params.image_x !== undefined || params.image_y !== undefined;
const ctxOrErr = lookupSession(manager, params, "click");
if (isRpcError(ctxOrErr)) return ctxOrErr;
const ctx = ctxOrErr;
@@ -251,67 +251,39 @@ export async function handleClick(
if (denied) return denied;
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
const button: MouseButton = params.button ?? "left";
const clickCount = params.click_count ?? 1;
if (!Number.isSafeInteger(clickCount) || clickCount < 1) {
return { code: "invalid_params", message: "click_count must be a positive integer" };
}
const modifiers = modifiersBitfield(params.modifiers);
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "click");
if (isRpcError(node)) return node;
if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "click aborted" };
}
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
const pointerTarget: ResolvedClickPointerTarget | RpcError = hasSurfacePointParams
? await resolveSurfacePointerTarget(
ctx,
target,
{
ref: params.ref,
selector: params.selector,
captureId: params.capture_id,
imageX: params.image_x,
imageY: params.image_y,
},
{
cdp: deps.cdp,
signal: deps.signal,
},
)
: await (async () => {
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "click");
if (isRpcError(node)) return node;
const geometry = await resolveNodeGeometry(
deps.cdp,
target.tabId,
{
target: node.cdpTarget,
backendNodeId: node.backendNodeId,
...(node.frameId ? { frameId: node.frameId } : {}),
},
{ scrollIntoView: true },
);
if (isRpcError(geometry)) return geometry;
return {
point: geometry.actionPoint,
usedRef: node.usedRef,
usedSelector: node.usedSelector,
moveSettleMs: 0,
};
})();
if (isRpcError(pointerTarget)) return pointerTarget;
const geometry = await resolveNodeGeometry(
deps.cdp,
target.tabId,
{
target: node.cdpTarget,
backendNodeId: node.backendNodeId,
...(node.frameId ? { frameId: node.frameId } : {}),
},
{ scrollIntoView: true },
);
if (isRpcError(geometry)) return geometry;
const centre = geometry.actionPoint;
if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "click aborted" };
}
const overlayBlocking = await checkOverlayAtPoint(
deps.cdp,
target.tabId,
pointerTarget.point.x,
pointerTarget.point.y,
);
const button: MouseButton = params.button ?? "left";
const clickCount = params.click_count ?? 1;
if (clickCount < 1) {
return { code: "invalid_params", message: "click_count must be greater than zero" };
}
const modifiers = modifiersBitfield(params.modifiers);
const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, centre.x, centre.y);
let automationBypassEnabled = false;
if (overlayBlocking && deps.bypassOverlay) {
try {
@@ -323,16 +295,47 @@ export async function handleClick(
}
try {
const dispatch = await dispatchMouseClick(deps.cdp, target.tabId, pointerTarget.point, {
// Move first so hover state activates, then press → release.
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
type: "mouseMoved",
x: centre.x,
y: centre.y,
modifiers,
});
if (throwIfAborted(deps.signal)) {
return { code: "cancelled", message: "click aborted" };
}
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
type: "mousePressed",
x: centre.x,
y: centre.y,
button,
clickCount,
modifiers,
signal: deps.signal,
moveSettleMs: pointerTarget.moveSettleMs,
});
if (dispatch === "cancelled") {
if (throwIfAborted(deps.signal)) {
try {
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
type: "mouseReleased",
x: centre.x,
y: centre.y,
button,
clickCount,
modifiers,
});
} catch (err) {
console.debug("[bsk interaction] best-effort mouseReleased after abort failed", err);
}
return { code: "cancelled", message: "click aborted" };
}
await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", {
type: "mouseReleased",
x: centre.x,
y: centre.y,
button,
clickCount,
modifiers,
});
} catch (err) {
return {
code: "cdp_failed",
@@ -350,17 +353,10 @@ export async function handleClick(
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
tab_id: target.tabId,
used_ref: pointerTarget.usedRef,
used_selector: pointerTarget.usedSelector,
x: pointerTarget.point.x,
y: pointerTarget.point.y,
...(pointerTarget.capture
? {
capture_id: pointerTarget.capture.id,
image_x: pointerTarget.capture.imageX,
image_y: pointerTarget.capture.imageY,
}
: {}),
used_ref: node.usedRef,
used_selector: node.usedSelector,
x: centre.x,
y: centre.y,
});
}
-96
View File
@@ -1,96 +0,0 @@
import type { MouseButton } from "@/transport/types";
import type { Point } from "./geometry";
import type { CdpRunner } from "./shared";
interface MouseClickDispatchOptions {
button: MouseButton;
clickCount: number;
modifiers: number;
signal?: AbortSignal;
moveSettleMs?: number;
}
type MouseClickDispatchResult = "completed" | "cancelled";
function waitForSettle(ms: number, signal: AbortSignal | undefined): Promise<boolean> {
if (ms <= 0) return Promise.resolve(signal?.aborted !== true);
if (signal?.aborted) return Promise.resolve(false);
return new Promise((resolve) => {
let settled = false;
const finish = (completed: boolean) => {
if (settled) return;
settled = true;
clearTimeout(timer);
signal?.removeEventListener("abort", onAbort);
resolve(completed);
};
const timer = setTimeout(() => finish(true), ms);
const onAbort = () => finish(false);
signal?.addEventListener("abort", onAbort, { once: true });
});
}
/**
* Dispatch one coherent CDP mouse click in top-viewport CSS coordinates.
*
* Keeping the sequence here prevents DOM and Surface interaction paths from
* duplicating cancellation and best-effort release behavior. Callers that
* target asynchronously hit-tested content may request a settle interval.
*/
export async function dispatchMouseClick(
cdp: CdpRunner,
tabId: number,
point: Point,
options: MouseClickDispatchOptions,
): Promise<MouseClickDispatchResult> {
const { button, clickCount, modifiers, signal, moveSettleMs = 0 } = options;
let pressed = false;
const release = () =>
cdp.send(tabId, "Input.dispatchMouseEvent", {
type: "mouseReleased",
x: point.x,
y: point.y,
button,
clickCount,
modifiers,
});
try {
await cdp.send(tabId, "Input.dispatchMouseEvent", {
type: "mouseMoved",
x: point.x,
y: point.y,
modifiers,
});
if (!(await waitForSettle(moveSettleMs, signal))) return "cancelled";
await cdp.send(tabId, "Input.dispatchMouseEvent", {
type: "mousePressed",
x: point.x,
y: point.y,
button,
clickCount,
modifiers,
});
pressed = true;
if (signal?.aborted) {
await release();
pressed = false;
return "cancelled";
}
await release();
pressed = false;
return "completed";
} catch (error) {
if (pressed) {
try {
await release();
} catch (releaseError) {
console.debug("[bsk mouse-input] best-effort mouse release failed", releaseError);
}
}
throw error;
}
}
+7 -96
View File
@@ -8,7 +8,6 @@ import {
applyVomInteractionRecovery,
type CondSurface,
isVomReferenceNode,
type Rect,
renderVom,
type VomNode,
type VomOptions,
@@ -48,8 +47,6 @@ import {
type ToolEffect,
} from "./shared";
import { resolveSnapshotRef } from "./snapshot-ref";
import { surfaceVisibleRect } from "./surface-coordinate";
import { captureSurfaceEnvironment } from "./surface-target";
import {
type CapturedNode,
type CapturedViewModel,
@@ -62,11 +59,6 @@ import {
type CaptureVomObservationResult,
projectRecordSafeObservation,
} from "./vom/record-safe-observation";
import {
clusterRenderedSurfaces,
discoverRenderedSurfaces,
projectRenderedSurfaces,
} from "./vom/rendered-surfaces";
import {
buildSemanticGraph,
buildSemanticVomScene,
@@ -98,9 +90,6 @@ export const normaliseRef = sharedNormaliseRef;
// screenshot — `tool.screenshot`
// ---------------------------------------------------------------------------
const VISUAL_SURFACE_SCREENSHOT_MAX_EDGE = 2_048;
const VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS = 4_000_000;
/**
* Strip the `data:image/...;base64,` prefix from a Chrome
* `captureVisibleTab` dataURL and return the raw base64 payload.
@@ -193,42 +182,18 @@ async function captureElementScreenshot(
target: CdpTarget,
backendNodeId: number,
frameId?: string,
visualSurface = false,
observedVisibleRect?: Rect,
signal?: AbortSignal,
): Promise<
{ image_base64: string; width: number; height: number; topViewportRect: Rect } | RpcError
> {
): Promise<{ image_base64: string; width: number; height: number } | RpcError> {
if (signal?.aborted) return cancelled("screenshot");
const geometry = await resolveNodeGeometry(
cdp,
tabId,
{ target, backendNodeId, ...(frameId ? { frameId } : {}) },
{ scrollIntoView: !visualSurface },
{ scrollIntoView: true },
);
if (isRpcError(geometry)) return geometry;
if (signal?.aborted) return cancelled("screenshot");
const topViewportRect = surfaceVisibleRect(
geometry.topBounds,
visualSurface ? observedVisibleRect : undefined,
);
if (!topViewportRect) {
return { code: "permission_denied", message: "surface is no longer visible" };
}
const rect = {
x: topViewportRect.x,
y: topViewportRect.y,
width: topViewportRect.w,
height: topViewportRect.h,
};
const scale = visualSurface
? Math.min(
1,
VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.width,
VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.height,
Math.sqrt(VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS / (rect.width * rect.height)),
)
: 1;
const rect = geometry.topBounds;
try {
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
@@ -238,7 +203,7 @@ async function captureElementScreenshot(
y: rect.y,
width: rect.width,
height: rect.height,
scale,
scale: 1,
},
});
if (signal?.aborted) return cancelled("screenshot");
@@ -250,7 +215,7 @@ async function captureElementScreenshot(
width: Math.round(rect.width),
height: Math.round(rect.height),
};
return { image_base64, width: dims.width, height: dims.height, topViewportRect };
return { image_base64, width: dims.width, height: dims.height };
} catch (err) {
return {
code: "cdp_failed",
@@ -345,7 +310,7 @@ export async function handleScreenshot(
if (!deps.cdp) {
return { code: "cdp_failed", message: "screenshot ref capture requires CDP" };
}
const node = resolveSnapshotRef(ctx, ref, target.tabId, "screenshot");
const node = resolveSnapshotRef(ctx, ref, target.tabId);
if (isRpcError(node)) return node;
if (signal?.aborted) return cancelled("screenshot");
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
@@ -365,54 +330,18 @@ export async function handleScreenshot(
nodeTarget,
node.backendNodeId,
node.frameId,
node.kind === "surface",
node.visibleRect,
signal,
),
deps.sendToTab,
);
if (isRpcError(captured)) return captured;
if (signal?.aborted) return cancelled("screenshot");
const capture =
node.kind === "surface"
? await (async () => {
const environment = await captureSurfaceEnvironment(cdp, target.tabId, node.frameId);
if (isRpcError(environment)) return environment;
return ctx.surfaceCaptures.create({
sessionId: ctx.sessionId,
tabId: target.tabId,
navigationIdentity: environment.navigationIdentity,
surface: {
ref: node.refKey,
...(node.frameId ? { frameId: node.frameId } : {}),
backendNodeId: node.backendNodeId,
observationGeneration: node.generation,
},
topViewportRect: captured.topViewportRect,
imageWidth: captured.width,
imageHeight: captured.height,
viewportSignature: environment.viewportSignature,
frameProjectionSignature: environment.frameProjectionSignature,
});
})()
: null;
if (capture && isRpcError(capture)) return capture;
return withShotDialogs({
image_base64: captured.image_base64,
width: captured.width,
height: captured.height,
format: "png",
tab_id: target.tabId,
...(capture
? {
capture: {
id: capture.id,
surface_ref: `@${capture.surface.ref}`,
coordinate_space: "capture-image-pixel" as const,
expires_at: capture.expiresAt,
},
}
: {}),
});
}
@@ -1094,7 +1023,6 @@ async function captureForVom(
export interface CaptureVomObservationOptions extends VomOptions {
conditionalSurfaceProbe?: boolean;
renderedSurfaceDiscovery?: boolean;
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
signal?: AbortSignal;
}
@@ -1136,20 +1064,7 @@ export async function captureVomObservation(
),
);
const decoratedScene = attachCapturedSceneAnnotations(scene, normalizedDocuments, captured);
const observedScene = options.renderedSurfaceDiscovery
? projectRenderedSurfaces(
decoratedScene,
normalizedDocuments,
clusterRenderedSurfaces(
discoverRenderedSurfaces(
normalizedDocuments,
captured.viewport,
captured.excludedBackendNodeIds,
),
),
)
: decoratedScene;
const rendered = renderVom(observedScene, {
const rendered = renderVom(decoratedScene, {
maxDepth: options.maxDepth,
maxTokens: options.maxTokens,
redactValues: options.redactValues,
@@ -1200,7 +1115,6 @@ async function handleVomObservation(
maxTokens: params.max_tokens,
activeRegionPolicy: true,
conditionalSurfaceProbe: effectiveConditionalSurfaceProbe,
renderedSurfaceDiscovery: toolName === "observe",
hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay,
signal,
});
@@ -1218,9 +1132,6 @@ async function handleVomObservation(
tabId: target.tabId,
...(ref.frameId ? { frameId: ref.frameId } : {}),
...(refTarget?.sessionId ? { cdpSessionId: refTarget.sessionId } : {}),
...(ref.visibleRect ? { visibleRect: ref.visibleRect } : {}),
...(ref.kind ? { kind: ref.kind } : {}),
...(ref.capabilities ? { capabilities: ref.capabilities } : {}),
},
] as const;
}),
-1
View File
@@ -243,7 +243,6 @@ export async function handleSessionStop(
// Step 2: clear the per-session RefStore (review M6/M7 parity).
ctx.refStore.clear();
ctx.surfaceCaptures.clear();
clearRecordingForSession(params.session_id);
+1 -19
View File
@@ -2,9 +2,8 @@
// compound CDP node identity via the session RefStore, and build stable
// `ref_not_found` errors for hard-failure tool paths.
import type { Rect } from "@browser-skill/vom";
import type { SessionContext } from "@/session-manager/manager";
import { normaliseRef, type RefCapability, type RefTargetKind } from "@/session-manager/ref-store";
import { normaliseRef } from "@/session-manager/ref-store";
import type { RpcError } from "@/transport/types";
import { rpcError } from "./errors";
@@ -13,10 +12,6 @@ export interface SnapshotRefLookup {
refKey: string;
frameId?: string;
cdpSessionId?: string;
visibleRect?: Rect;
kind: RefTargetKind;
capabilities: RefCapability[];
generation: number;
}
function refEntryForTab(ctx: SessionContext, refKey: string, tabId: number) {
@@ -42,10 +37,6 @@ export function lookupSnapshotRef(
refKey,
...(entry.frameId ? { frameId: entry.frameId } : {}),
...(entry.cdpSessionId ? { cdpSessionId: entry.cdpSessionId } : {}),
...(entry.visibleRect ? { visibleRect: entry.visibleRect } : {}),
kind: entry.kind,
capabilities: entry.capabilities,
generation: entry.generation,
};
}
@@ -58,7 +49,6 @@ export function resolveSnapshotRef(
ctx: SessionContext,
ref: string,
tabId: number,
requiredCapability?: RefCapability,
): SnapshotRefLookup | RpcError {
const looked = lookupSnapshotRef(ctx, ref, tabId);
if (looked === null) {
@@ -68,13 +58,5 @@ export function resolveSnapshotRef(
`ref ${ref} unknown for tab ${tabId} in session ${ctx.sessionId}`,
);
}
if (requiredCapability && !looked.capabilities.includes(requiredCapability)) {
return rpcError(
"permission_denied",
"ref_capability_denied",
`ref ${ref} does not support ${requiredCapability}`,
{ ref: looked.refKey, required_capability: requiredCapability, kind: looked.kind },
);
}
return looked;
}
@@ -1,88 +0,0 @@
import type { Rect } from "@browser-skill/vom";
import type { Point, Region, ViewportRect } from "./geometry";
const RECT_EPSILON = 0.01;
export function surfaceVisibleRect(
liveRect: ViewportRect,
observedRect: Rect | undefined,
): Rect | null {
const observed = observedRect ?? {
x: liveRect.x,
y: liveRect.y,
w: liveRect.width,
h: liveRect.height,
};
const x = Math.max(liveRect.x, observed.x);
const y = Math.max(liveRect.y, observed.y);
const right = Math.min(liveRect.x + liveRect.width, observed.x + observed.w);
const bottom = Math.min(liveRect.y + liveRect.height, observed.y + observed.h);
return right > x && bottom > y ? { x, y, w: right - x, h: bottom - y } : null;
}
export function sameRect(a: Rect, b: Rect): boolean {
return (
Math.abs(a.x - b.x) <= RECT_EPSILON &&
Math.abs(a.y - b.y) <= RECT_EPSILON &&
Math.abs(a.w - b.w) <= RECT_EPSILON &&
Math.abs(a.h - b.h) <= RECT_EPSILON
);
}
export function mapImagePointToViewport(
rect: Rect,
imageWidth: number,
imageHeight: number,
imageX: number,
imageY: number,
): Point | null {
if (
!Number.isFinite(imageX) ||
!Number.isFinite(imageY) ||
!Number.isSafeInteger(imageWidth) ||
!Number.isSafeInteger(imageHeight) ||
imageWidth <= 0 ||
imageHeight <= 0 ||
imageX < 0 ||
imageY < 0 ||
imageX >= imageWidth ||
imageY >= imageHeight ||
!Number.isFinite(rect.x) ||
!Number.isFinite(rect.y) ||
!Number.isFinite(rect.w) ||
!Number.isFinite(rect.h) ||
rect.w <= 0 ||
rect.h <= 0
) {
return null;
}
return {
x: rect.x + (imageX / imageWidth) * rect.w,
y: rect.y + (imageY / imageHeight) * rect.h,
};
}
function pointInPolygon(point: Point, polygon: Point[]): boolean {
for (let index = 0; index < polygon.length; index += 1) {
const a = polygon[index];
const b = polygon[(index + 1) % polygon.length];
const cross = (point.x - a.x) * (b.y - a.y) - (point.y - a.y) * (b.x - a.x);
const withinX = point.x >= Math.min(a.x, b.x) - 1e-9 && point.x <= Math.max(a.x, b.x) + 1e-9;
const withinY = point.y >= Math.min(a.y, b.y) - 1e-9 && point.y <= Math.max(a.y, b.y) + 1e-9;
if (Math.abs(cross) <= 1e-9 && withinX && withinY) return true;
}
let inside = false;
for (let index = 0, previous = polygon.length - 1; index < polygon.length; previous = index++) {
const a = polygon[index];
const b = polygon[previous];
const intersects =
a.y > point.y !== b.y > point.y &&
point.x < ((b.x - a.x) * (point.y - a.y)) / (b.y - a.y) + a.x;
if (intersects) inside = !inside;
}
return inside;
}
export function pointInRegion(point: Point, region: Region): boolean {
return region.some((polygon) => polygon.length >= 3 && pointInPolygon(point, polygon));
}
-260
View File
@@ -1,260 +0,0 @@
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
import type { SessionContext } from "@/session-manager/manager";
import type { RpcError } from "@/transport/types";
import { rpcError } from "./errors";
import { resolveNodeGeometry } from "./frame-geometry";
import { type CdpRunner, isRpcError, normaliseRef } from "./shared";
import { resolveSnapshotRef } from "./snapshot-ref";
import {
mapImagePointToViewport,
pointInRegion,
sameRect,
surfaceVisibleRect,
} from "./surface-coordinate";
export interface SurfaceCaptureEnvironment {
navigationIdentity: string;
viewportSignature: string;
frameProjectionSignature: string;
}
export interface SurfaceTargetResolverDeps {
cdp: CdpRunner;
signal?: AbortSignal;
}
export interface SurfacePointerTargetInput {
ref?: string;
selector?: string;
captureId?: string;
imageX?: number;
imageY?: number;
}
export interface ResolvedSurfacePointerTarget {
point: { x: number; y: number };
usedRef: string;
moveSettleMs: number;
capture: {
id: string;
imageX: number;
imageY: number;
};
}
// Canvas-style controls commonly update their pointer hit-test on the next
// animation frame. Leave a scheduling boundary after moving and before pressing.
export const SURFACE_POINTER_MOVE_SETTLE_MS = 32;
function stale(message: string): RpcError {
return rpcError("permission_denied", "surface_capture_stale", message);
}
function finiteObject(value: unknown): unknown {
if (Array.isArray(value)) return value.map(finiteObject);
if (typeof value !== "object" || value === null) return value;
return Object.fromEntries(
Object.entries(value)
.filter(
([, entry]) => entry !== undefined && (typeof entry !== "number" || Number.isFinite(entry)),
)
.sort(([a], [b]) => a.localeCompare(b))
.map(([key, entry]) => [key, finiteObject(entry)]),
);
}
function framePathSignature(graph: CdpFrameGraph, frameId: string | undefined): string | null {
if (!frameId) return "top";
const frames = new Map(graph.frames.map((frame) => [frame.frameId, frame]));
const path: Array<{
frameId: string;
parentFrameId?: string;
ownerBackendNodeId?: number;
targetSessionId?: string;
}> = [];
const seen = new Set<string>();
let current = frames.get(frameId);
while (current) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
path.push({
frameId: current.frameId,
...(current.parentFrameId ? { parentFrameId: current.parentFrameId } : {}),
...(current.ownerBackendNodeId !== undefined
? { ownerBackendNodeId: current.ownerBackendNodeId }
: {}),
...(current.target.sessionId ? { targetSessionId: current.target.sessionId } : {}),
});
if (!current.parentFrameId) return JSON.stringify(path);
const parent = frames.get(current.parentFrameId);
if (!parent) return null;
current = parent;
}
return null;
}
export async function captureSurfaceEnvironment(
cdp: CdpRunner,
tabId: number,
frameId: string | undefined,
): Promise<SurfaceCaptureEnvironment | RpcError> {
try {
const [tree, metrics, graph] = await Promise.all([
cdp.send<{
frameTree?: { frame?: { id?: string; loaderId?: string; url?: string } };
}>(tabId, "Page.getFrameTree", {}),
cdp.send<Record<string, unknown>>(tabId, "Page.getLayoutMetrics", {}),
frameId && cdp.getFrameGraph ? cdp.getFrameGraph(tabId) : Promise.resolve(null),
]);
const root = tree.frameTree?.frame;
if (!root?.id || !root.loaderId) {
return { code: "cdp_failed", message: "could not identify the current navigation" };
}
const frameProjectionSignature = frameId
? graph
? framePathSignature(graph, frameId)
: null
: "top";
if (!frameProjectionSignature) {
return { code: "cdp_failed", message: `could not identify frame projection for ${frameId}` };
}
return {
navigationIdentity: JSON.stringify({ id: root.id, loaderId: root.loaderId, url: root.url }),
viewportSignature: JSON.stringify(
finiteObject({
cssLayoutViewport: metrics.cssLayoutViewport,
cssVisualViewport: metrics.cssVisualViewport,
}),
),
frameProjectionSignature,
};
} catch (error) {
return { code: "cdp_failed", message: error instanceof Error ? error.message : String(error) };
}
}
function validatePointParams(input: SurfacePointerTargetInput): RpcError | null {
if (!input.captureId || typeof input.captureId !== "string") {
return { code: "invalid_params", message: "surface point click requires capture_id" };
}
if (typeof input.imageX !== "number" || !Number.isFinite(input.imageX)) {
return { code: "invalid_params", message: "image_x must be a finite number" };
}
if (typeof input.imageY !== "number" || !Number.isFinite(input.imageY)) {
return { code: "invalid_params", message: "image_y must be a finite number" };
}
if (!input.ref || input.selector) {
return {
code: "invalid_params",
message: "surface point click requires one Surface ref and does not accept a selector",
};
}
return null;
}
function captureConsumeError(reason: "not_found" | "expired" | "consumed"): RpcError {
const code = reason === "not_found" ? "not_found" : "permission_denied";
return rpcError(code, `surface_capture_${reason}`, `surface capture is ${reason}`);
}
function aborted(signal: AbortSignal | undefined): RpcError | null {
return signal?.aborted
? { code: "cancelled", message: "surface target resolution aborted" }
: null;
}
/** Resolve one fresh Surface capture coordinate into a validated top-viewport pointer target. */
export async function resolveSurfacePointerTarget(
ctx: SessionContext,
target: { tabId: number; url?: string },
input: SurfacePointerTargetInput,
deps: SurfaceTargetResolverDeps,
): Promise<ResolvedSurfacePointerTarget | RpcError> {
const invalid = validatePointParams(input);
if (invalid) return invalid;
const earlyAbort = aborted(deps.signal);
if (earlyAbort) return earlyAbort;
const consumed = ctx.surfaceCaptures.consume(input.captureId as string);
if (!consumed.ok) return captureConsumeError(consumed.reason);
const capture = consumed.capture;
const postConsumeAbort = aborted(deps.signal);
if (postConsumeAbort) return postConsumeAbort;
if (capture.sessionId !== ctx.sessionId || capture.tabId !== target.tabId) {
return stale("surface capture belongs to a different session or tab");
}
if (normaliseRef(input.ref as string) !== capture.surface.ref) {
return stale("surface ref does not match the capture");
}
const node = resolveSnapshotRef(ctx, input.ref as string, target.tabId, "screenshot");
if (isRpcError(node)) return stale(node.message);
if (
node.kind !== "surface" ||
node.backendNodeId !== capture.surface.backendNodeId ||
node.frameId !== capture.surface.frameId ||
node.generation !== capture.surface.observationGeneration
) {
return stale("surface observation generation or node identity changed");
}
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
try {
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
} catch (error) {
return { code: "cdp_failed", message: error instanceof Error ? error.message : String(error) };
}
const environment = await captureSurfaceEnvironment(deps.cdp, target.tabId, node.frameId);
if (isRpcError(environment)) return environment;
if (
environment.navigationIdentity !== capture.navigationIdentity ||
environment.viewportSignature !== capture.viewportSignature ||
environment.frameProjectionSignature !== capture.frameProjectionSignature
) {
return stale("navigation, viewport, zoom, scroll, or frame projection changed");
}
const geometry = await resolveNodeGeometry(
deps.cdp,
target.tabId,
{
target: {
tabId: target.tabId,
...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}),
},
backendNodeId: node.backendNodeId,
...(node.frameId ? { frameId: node.frameId } : {}),
},
{ scrollIntoView: false },
);
if (isRpcError(geometry)) return stale(geometry.message);
const currentRect = surfaceVisibleRect(geometry.topBounds, node.visibleRect);
if (!currentRect || !sameRect(currentRect, capture.topViewportRect)) {
return stale("surface visible region changed");
}
const point = mapImagePointToViewport(
capture.topViewportRect,
capture.imageWidth,
capture.imageHeight,
input.imageX as number,
input.imageY as number,
);
if (!point || !pointInRegion(point, geometry.topVisibleRegions)) {
return rpcError(
"invalid_params",
"surface_coordinate_invalid",
"image coordinate is outside the captured Surface",
);
}
return {
point,
usedRef: capture.surface.ref,
moveSettleMs: SURFACE_POINTER_MOVE_SETTLE_MS,
capture: {
id: capture.id,
imageX: input.imageX as number,
imageY: input.imageY as number,
},
};
}
@@ -722,44 +722,6 @@ describe("captureViewModel", () => {
expect(nodes.find((node) => node.backendNodeId === 14)?.rendered).toBe(false);
});
it("propagates fully transparent ancestor suppression to painted descendants", async () => {
const S = ["html", "body", "div", "canvas", "static", "auto", "visible", "1", "0"];
const i = (value: string) => S.indexOf(value);
const snapshot = {
strings: S,
documents: [
{
nodes: {
parentIndex: [-1, 0, 1, 2],
nodeName: [i("html"), i("body"), i("div"), i("canvas")],
backendNodeId: [10, 11, 12, 13],
attributes: [[], [], [], []],
},
layout: {
nodeIndex: [1, 2, 3],
styles: [
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
[i("static"), i("auto"), i("auto"), i("visible"), i("0")],
[i("static"), i("auto"), i("auto"), i("visible"), i("1")],
],
bounds: [
[0, 0, 1000, 800],
[10, 10, 200, 100],
[10, 10, 200, 100],
],
paintOrders: [0, 1, 2],
},
},
],
};
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
const canvas = nodes.find((node) => node.backendNodeId === 13);
expect(canvas?.rendered).toBe(true);
expect(canvas?.visuallySuppressed).toBe(true);
});
it("subtracts scroll offset to make rects viewport-relative", async () => {
const cdp = {
send: vi.fn(async (_t: number, method: string) => {
@@ -1,256 +0,0 @@
import type { VomScene } from "@browser-skill/vom";
import { describe, expect, it } from "vitest";
import type { CapturedNode } from "../capture";
import type { CapturedFrameDocument } from "../frame-capture";
import {
clusterRenderedSurfaces,
discoverRenderedSurfaces,
projectRenderedSurfaces,
} from "../rendered-surfaces";
import type { SemanticAxNode } from "../semantic-graph";
function domNode(backendNodeId: number, overrides: Partial<CapturedNode> = {}): CapturedNode {
return {
backendNodeId,
parentBackendNodeId: null,
frameId: "main",
ownerFrameBackendNodeId: null,
tag: "canvas",
attrs: {},
rect: { x: 10, y: 20, w: 800, h: 500 },
localRect: { x: 10, y: 20, w: 800, h: 500 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
rendered: true,
...overrides,
};
}
function document(
domNodes: CapturedNode[],
axNodes: SemanticAxNode[] = [],
): CapturedFrameDocument<SemanticAxNode> {
return {
frameId: domNodes[0]?.frameId ?? "main",
contextScopeId: domNodes[0]?.frameId ?? "main",
target: { tabId: 4 },
domNodes,
axNodes,
};
}
describe("rendered surface discovery", () => {
it("finds prominent rendered canvases and excludes hidden descendants", () => {
const visible = domNode(2, { parentBackendNodeId: 1 });
const hidden = domNode(3, { parentBackendNodeId: 4 });
const surfaces = discoverRenderedSurfaces(
[
document([
domNode(1, { tag: "main", rect: null, localRect: null }),
visible,
domNode(4, { tag: "section", attrs: { "aria-hidden": "true" }, rect: null }),
hidden,
]),
],
{ width: 1280, height: 720 },
);
expect(surfaces).toHaveLength(1);
expect(surfaces[0]).toMatchObject({ backendNodeId: 2 });
});
it("keeps tiny canvases as individually addressable exact discoveries", () => {
const surfaces = discoverRenderedSurfaces(
[document([domNode(2, { rect: { x: 1, y: 1, w: 8, h: 8 } })])],
{ width: 1280, height: 720 },
);
expect(surfaces).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
});
it("keeps a surface when the canvas only has partial native accessibility content", () => {
const axNodes = [
{
nodeId: "canvas",
backendDOMNodeId: 2,
role: { type: "role", value: "presentation" },
childIds: ["button"],
frameId: "main",
},
{
nodeId: "button",
backendDOMNodeId: 3,
role: { type: "role", value: "button" },
frameId: "main",
},
] as SemanticAxNode[];
expect(
discoverRenderedSurfaces([document([domNode(2)], axNodes)], { width: 1280, height: 720 }),
).toEqual([expect.objectContaining({ backendNodeId: 2 })]);
});
it("discovers unnamed canvases without guessing labels from engineering attributes", () => {
const canvas = domNode(2, {
attrs: {
id: "must-not-be-a-label",
class: "must-not-be-a-label",
"data-testid": "must-not-be-a-label",
},
});
expect(discoverRenderedSurfaces([document([canvas])], { width: 1280, height: 720 })).toEqual([
expect.not.objectContaining({
label: expect.anything(),
}),
]);
});
it("treats blank explicit Canvas names as unnamed", () => {
expect(
discoverRenderedSurfaces(
[document([domNode(2, { attrs: { "aria-label": " ", title: "\n\t" } })])],
{ width: 1280, height: 720 },
),
).toEqual([expect.not.objectContaining({ label: expect.anything() })]);
});
it("excludes canvases inside BrowserSkill-owned DOM subtrees", () => {
const owner = domNode(1, { tag: "div", rect: null, localRect: null });
const canvas = domNode(2, { parentBackendNodeId: 1 });
expect(
discoverRenderedSurfaces(
[document([owner, canvas])],
{ width: 1280, height: 720 },
new Set([1]),
),
).toEqual([]);
});
it("excludes canvases suppressed by a fully transparent ancestor", () => {
const owner = domNode(1, {
tag: "div",
rect: { x: 0, y: 0, w: 900, h: 600 },
visuallySuppressed: true,
});
const canvas = domNode(2, { parentBackendNodeId: 1 });
expect(
discoverRenderedSurfaces([document([owner, canvas])], { width: 1280, height: 720 }),
).toEqual([]);
});
it("clips canvases to overflow ancestors and drops fully clipped canvases", () => {
const owner = domNode(1, {
tag: "div",
rect: { x: 10, y: 20, w: 100, h: 70 },
overflowX: "hidden",
overflowY: "hidden",
});
const partial = domNode(2, {
parentBackendNodeId: 1,
rect: { x: 55, y: 30, w: 140, h: 60 },
});
const fullyClipped = domNode(3, {
parentBackendNodeId: 1,
rect: { x: 140, y: 30, w: 80, h: 50 },
});
expect(
discoverRenderedSurfaces([document([owner, partial, fullyClipped])], {
width: 1280,
height: 720,
}),
).toEqual([
expect.objectContaining({
backendNodeId: 2,
visibleRect: { x: 55, y: 30, w: 55, h: 60 },
}),
]);
});
});
describe("rendered surface clustering and projection", () => {
it("collapses overlapping canvas layers and attaches them to the nearest retained parent", () => {
const body = domNode(1, { tag: "body", rect: null, localRect: null });
const surfaces = discoverRenderedSurfaces(
[
document([
body,
domNode(2, { parentBackendNodeId: 1, paintOrder: 1 }),
domNode(3, { parentBackendNodeId: 1, paintOrder: 2 }),
]),
],
{ width: 1280, height: 720 },
);
const groups = clusterRenderedSurfaces(surfaces);
const scene: VomScene = {
viewport: { width: 1280, height: 720 },
nodes: [
{
id: 10,
parentId: null,
backendNodeId: 1,
frameId: "main",
tag: "body",
role: "RootWebArea",
rect: null,
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
],
};
expect(groups).toHaveLength(1);
expect(groups[0]?.members).toHaveLength(2);
expect(groups[0]?.label).toBeUndefined();
expect(
projectRenderedSurfaces(
scene,
[document([body, ...surfaces.map((s) => domNode(s.backendNodeId))])],
groups,
),
).toMatchObject({
visualSurfaces: [expect.objectContaining({ parentId: 10, backendNodeId: 3, memberCount: 2 })],
});
});
it("does not merge a contained canvas with a larger independent canvas", () => {
const groups = clusterRenderedSurfaces([
...discoverRenderedSurfaces(
[
document([
domNode(1, { tag: "main", rect: null, localRect: null }),
domNode(2, { parentBackendNodeId: 1 }),
domNode(3, {
parentBackendNodeId: 1,
rect: { x: 30, y: 40, w: 200, h: 100 },
}),
]),
],
{ width: 1280, height: 720 },
),
]);
expect(groups).toHaveLength(2);
});
it("does not merge coincident canvases owned by different containers", () => {
const groups = clusterRenderedSurfaces(
discoverRenderedSurfaces(
[
document([
domNode(1, { tag: "section", rect: null, localRect: null }),
domNode(2, { tag: "section", rect: null, localRect: null }),
domNode(3, { parentBackendNodeId: 1 }),
domNode(4, { parentBackendNodeId: 2 }),
]),
],
{ width: 1280, height: 720 },
),
);
expect(groups).toHaveLength(2);
});
});
+1 -30
View File
@@ -11,15 +11,7 @@ import { childFrameProjection, type GeometryProjection, projectRectToViewport }
import type { CdpRunner } from "../shared";
import { clearHover, waitForHover } from "./hover-perception";
const REQUESTED_STYLES = [
"position",
"pointer-events",
"cursor",
"visibility",
"opacity",
"overflow-x",
"overflow-y",
] as const;
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
const STYLE_COL = Object.fromEntries(
REQUESTED_STYLES.map((name, index) => [name, index]),
) as Record<(typeof REQUESTED_STYLES)[number], number>;
@@ -47,10 +39,6 @@ export interface CapturedNode {
* `textContent`: the live parser always sets it, hand-built fixtures may not.
*/
cursor?: string;
overflowX?: string;
overflowY?: string;
/** True when this node or one of its DOM ancestors makes the subtree fully transparent/hidden. */
visuallySuppressed?: boolean;
/**
* Whether the live DOM snapshot provides a painted, non-hidden box for this
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
@@ -808,8 +796,6 @@ function parseDocumentNodes(
const cursorCol = STYLE_COL.cursor;
const visibilityCol = STYLE_COL.visibility;
const opacityCol = STYLE_COL.opacity;
const overflowXCol = STYLE_COL["overflow-x"];
const overflowYCol = STYLE_COL["overflow-y"];
const inputValues = sparseIndexMap(dn.inputValue);
const textValues = sparseIndexMap(dn.textValue);
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
@@ -835,7 +821,6 @@ function parseDocumentNodes(
// Build CapturedNodes.
const nodes: CapturedNode[] = [];
const visuallySuppressedByNodeIndex = new Array<boolean>(count).fill(false);
for (let n = 0; n < count; n++) {
if (excluded[n]) continue;
const backendNodeId = dn.backendNodeId[n];
@@ -855,11 +840,7 @@ function parseDocumentNodes(
let position = "static";
let pointerEvents = "auto";
let cursor = "auto";
let overflowX = "visible";
let overflowY = "visible";
let rendered = false;
let visuallySuppressed =
parentIdx >= 0 ? visuallySuppressedByNodeIndex[parentIdx] === true : false;
const li = layoutByNode.get(n);
if (li !== undefined) {
const b = dl?.bounds?.[li];
@@ -882,19 +863,12 @@ function parseDocumentNodes(
cursor = str(strings, styleRow[cursorCol]) || "auto";
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
const opacity = str(strings, styleRow[opacityCol]) || "1";
overflowX = str(strings, styleRow[overflowXCol]) || "visible";
overflowY = str(strings, styleRow[overflowYCol]) || "visible";
visuallySuppressed ||=
visibility === "hidden" ||
visibility === "collapse" ||
(Number.parseFloat(opacity) || 0) <= 0;
rendered =
localRect !== null &&
visibility !== "hidden" &&
visibility !== "collapse" &&
(Number.parseFloat(opacity) || 0) > 0;
}
visuallySuppressedByNodeIndex[n] = visuallySuppressed;
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
// geometry and are never queried by callers.
@@ -929,9 +903,6 @@ function parseDocumentNodes(
position,
pointerEvents,
cursor,
overflowX,
overflowY,
visuallySuppressed,
rendered,
textContent,
...(formValue !== undefined ? { formValue } : {}),
@@ -1,65 +0,0 @@
import type { Rect } from "@browser-skill/vom";
import type { RenderedSurface, RenderedSurfaceGroup } from "./types";
const STACK_IOU_RATIO = 0.9;
const STACK_SIZE_RATIO = 0.9;
function area(rect: Rect): number {
return Math.max(0, rect.w) * Math.max(0, rect.h);
}
function intersectionArea(a: Rect, b: Rect): number {
const width = Math.max(0, Math.min(a.x + a.w, b.x + b.w) - Math.max(a.x, b.x));
const height = Math.max(0, Math.min(a.y + a.h, b.y + b.h) - Math.max(a.y, b.y));
return width * height;
}
function sameVisualStack(a: RenderedSurface, b: RenderedSurface): boolean {
if (a.frameId !== b.frameId) return false;
if (a.parentBackendNodeId !== b.parentBackendNodeId) return false;
const aArea = area(a.visibleRect);
const bArea = area(b.visibleRect);
const intersection = intersectionArea(a.visibleRect, b.visibleRect);
const union = aArea + bArea - intersection;
const widthRatio =
Math.min(a.visibleRect.w, b.visibleRect.w) / Math.max(a.visibleRect.w, b.visibleRect.w);
const heightRatio =
Math.min(a.visibleRect.h, b.visibleRect.h) / Math.max(a.visibleRect.h, b.visibleRect.h);
return (
union > 0 &&
intersection / union >= STACK_IOU_RATIO &&
widthRatio >= STACK_SIZE_RATIO &&
heightRatio >= STACK_SIZE_RATIO
);
}
function toGroup(members: RenderedSurface[]): RenderedSurfaceGroup {
const representative = [...members].sort(
(a, b) => area(b.visibleRect) - area(a.visibleRect) || b.paintOrder - a.paintOrder,
)[0];
return {
frameId: representative.frameId,
members,
representative,
label: members.map((surface) => surface.label).find((label) => label !== undefined),
};
}
export function clusterRenderedSurfaces(surfaces: RenderedSurface[]): RenderedSurfaceGroup[] {
const groups: RenderedSurface[][] = [];
for (const surface of surfaces) {
const matching = groups.find((group) =>
group.some((member) => sameVisualStack(member, surface)),
);
if (matching) matching.push(surface);
else groups.push([surface]);
}
return groups
.map(toGroup)
.sort(
(a, b) =>
a.frameId.localeCompare(b.frameId) ||
a.representative.visibleRect.y - b.representative.visibleRect.y ||
a.representative.visibleRect.x - b.representative.visibleRect.x,
);
}
@@ -1,113 +0,0 @@
import type { Rect, Viewport } from "@browser-skill/vom";
import type { CapturedNode } from "../capture";
import type { CapturedFrameDocument } from "../frame-capture";
import type { SemanticAxNode } from "../semantic-graph";
import type { RenderedSurface } from "./types";
function clean(value: string | undefined): string | undefined {
const normalized = value?.replace(/\s+/g, " ").trim();
return normalized || undefined;
}
function intersectViewport(rect: Rect, viewport: Viewport): Rect | null {
return intersectRect(rect, { x: 0, y: 0, w: viewport.width, h: viewport.height });
}
function intersectRect(a: Rect, b: Rect): Rect | null {
const x = Math.max(a.x, b.x);
const y = Math.max(a.y, b.y);
const right = Math.min(a.x + a.w, b.x + b.w);
const bottom = Math.min(a.y + a.h, b.y + b.h);
if (right <= x || bottom <= y) return null;
return { x, y, w: right - x, h: bottom - y };
}
const CLIPPING_OVERFLOW = new Set(["auto", "clip", "hidden", "scroll"]);
function clipToOverflowAncestors(
node: CapturedNode,
rect: Rect,
nodesByBackend: ReadonlyMap<number, CapturedNode>,
): Rect | null {
let clipped: Rect | null = rect;
let parentBackendNodeId = node.parentBackendNodeId;
let guard = 0;
while (clipped && parentBackendNodeId !== null && guard <= nodesByBackend.size) {
const parent = nodesByBackend.get(parentBackendNodeId);
if (!parent) break;
if (parent.rect) {
const clipX = CLIPPING_OVERFLOW.has(parent.overflowX ?? "visible");
const clipY = CLIPPING_OVERFLOW.has(parent.overflowY ?? "visible");
if (clipX || clipY) {
const clipRect = {
x: clipX ? parent.rect.x : clipped.x,
y: clipY ? parent.rect.y : clipped.y,
w: clipX ? parent.rect.w : clipped.w,
h: clipY ? parent.rect.h : clipped.h,
};
clipped = intersectRect(clipped, clipRect);
}
}
parentBackendNodeId = parent.parentBackendNodeId;
guard += 1;
}
return clipped;
}
function excludedByDomPolicy(
node: CapturedNode,
nodesByBackend: Map<number, CapturedNode>,
excludedBackendNodeIds: ReadonlySet<number>,
): boolean {
let current: CapturedNode | undefined = node;
let guard = 0;
while (current && guard <= nodesByBackend.size) {
if (excludedBackendNodeIds.has(current.backendNodeId)) return true;
if (current.visuallySuppressed === true) return true;
const attrs = current.attrs;
if (
Object.prototype.hasOwnProperty.call(attrs, "hidden") ||
Object.prototype.hasOwnProperty.call(attrs, "inert") ||
(attrs["aria-hidden"] ?? "").toLowerCase() === "true"
) {
return true;
}
current =
current.parentBackendNodeId === null
? undefined
: nodesByBackend.get(current.parentBackendNodeId);
guard += 1;
}
return false;
}
export function discoverRenderedSurfaces(
documents: CapturedFrameDocument<SemanticAxNode>[],
viewport: Viewport,
excludedBackendNodeIds: ReadonlySet<number> = new Set(),
): RenderedSurface[] {
const surfaces: RenderedSurface[] = [];
for (const document of documents) {
const nodesByBackend = new Map(document.domNodes.map((node) => [node.backendNodeId, node]));
for (const node of document.domNodes) {
if (node.tag.toLowerCase() !== "canvas" || node.rendered !== true || !node.rect) continue;
if (excludedByDomPolicy(node, nodesByBackend, excludedBackendNodeIds)) continue;
const viewportRect = intersectViewport(node.rect, viewport);
const visibleRect = viewportRect
? clipToOverflowAncestors(node, viewportRect, nodesByBackend)
: null;
if (!visibleRect) continue;
const label = clean(node.attrs["aria-label"] ?? node.attrs.title);
surfaces.push({
renderingKind: "canvas",
frameId: document.frameId,
backendNodeId: node.backendNodeId,
parentBackendNodeId: node.parentBackendNodeId,
visibleRect,
paintOrder: node.paintOrder,
...(label ? { label } : {}),
});
}
}
return surfaces;
}
@@ -1,4 +0,0 @@
export { clusterRenderedSurfaces } from "./cluster";
export { discoverRenderedSurfaces } from "./discovery";
export { projectRenderedSurfaces } from "./project";
export type { RenderedSurface, RenderedSurfaceGroup } from "./types";
@@ -1,100 +0,0 @@
import type { VomScene, VomVisualSurface } from "@browser-skill/vom";
import type { CapturedFrameDocument } from "../frame-capture";
import type { SemanticAxNode } from "../semantic-graph";
import type { RenderedSurfaceGroup } from "./types";
function frameBackendKey(frameId: string, backendNodeId: number): string {
return `${frameId}\u0000${backendNodeId}`;
}
interface ProjectionIndex {
sceneByBackend: ReadonlyMap<string, number>;
documentByFrame: ReadonlyMap<string, CapturedFrameDocument<SemanticAxNode>>;
domByFrame: ReadonlyMap<
string,
ReadonlyMap<number, CapturedFrameDocument<SemanticAxNode>["domNodes"][number]>
>;
rootSceneIdByFrame: ReadonlyMap<string, number>;
}
function nearestSceneParent(index: ProjectionIndex, group: RenderedSurfaceGroup): number | null {
const document = index.documentByFrame.get(group.frameId);
const domByBackend = index.domByFrame.get(group.frameId) ?? new Map();
let parentBackendNodeId = group.representative.parentBackendNodeId;
let guard = 0;
while (parentBackendNodeId !== null && guard <= domByBackend.size) {
const sceneId = index.sceneByBackend.get(frameBackendKey(group.frameId, parentBackendNodeId));
if (sceneId !== undefined) return sceneId;
parentBackendNodeId = domByBackend.get(parentBackendNodeId)?.parentBackendNodeId ?? null;
guard += 1;
}
if (document?.parentFrameId && document.ownerBackendNodeId !== undefined) {
const owner = index.sceneByBackend.get(
frameBackendKey(document.parentFrameId, document.ownerBackendNodeId),
);
if (owner !== undefined) return owner;
}
return index.rootSceneIdByFrame.get(group.frameId) ?? null;
}
function buildProjectionIndex(
scene: VomScene,
documents: CapturedFrameDocument<SemanticAxNode>[],
): ProjectionIndex {
const rootSceneIdByFrame = new Map<string, number>();
for (const node of scene.nodes) {
if (
node.frameId &&
!rootSceneIdByFrame.has(node.frameId) &&
["rootwebarea", "webarea"].includes(node.role?.toLowerCase() ?? "")
) {
rootSceneIdByFrame.set(node.frameId, node.id);
}
}
return {
sceneByBackend: new Map(
scene.nodes.flatMap((node) =>
node.backendNodeId === undefined
? []
: [[frameBackendKey(node.frameId ?? "", node.backendNodeId), node.id] as const],
),
),
documentByFrame: new Map(documents.map((document) => [document.frameId, document])),
domByFrame: new Map(
documents.map((document) => [
document.frameId,
new Map(document.domNodes.map((node) => [node.backendNodeId, node])),
]),
),
rootSceneIdByFrame,
};
}
export function projectRenderedSurfaces(
scene: VomScene,
documents: CapturedFrameDocument<SemanticAxNode>[],
groups: RenderedSurfaceGroup[],
): VomScene {
const index = buildProjectionIndex(scene, documents);
const visualSurfaces: VomVisualSurface[] = [];
for (const group of groups) {
const parentId = nearestSceneParent(index, group);
visualSurfaces.push({
parentId,
backendNodeId: group.representative.backendNodeId,
frameId: group.frameId,
renderingKind: "canvas",
visibleRect: group.representative.visibleRect,
...(group.label ? { label: group.label } : {}),
memberCount: group.members.length,
});
}
return {
...scene,
...(visualSurfaces.length > 0 ? { visualSurfaces } : {}),
};
}
@@ -1,18 +0,0 @@
import type { Rect } from "@browser-skill/vom";
export interface RenderedSurface {
renderingKind: "canvas";
frameId: string;
backendNodeId: number;
parentBackendNodeId: number | null;
visibleRect: Rect;
paintOrder: number;
label?: string;
}
export interface RenderedSurfaceGroup {
frameId: string;
members: RenderedSurface[];
representative: RenderedSurface;
label?: string;
}
+1 -20
View File
@@ -23,8 +23,8 @@ export type ErrorCode =
export type RpcErrorReason =
| "agent_window_scope"
| "element_not_visible"
| "ref_capability_denied"
| "ref_not_found"
| "invalid_selector"
| "selector_not_found"
| "target_not_fillable"
| "target_not_select"
@@ -34,11 +34,6 @@ export type RpcErrorReason =
| "restricted_tab_url"
| "borrow_conflict"
| "screenshot_capture_failed"
| "surface_capture_not_found"
| "surface_capture_expired"
| "surface_capture_consumed"
| "surface_capture_stale"
| "surface_coordinate_invalid"
| "cleanup_failed";
export interface RpcErrorData {
@@ -304,17 +299,9 @@ export interface ScreenshotResult {
height: number;
format: string;
tab_id: number;
capture?: SurfaceCaptureInfo;
dialogs?: JavaScriptDialogInfo[];
}
export interface SurfaceCaptureInfo {
id: string;
surface_ref: string;
coordinate_space: "capture-image-pixel";
expires_at: number;
}
export interface SnapshotParams {
session_id: string;
tab_id?: number;
@@ -428,9 +415,6 @@ export interface ClickParams {
click_count?: number;
modifiers?: KeyModifier[];
timeout_ms?: number;
capture_id?: string;
image_x?: number;
image_y?: number;
}
export interface ClickResult {
@@ -439,9 +423,6 @@ export interface ClickResult {
used_selector?: string;
x: number;
y: number;
capture_id?: string;
image_x?: number;
image_y?: number;
dialogs?: JavaScriptDialogInfo[];
}
+54 -1
View File
@@ -1,3 +1,4 @@
import type { DiagnosticSink } from "@/lib/diagnostics";
import type { ConnectionStateHandler, Disposable, FrameHandler, Transport } from "./transport";
import type { ConnectionState, ProtocolFrame } from "./types";
@@ -18,6 +19,7 @@ export interface WSTransportOptions {
* Inject a fake WebSocket constructor in tests; defaults to `globalThis.WebSocket`.
*/
webSocketFactory?: WebSocketFactory;
diagnostics?: DiagnosticSink;
}
interface CloseLikeEvent {
@@ -44,6 +46,7 @@ export class WSTransport implements Transport {
private readonly factory: WebSocketFactory;
private readonly initialDelayMs: number;
private readonly maxDelayMs: number;
private readonly diagnostics?: DiagnosticSink;
private socket: WebSocket | null = null;
private currentState: ConnectionState = "disconnected";
@@ -63,6 +66,7 @@ export class WSTransport implements Transport {
this.factory = options.webSocketFactory ?? ((url: string) => new WebSocket(url));
this.initialDelayMs = options.reconnect?.initialDelayMs ?? DEFAULT_INITIAL_DELAY_MS;
this.maxDelayMs = options.reconnect?.maxDelayMs ?? DEFAULT_MAX_DELAY_MS;
this.diagnostics = options.diagnostics;
}
get state(): ConnectionState {
@@ -70,6 +74,13 @@ export class WSTransport implements Transport {
}
connect(): Promise<void> {
this.diagnostics?.("transport.connect.requested", {
state: this.currentState,
socket_generation: this.socketGeneration,
has_socket: this.socket !== null,
has_connecting_promise: this.connectingPromise !== null,
has_reconnect_timer: this.reconnectTimer !== null,
});
if (this.currentState === "connected") return Promise.resolve();
this.explicitlyClosed = false;
@@ -94,6 +105,11 @@ export class WSTransport implements Transport {
}
async disconnect(): Promise<void> {
this.diagnostics?.("transport.disconnect.requested", {
state: this.currentState,
socket_generation: this.socketGeneration,
has_socket: this.socket !== null,
});
this.explicitlyClosed = true;
if (this.reconnectTimer) {
clearTimeout(this.reconnectTimer);
@@ -124,6 +140,12 @@ export class WSTransport implements Transport {
send(msg: ProtocolFrame): void {
if (!this.socket || this.socket.readyState !== 1 /* OPEN */) {
this.diagnostics?.("transport.send.rejected", {
state: this.currentState,
socket_generation: this.socketGeneration,
socket_ready_state: this.socket?.readyState ?? null,
frame_kind: "method" in msg ? msg.method : "event" in msg ? msg.event : "response",
});
throw new Error("[WSTransport] cannot send while not connected");
}
this.socket.send(JSON.stringify(msg));
@@ -152,10 +174,16 @@ export class WSTransport implements Transport {
const generation = ++this.socketGeneration;
const socket = this.factory(this.url);
this.socket = socket;
this.diagnostics?.("transport.socket.created", {
socket_generation: generation,
reconnect_attempt: this.reconnectAttempt,
url: this.url,
});
socket.addEventListener("open", () => {
if (!this.isCurrentSocket(socket, generation)) return;
this.reconnectAttempt = 0;
this.diagnostics?.("transport.socket.open", { socket_generation: generation });
this.setState("connected");
const resolve = this.resolveConnect;
this.resolveConnect = null;
@@ -174,6 +202,7 @@ export class WSTransport implements Transport {
});
socket.addEventListener("error", () => {
this.diagnostics?.("transport.socket.error", { socket_generation: generation });
// The 'close' handler always fires after 'error' in browsers, so we
// just record the error and let close drive reconnect logic.
});
@@ -197,7 +226,16 @@ export class WSTransport implements Transport {
}
private handleClose(socket: WebSocket, generation: number, _ev: CloseLikeEvent): void {
if (!this.isCurrentSocket(socket, generation)) return;
const isCurrent = this.isCurrentSocket(socket, generation);
this.diagnostics?.("transport.socket.close", {
socket_generation: generation,
current_socket_generation: this.socketGeneration,
is_current: isCurrent,
explicitly_closed: this.explicitlyClosed,
close_code: _ev.code ?? null,
close_reason: _ev.reason ?? "",
});
if (!isCurrent) return;
this.socket = null;
if (this.explicitlyClosed) {
this.setState("disconnected");
@@ -211,9 +249,18 @@ export class WSTransport implements Transport {
if (this.reconnectTimer) return;
const delay = Math.min(this.initialDelayMs * 2 ** this.reconnectAttempt, this.maxDelayMs);
this.reconnectAttempt += 1;
this.diagnostics?.("transport.reconnect.scheduled", {
delay_ms: delay,
reconnect_attempt: this.reconnectAttempt,
socket_generation: this.socketGeneration,
});
this.reconnectTimer = setTimeout(() => {
this.reconnectTimer = null;
if (this.explicitlyClosed) return;
this.diagnostics?.("transport.reconnect.timer_fired", {
reconnect_attempt: this.reconnectAttempt,
socket_generation: this.socketGeneration,
});
void this.connect().catch((err) => {
console.debug("[WSTransport] reconnect attempt failed", err);
});
@@ -226,7 +273,13 @@ export class WSTransport implements Transport {
private setState(next: ConnectionState): void {
if (this.currentState === next) return;
const previous = this.currentState;
this.currentState = next;
this.diagnostics?.("transport.state.changed", {
previous,
next,
socket_generation: this.socketGeneration,
});
for (const h of this.stateHandlers) {
try {
h(next);
@@ -1,113 +0,0 @@
# Rendered Surface browser fixtures
These pages exercise Canvas discovery through a real browser DOMSnapshot and AX tree. They are deliberately framework- and site-independent.
## Serve
From the repository root:
```bash
python3 -m http.server 4173 --bind 127.0.0.1 \
--directory apps/extension/test-fixtures/rendered-surfaces
```
Use an Agent Window large enough to keep the primary matrix visible:
```bash
bsk session start --width 1440 --height 1100
bsk navigate http://127.0.0.1:4173/index.html --session <id>
bsk observe --session <id>
```
Always stop the session after the run.
## Pages
### `index.html`
Expected individual Surface labels:
- `fixture-basic-large`
- `fixture-tiny-8px`
- `fixture-pointer-events-none`
- `fixture-partial-ax`
- `fixture-contained-outer`
- `fixture-contained-inner`
- `fixture-different-parent-a`
- `fixture-different-parent-b`
Expected one grouped Surface:
- `fixture-layered-group`, with `layers=2`
Expected absent:
- `fixture-display-none`
- `fixture-visibility-hidden`
- `fixture-opacity-zero`
- `fixture-parent-opacity-zero`
- `fixture-hidden-attribute`
- `fixture-inert-parent`
- `fixture-aria-hidden-parent`
- `fixture-zero-size`
- `fixture-offscreen-right`
- `fixture-fully-overflow-clipped`
`fixture-partially-overflow-clipped` must remain present because part of it is visible. Its screenshot must contain only the browser's final composited view.
### `unlabeled.html`
This page contains eight Canvas elements but must render seven generic Surface markers because
the two coincident layers form one group.
- Every marker uses the stable fallback label `canvas visual surface`.
- Exactly one marker has `layers=2`.
- The two contained canvases remain separate markers.
- `id`, `class`, `data-testid`, nearby prose, and fallback AX content never become a Surface label.
- Blank `aria-label` and `title` values behave as no label.
- Fallback AX content does not suppress the corresponding Surface.
### `frames.html`
Expected Surface labels:
- `fixture-same-origin-frame`
- `fixture-cross-origin-frame`
- `fixture-nested-frame`
The cross-origin frame uses `http://localhost:4173` while the parent uses `http://127.0.0.1:4173`. With Chromium site isolation this exercises the OOPIF path without an external service.
Each frame Canvas also exposes a `pointer status` live region. A successful
screenshot-bound point click must report
`down button=0 buttons=1 hoverReady=true; click received` and turn the Canvas
yellow. The hover flag is set on the animation frame after `pointermove`, so the
fixture verifies both child-document delivery and the scheduling boundary
required before `pointerdown`, rather than merely validating projected
coordinates or a successful CDP response.
### `viewport.html`
Run with a `900x700` Agent Window.
- `fixture-partial-viewport` is partly visible and must be represented.
- `fixture-below-fold` is absent initially.
- After scrolling it into view and observing again, `fixture-below-fold` must be represented with a fresh ref.
### `dynamic.html`
Use the normal DOM buttons and re-observe after each action:
- Initial state: `fixture-dynamic` absent.
- After **Insert canvas**: present.
- After **Resize to zero**: absent.
- After **Restore size**: present with a fresh ref.
- After **Remove canvas**: absent.
## Invariants
- `snapshot` must not emit any of these Surface markers.
- `observe` must emit only currently visible Canvas surfaces.
- Existing DOM/AX controls remain readable alongside Surface markers.
- Missing labels never prevent discovery; naming and discovery are independent.
- A Surface ref allows `screenshot --ref` and rejects ordinary click/fill/hover/select.
- No fixture URL, id, label, or layout constant may appear in production discovery code.
@@ -1,46 +0,0 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Rendered Surface dynamic matrix</title>
<style>
html, body { margin: 0; font: 14px system-ui, sans-serif; }
body { padding: 16px; }
.controls { display: flex; gap: 8px; margin-bottom: 16px; }
button { padding: 8px 12px; }
canvas { display: block; width: 260px; height: 120px; background: #e8daef; border: 2px solid #7d3c98; }
</style>
</head>
<body>
<h1>Dynamic Canvas lifecycle</h1>
<div class="controls">
<button id="insert" type="button">Insert canvas</button>
<button id="zero" type="button">Resize to zero</button>
<button id="restore" type="button">Restore size</button>
<button id="remove" type="button">Remove canvas</button>
</div>
<div id="mount"></div>
<script>
const mount = document.querySelector("#mount");
const getCanvas = () => mount.querySelector("canvas");
document.querySelector("#insert").addEventListener("click", () => {
if (getCanvas()) return;
const canvas = document.createElement("canvas");
canvas.width = 260;
canvas.height = 120;
canvas.setAttribute("aria-label", "fixture-dynamic");
mount.append(canvas);
});
document.querySelector("#zero").addEventListener("click", () => {
const canvas = getCanvas();
if (canvas) canvas.style.cssText = "width:0;height:0;border:0";
});
document.querySelector("#restore").addEventListener("click", () => {
const canvas = getCanvas();
if (canvas) canvas.removeAttribute("style");
});
document.querySelector("#remove").addEventListener("click", () => getCanvas()?.remove());
</script>
</body>
</html>
@@ -1,52 +0,0 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<style>
html, body { margin: 0; font: 14px system-ui, sans-serif; }
body { padding: 12px; background: #fdfefe; }
canvas { display: block; width: 220px; height: 90px; background: #d5f5e3; border: 2px solid #1e8449; }
iframe { width: 280px; height: 150px; margin-top: 10px; border: 2px solid #85929e; }
#pointer-status { margin: 8px 0 0; min-height: 20px; }
</style>
</head>
<body>
<canvas id="frame-canvas" width="220" height="90"></canvas>
<script>
const params = new URLSearchParams(location.search);
const canvas = document.querySelector("#frame-canvas");
if (params.get("canvas") === "0") canvas.remove();
else {
canvas.setAttribute("aria-label", params.get("label") || "fixture-frame-default");
if (params.get("interactive") === "1") {
const status = document.createElement("p");
status.id = "pointer-status";
status.setAttribute("aria-live", "polite");
status.textContent = "pointer status: idle";
canvas.insertAdjacentElement("afterend", status);
let hoverReady = false;
canvas.addEventListener("pointermove", () => {
hoverReady = false;
requestAnimationFrame(() => {
hoverReady = true;
});
});
canvas.addEventListener("pointerdown", (event) => {
status.textContent = `pointer status: down button=${event.button} buttons=${event.buttons} hoverReady=${hoverReady}`;
if (event.buttons === 1 && hoverReady) canvas.style.background = "#f9e79f";
});
canvas.addEventListener("click", () => {
status.textContent += "; click received";
});
}
}
if (params.get("nested") === "1") {
const iframe = document.createElement("iframe");
iframe.title = "nested fixture frame";
iframe.src = "frame.html?label=fixture-nested-frame&interactive=1";
document.body.append(iframe);
}
</script>
</body>
</html>
@@ -1,22 +0,0 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Rendered Surface frame matrix</title>
<style>
html, body { margin: 0; font: 14px system-ui, sans-serif; }
body { padding: 16px; }
.frames { display: grid; grid-template-columns: repeat(3, 320px); gap: 16px; }
iframe { width: 300px; height: 300px; border: 2px solid #5d6d7e; }
</style>
</head>
<body>
<h1>Frame matrix</h1>
<div class="frames">
<iframe title="same-origin fixture" src="frame.html?label=fixture-same-origin-frame&interactive=1"></iframe>
<iframe title="cross-origin fixture" src="http://localhost:4173/frame.html?label=fixture-cross-origin-frame&interactive=1"></iframe>
<iframe title="nested fixture" src="frame.html?canvas=0&nested=1"></iframe>
</div>
</body>
</html>
@@ -1,118 +0,0 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Rendered Surface discovery matrix</title>
<style>
* { box-sizing: border-box; }
html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
body { padding: 16px; background: #f4f6f8; color: #17202a; }
h1 { margin: 0 0 12px; font-size: 20px; }
.matrix { display: grid; grid-template-columns: repeat(4, minmax(220px, 1fr)); gap: 12px; }
.case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
.case h2 { margin: 0 0 8px; font-size: 13px; }
canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
.stack { position: relative; width: 180px; height: 90px; }
.stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
.stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
.contained { position: relative; width: 190px; height: 100px; }
.contained > canvas:first-child { width: 190px; height: 100px; }
.contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
.same-place { position: relative; width: 180px; height: 90px; }
.same-place > div { position: absolute; inset: 0; }
.same-place canvas { width: 180px; height: 90px; }
.same-place > div:last-child canvas { border-color: #7d3c98; background: rgb(210 180 222 / 35%); }
.clip-box { width: 100px; height: 70px; overflow: hidden; border: 2px dashed #566573; }
.partially-clipped { width: 140px; height: 60px; transform: translateX(45px); }
.fully-clipped { width: 80px; height: 50px; transform: translateX(130px); }
.tiny { width: 8px; height: 8px; border-width: 1px; }
</style>
</head>
<body>
<h1>Rendered Surface discovery matrix</h1>
<main class="matrix">
<section class="case">
<h2>Visible named canvas</h2>
<canvas aria-label="fixture-basic-large" width="200" height="90"></canvas>
</section>
<section class="case">
<h2>Tiny canvas remains addressable</h2>
<canvas class="tiny" aria-label="fixture-tiny-8px" width="8" height="8"></canvas>
</section>
<section class="case">
<h2>Visible but pointer-events none</h2>
<canvas aria-label="fixture-pointer-events-none" width="180" height="80" style="pointer-events: none"></canvas>
</section>
<section class="case">
<h2>Partial AX/fallback content</h2>
<canvas aria-label="fixture-partial-ax" width="180" height="80">
<button type="button">Fallback button must not suppress the surface</button>
</canvas>
</section>
<section class="case">
<h2>Two real rendering layers</h2>
<div class="stack">
<canvas aria-label="fixture-layered-group" width="180" height="90"></canvas>
<canvas width="180" height="90"></canvas>
</div>
</section>
<section class="case">
<h2>Contained canvases are independent</h2>
<div class="contained">
<canvas aria-label="fixture-contained-outer" width="190" height="100"></canvas>
<canvas aria-label="fixture-contained-inner" width="70" height="40"></canvas>
</div>
</section>
<section class="case">
<h2>Same geometry, different parents</h2>
<div class="same-place">
<div><canvas aria-label="fixture-different-parent-a" width="180" height="90"></canvas></div>
<div><canvas aria-label="fixture-different-parent-b" width="180" height="90"></canvas></div>
</div>
</section>
<section class="case">
<h2>Partially overflow-clipped</h2>
<div class="clip-box">
<canvas class="partially-clipped" aria-label="fixture-partially-overflow-clipped" width="140" height="60"></canvas>
</div>
</section>
<section class="case">
<h2>Fully overflow-clipped</h2>
<div class="clip-box">
<canvas class="fully-clipped" aria-label="fixture-fully-overflow-clipped" width="80" height="50"></canvas>
</div>
</section>
<section class="case">
<h2>CSS-hidden variants</h2>
<canvas aria-label="fixture-display-none" width="80" height="40" style="display: none"></canvas>
<canvas aria-label="fixture-visibility-hidden" width="80" height="40" style="visibility: hidden"></canvas>
<canvas aria-label="fixture-opacity-zero" width="80" height="40" style="opacity: 0"></canvas>
</section>
<section class="case">
<h2>Hidden ancestor variants</h2>
<div style="opacity: 0"><canvas aria-label="fixture-parent-opacity-zero" width="80" height="40"></canvas></div>
<div hidden><canvas aria-label="fixture-hidden-attribute" width="80" height="40"></canvas></div>
<div inert><canvas aria-label="fixture-inert-parent" width="80" height="40"></canvas></div>
<div aria-hidden="true"><canvas aria-label="fixture-aria-hidden-parent" width="80" height="40"></canvas></div>
</section>
<section class="case">
<h2>Zero-size and offscreen</h2>
<canvas aria-label="fixture-zero-size" width="80" height="40" style="width: 0; height: 0; border: 0"></canvas>
<canvas aria-label="fixture-offscreen-right" width="80" height="40" style="position: fixed; left: 5000px; top: 0"></canvas>
</section>
</main>
</body>
</html>
@@ -1,68 +0,0 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Unlabeled Rendered Surface matrix</title>
<style>
* { box-sizing: border-box; }
html, body { margin: 0; min-height: 100%; font: 14px/1.4 system-ui, sans-serif; }
body { padding: 16px; background: #f4f6f8; color: #17202a; }
h1 { margin: 0 0 12px; font-size: 20px; }
.matrix { display: grid; grid-template-columns: repeat(3, minmax(240px, 1fr)); gap: 12px; }
.case { min-height: 150px; padding: 10px; overflow: hidden; border: 1px solid #aeb6bf; border-radius: 6px; background: white; }
.case h2 { margin: 0 0 8px; font-size: 13px; }
.case p { margin: 0 0 8px; }
canvas { display: block; background: #d6eaf8; border: 2px solid #2874a6; }
.stack { position: relative; width: 180px; height: 90px; }
.stack canvas { position: absolute; inset: 0; width: 180px; height: 90px; }
.stack canvas + canvas { background: rgb(244 208 63 / 45%); border-color: #b7950b; }
.contained { position: relative; width: 190px; height: 100px; }
.contained > canvas:first-child { width: 190px; height: 100px; }
.contained > canvas:last-child { position: absolute; left: 55px; top: 30px; width: 70px; height: 40px; background: #fadbd8; border-color: #c0392b; }
</style>
</head>
<body>
<h1>Unlabeled Rendered Surface matrix</h1>
<main class="matrix">
<section class="case">
<h2>Only engineering attributes</h2>
<canvas id="must-not-be-a-label" class="must-not-be-a-label" data-testid="must-not-be-a-label" width="200" height="90"></canvas>
</section>
<section class="case">
<h2>Blank explicit names</h2>
<canvas aria-label=" " title=" " width="180" height="80"></canvas>
</section>
<section class="case">
<h2>Nearby prose is not a Canvas name</h2>
<p>must-not-be-inferred-as-a-surface-label</p>
<canvas width="180" height="80"></canvas>
</section>
<section class="case">
<h2>Fallback AX content does not name or suppress the Canvas</h2>
<canvas width="180" height="80">
<button type="button">Fallback control remains separate</button>
</canvas>
</section>
<section class="case">
<h2>Unnamed rendering layers still cluster</h2>
<div class="stack">
<canvas width="180" height="90"></canvas>
<canvas width="180" height="90"></canvas>
</div>
</section>
<section class="case">
<h2>Unnamed contained canvases stay independent</h2>
<div class="contained">
<canvas width="190" height="100"></canvas>
<canvas width="70" height="40"></canvas>
</div>
</section>
</main>
</body>
</html>
@@ -1,22 +0,0 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Rendered Surface viewport matrix</title>
<style>
html, body { margin: 0; font: 14px system-ui, sans-serif; }
body { min-height: 1800px; padding: 16px; }
canvas { display: block; background: #fdebd0; border: 2px solid #ca6f1e; }
#partial { position: fixed; left: -70px; top: 120px; width: 180px; height: 100px; }
#below { position: absolute; left: 40px; top: 1350px; width: 240px; height: 120px; }
</style>
</head>
<body>
<h1>Viewport matrix</h1>
<p>The orange canvas on the left is intentionally clipped by the viewport.</p>
<canvas id="partial" aria-label="fixture-partial-viewport" width="180" height="100"></canvas>
<canvas id="below" aria-label="fixture-below-fold" width="240" height="120"></canvas>
</body>
</html>
+3 -2
View File
@@ -22,7 +22,8 @@ export default defineConfig({
outDir: "dist",
modules: ["@wxt-dev/module-react"],
manifest: {
name: "BrowserSkill",
name: "BrowserSkill (Lumi Diagnostics)",
version_name: `${EXTENSION_VERSION}-lumi-session-loss-v2`,
description:
"Let AI agents use your logged-in browser in a separate Agent Window—without interrupting your work. Powered by the bsk CLI.",
// Flat debugger sessions are required to address out-of-process iframes.
@@ -45,7 +46,7 @@ export default defineConfig({
128: "icon/logo.png",
},
action: {
default_title: "BrowserSkill",
default_title: "BrowserSkill (Lumi Diagnostics)",
default_icon: {
16: "icon/logo.png",
32: "icon/logo.png",
+2 -12
View File
@@ -80,16 +80,6 @@ Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` /
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
When `bsk observe` renders `@eN surface ... [visual-only; requires=image-understanding; ...]`, the ref identifies rendered canvas content that is not represented by the text observation, not an interactable DOM control. Keep using any reliable DOM/AX text and controls that appear alongside it. If you lack multimodal or image-reading capability, do not take a screenshot and pretend to know its contents, and never guess coordinates; tell the user that they need to switch to a model with image-understanding capability.
If you can actually inspect image output, use `bsk screenshot --ref @eN --session <id>` to obtain the visible crop and its short-lived, single-use Surface capture id. Only when the task requires a point action and the image provides a clear target, bind that exact screenshot to the same general click command:
```bash
bsk click @eN --capture <capture-id> --image-x <px> --image-y <px> --session <id>
```
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/press workflow. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/hover/select reject them.
## Observation priority
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
@@ -184,7 +174,7 @@ bsk emulate --session <id> --off
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional); Surface crops return a short-lived capture id |
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
### Console & network debugging (read-only; require `--session`)
@@ -210,7 +200,7 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
| Command | Summary |
|---------|---------|
| `bsk click <ref-or-selector>` | Click a DOM element (`--button`, `--click-count`, `--modifiers`), or perform one screenshot-bound Surface point click with `--capture`, `--image-x`, and `--image-y` |
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
+17
View File
@@ -497,4 +497,21 @@ mod tests {
.contains("CSS selector")
);
}
#[test]
fn invalid_selector_explains_css_contract_in_json() {
let cli = CliError::from_rpc(RpcError {
code: ErrorCode::InvalidParams,
message: "selector is not valid CSS: text=submit".into(),
data: Some(serde_json::json!({ "reason": "invalid_selector" })),
});
let exit = cli.exit_code();
let hint = hint_for(&cli, render_info_for(&cli).as_ref());
let json = json_error_string(&cli, exit, hint);
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
let hint = parsed.get("hint").and_then(|v| v.as_str()).unwrap();
assert!(hint.contains("CSS selector"));
assert!(hint.contains("text="));
assert!(hint.contains("@eN"));
}
}
-31
View File
@@ -136,18 +136,6 @@ pub struct ClickArgs {
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
pub timeout: u32,
/// Capture id returned by `bsk screenshot --ref <surface>`.
#[arg(long)]
pub capture: Option<String>,
/// X coordinate in capture-image pixels. Requires `--capture` and `--image-y`.
#[arg(long = "image-x", allow_hyphen_values = true)]
pub image_x: Option<f64>,
/// Y coordinate in capture-image pixels. Requires `--capture` and `--image-x`.
#[arg(long = "image-y", allow_hyphen_values = true)]
pub image_y: Option<f64>,
}
pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
@@ -159,14 +147,6 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
)?;
let modifiers = parse_modifiers(&args.modifiers)
.map_err(|e| CliError::Local(anyhow::anyhow!("--modifiers: {e}")))?;
let point_arg_count = usize::from(args.capture.is_some())
+ usize::from(args.image_x.is_some())
+ usize::from(args.image_y.is_some());
if point_arg_count != 0 && point_arg_count != 3 {
return Err(CliError::Local(anyhow::anyhow!(
"--capture, --image-x, and --image-y must be given together"
)));
}
let params = ClickParams {
session_id: args.session.clone(),
ref_,
@@ -180,9 +160,6 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
Some(modifiers)
},
timeout_ms: Some(args.timeout),
capture_id: args.capture.clone(),
image_x: args.image_x,
image_y: args.image_y,
};
let reply: ClickResult = call(
info.sock_path,
@@ -206,14 +183,6 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
"click ok tab={} target={target} at=({}, {})",
reply.tab_id, reply.x, reply.y
);
if let Some(capture_id) = &reply.capture_id {
println!(
"capture={} image=({}, {}) coordinate_space=capture-image-pixel",
capture_id,
reply.image_x.unwrap_or_default(),
reply.image_y.unwrap_or_default()
);
}
print_dialog_summaries(&reply.dialogs);
}
}
+8 -79
View File
@@ -40,8 +40,8 @@ use bsk_protocol::ErrorCode;
pub mod reason {
pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope";
pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible";
pub const REF_CAPABILITY_DENIED: &str = "ref_capability_denied";
pub const REF_NOT_FOUND: &str = "ref_not_found";
pub const INVALID_SELECTOR: &str = "invalid_selector";
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
pub const TARGET_NOT_SELECT: &str = "target_not_select";
@@ -50,11 +50,6 @@ pub mod reason {
pub const TAB_NOT_ACTIVE: &str = "tab_not_active";
pub const BORROW_CONFLICT: &str = "borrow_conflict";
pub const SCREENSHOT_CAPTURE_FAILED: &str = "screenshot_capture_failed";
pub const SURFACE_CAPTURE_NOT_FOUND: &str = "surface_capture_not_found";
pub const SURFACE_CAPTURE_EXPIRED: &str = "surface_capture_expired";
pub const SURFACE_CAPTURE_CONSUMED: &str = "surface_capture_consumed";
pub const SURFACE_CAPTURE_STALE: &str = "surface_capture_stale";
pub const SURFACE_COORDINATE_INVALID: &str = "surface_coordinate_invalid";
pub const SESSION_BUSY: &str = crate::rpc_reason::SESSION_BUSY;
pub const RECORD_START_PAGE_UNREACHABLE: &str = "record_start_page_unreachable";
}
@@ -227,6 +222,13 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
hint: Some("verify the CSS selector or wait for the element to appear before retrying"),
exit_code: base.exit_code,
},
(ErrorCode::InvalidParams, reason::INVALID_SELECTOR) => RenderInfo {
summary: "selector is not valid CSS",
hint: Some(
"use a standard CSS selector or a fresh @eN ref; Playwright text= selectors and XPath are not supported",
),
exit_code: base.exit_code,
},
(ErrorCode::InvalidParams, reason::TARGET_NOT_FILLABLE) => RenderInfo {
summary: "target element is not fillable",
hint: Some(
@@ -234,13 +236,6 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
),
exit_code: base.exit_code,
},
(ErrorCode::PermissionDenied, reason::REF_CAPABILITY_DENIED) => RenderInfo {
summary: "snapshot ref does not support this operation",
hint: Some(
"visual Surface refs require image understanding; inspect `bsk screenshot --ref <ref> --session <id>`, then bind its capture id and image coordinates to `bsk click`",
),
exit_code: base.exit_code,
},
(ErrorCode::InvalidParams, reason::TARGET_NOT_SELECT) => RenderInfo {
summary: "target element is not a <select>",
hint: Some(
@@ -283,41 +278,6 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
),
exit_code: base.exit_code,
},
(ErrorCode::NotFound, reason::SURFACE_CAPTURE_NOT_FOUND) => RenderInfo {
summary: "surface capture transaction was not found",
hint: Some(
"take a fresh surface screenshot and use its capture id with coordinates from that image",
),
exit_code: base.exit_code,
},
(ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_EXPIRED) => RenderInfo {
summary: "surface capture transaction has expired",
hint: Some(
"take a fresh surface screenshot, then click promptly with that capture id and image coordinates",
),
exit_code: base.exit_code,
},
(ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_CONSUMED) => RenderInfo {
summary: "surface capture transaction was already used",
hint: Some(
"each capture id is single-use; take a fresh surface screenshot before another point action",
),
exit_code: base.exit_code,
},
(ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_STALE) => RenderInfo {
summary: "page state changed since the surface screenshot",
hint: Some(
"take a fresh surface screenshot and choose coordinates from the new image before retrying",
),
exit_code: base.exit_code,
},
(ErrorCode::InvalidParams, reason::SURFACE_COORDINATE_INVALID) => RenderInfo {
summary: "surface image coordinate is invalid",
hint: Some(
"use finite pixel coordinates inside the captured image; its top-left corner is (0, 0)",
),
exit_code: base.exit_code,
},
(
ErrorCode::CdpFailed | ErrorCode::Timeout | ErrorCode::InvalidParams,
reason::RECORD_START_PAGE_UNREACHABLE,
@@ -425,18 +385,6 @@ mod tests {
assert!(!info.summary.contains("sandbox"));
}
#[test]
fn ref_capability_denied_explains_visual_surface_refs() {
let data = serde_json::json!({ "reason": reason::REF_CAPABILITY_DENIED });
let info = info_for_error(ErrorCode::PermissionDenied, Some(&data));
assert_eq!(info.summary, "snapshot ref does not support this operation");
assert!(
info.hint.unwrap().contains("capture id"),
"expected visual-surface-specific hint"
);
assert_eq!(info.exit_code, 1);
}
#[test]
fn session_busy_overrides_timeout_copy() {
let data = serde_json::json!({ "reason": reason::SESSION_BUSY });
@@ -479,25 +427,6 @@ mod tests {
assert_eq!(info.exit_code, 3);
}
#[test]
fn surface_capture_stale_requests_a_fresh_screenshot() {
let data = serde_json::json!({ "reason": reason::SURFACE_CAPTURE_STALE });
let info = info_for_error(ErrorCode::PermissionDenied, Some(&data));
assert_eq!(
info.summary,
"page state changed since the surface screenshot"
);
assert!(info.hint.unwrap().contains("fresh surface screenshot"));
}
#[test]
fn surface_coordinate_invalid_explains_image_pixel_bounds() {
let data = serde_json::json!({ "reason": reason::SURFACE_COORDINATE_INVALID });
let info = info_for_error(ErrorCode::InvalidParams, Some(&data));
assert_eq!(info.summary, "surface image coordinate is invalid");
assert!(info.hint.unwrap().contains("top-left corner is (0, 0)"));
}
#[test]
fn ref_not_found_overrides_not_found_copy() {
let data = serde_json::json!({ "reason": reason::REF_NOT_FOUND });
+1 -11
View File
@@ -59,7 +59,7 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
.map_err(CliError::Local)?;
match format {
Format::Json => {
let mut json = serde_json::json!({
let json = serde_json::json!({
"tab_id": reply.tab_id,
"width": reply.width,
"height": reply.height,
@@ -67,10 +67,6 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
"path": out_path.to_string_lossy(),
"byte_size": bytes.len(),
});
if let Some(capture) = &reply.capture {
json["capture"] = serde_json::to_value(capture)
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?;
}
println!(
"{}",
serde_json::to_string_pretty(&json)
@@ -79,12 +75,6 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
}
Format::Human => {
println!("{}", out_path.display());
if let Some(capture) = &reply.capture {
println!(
"capture={} surface={} coordinate_space={} expires_at={}",
capture.id, capture.surface_ref, capture.coordinate_space, capture.expires_at
);
}
print_dialog_summaries(&reply.dialogs);
}
}
+25 -2
View File
@@ -223,6 +223,14 @@ impl BrowserRegistry {
"browser reconnect: replacing previous registration"
);
}
tracing::info!(
diagnostic = true,
event = "browser.registry.insert",
id = %client.id,
generation = client.generation,
previous_generation = guard.get(&client.id).map(|previous| previous.generation),
"diagnostic browser registry mutation"
);
guard.insert(client.id.clone(), client);
}
@@ -239,16 +247,31 @@ impl BrowserRegistry {
/// took over the same `id` while the old socket was tearing down
/// (review M4/M5 round 2 Important #1). Returns the removed entry
/// when a removal actually happened.
#[track_caller]
pub fn remove_if_generation_matches(
&self,
id: &BrowserId,
generation: u64,
) -> Option<std::sync::Arc<BrowserClient>> {
let caller = std::panic::Location::caller();
let mut guard = self.inner.lock().expect("browser registry poisoned");
match guard.get(id) {
let current_generation = guard.get(id).map(|current| current.generation);
let removed = match guard.get(id) {
Some(current) if current.generation == generation => guard.remove(id),
_ => None,
}
};
tracing::info!(
diagnostic = true,
event = "browser.registry.remove_if_generation_matches",
id = %id,
requested_generation = generation,
?current_generation,
removed = removed.is_some(),
caller_file = caller.file(),
caller_line = caller.line(),
"diagnostic browser registry mutation"
);
removed
}
pub fn get(&self, id: &BrowserId) -> Option<std::sync::Arc<BrowserClient>> {
+48 -3
View File
@@ -37,7 +37,7 @@ use serde::{Deserialize, Serialize};
use serde_json::Value;
use tokio::sync::Notify;
use tokio::task::JoinHandle;
use tracing::{debug, warn};
use tracing::{debug, info, warn};
use super::abort::AbortRegistry;
use super::queue::{DEFAULT_TOOL_TIMEOUT, DispatchError};
@@ -197,7 +197,22 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
let status = status.clone();
let state = Arc::clone(&state);
Box::pin(async move {
match method {
let method_name = format!("{method:?}");
let diagnostic_rpc_id = rpc_id.clone();
let session_id = params
.get("session_id")
.and_then(Value::as_str)
.map(str::to_owned);
let started = Instant::now();
info!(
diagnostic = true,
event = "ipc.rpc.started",
rpc_id = %diagnostic_rpc_id,
method = %method_name,
session = session_id.as_deref().unwrap_or(""),
"diagnostic IPC RPC"
);
let body = match method {
Method::SystemPing => {
let result = PingResult { pong: true };
ResponseBody::Ok(serde_json::to_value(result).unwrap_or(Value::Null))
@@ -261,7 +276,23 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
message: format!("method not implemented yet: {other:?}"),
data: None,
}),
}
};
let (outcome, error_code) = match &body {
ResponseBody::Ok(_) => ("ok", String::new()),
ResponseBody::Err(error) => ("error", format!("{:?}", error.code)),
};
info!(
diagnostic = true,
event = "ipc.rpc.completed",
rpc_id = %diagnostic_rpc_id,
method = %method_name,
session = session_id.as_deref().unwrap_or(""),
outcome,
error_code = %error_code,
elapsed_ms = started.elapsed().as_millis() as u64,
"diagnostic IPC RPC"
);
body
})
})
}
@@ -465,8 +496,14 @@ fn handle_cancel(state: &Arc<DaemonState>, params: Value) -> ResponseBody {
};
let local = state.abort_registry.cancel(&params.rpc_id);
let mut cancelled = local;
let mut surface = if local { "abort_registry" } else { "none" };
if !local && let Some(snap) = state.tool_inflight.cancel(&params.rpc_id) {
cancelled = true;
surface = if snap.ws_rpc_id.is_some() {
"tool_inflight_forwarded"
} else {
"tool_inflight_queued"
};
// Forward the WS-side cancel only when the worker has
// already promoted the entry to "forwarded"; queued entries
// short-circuit on their own pre-flight without ever
@@ -484,6 +521,14 @@ fn handle_cancel(state: &Arc<DaemonState>, params: Value) -> ResponseBody {
);
}
}
info!(
diagnostic = true,
event = "ipc.cancel.resolved",
target_rpc_id = %params.rpc_id,
cancelled,
surface,
"diagnostic IPC cancellation"
);
ResponseBody::Ok(serde_json::to_value(CancelResult { cancelled }).unwrap_or(Value::Null))
}
+48 -1
View File
@@ -41,7 +41,7 @@ use bsk_protocol::{ErrorCode, Frame, Method, RequestFrame, ResponseBody, RpcErro
use rand::Rng;
use serde_json::Value;
use tokio::sync::{mpsc, oneshot};
use tracing::{debug, warn};
use tracing::{debug, info, warn};
use super::abort::AbortToken;
use super::browsers::BrowserRegistry;
@@ -515,6 +515,16 @@ async fn forward_one(
sessions: &Arc<SessionRegistry>,
job: &ToolJob,
) -> Result<Value, RpcError> {
let started = std::time::Instant::now();
info!(
diagnostic = true,
event = "tool.dispatch.started",
session = %sid,
method = ?job.method,
has_inflight = job.inflight.is_some(),
has_lifecycle_cancel = job.lifecycle_cancel.is_some(),
"diagnostic tool dispatch"
);
// Pre-flight: a cancel that landed while this job was still in
// the per-session channel must short-circuit before any session
// / browser resolution work, and before any WS frame leaves the
@@ -535,6 +545,13 @@ async fn forward_one(
));
}
let Some(session) = sessions.get(sid) else {
warn!(
diagnostic = true,
event = "tool.dispatch.session_missing",
session = %sid,
method = ?job.method,
"diagnostic tool dispatch"
);
return Err(RpcError {
code: ErrorCode::NotFound,
message: format!("session {sid} no longer exists"),
@@ -542,6 +559,14 @@ async fn forward_one(
});
};
let Some(client) = browsers.get(&session.browser_id) else {
warn!(
diagnostic = true,
event = "tool.dispatch.browser_missing",
session = %sid,
method = ?job.method,
browser = %session.browser_id,
"diagnostic tool dispatch"
);
return Err(RpcError {
code: ErrorCode::NotFound,
message: "owning browser is no longer connected".into(),
@@ -558,6 +583,16 @@ async fn forward_one(
method: job.method.clone(),
params: Some(job.params.clone()),
});
info!(
diagnostic = true,
event = "tool.dispatch.forwarding",
session = %sid,
method = ?job.method,
browser = %session.browser_id,
browser_generation = client.generation,
ws_rpc_id = %rpc_id,
"diagnostic tool dispatch"
);
// Promote the inflight entry to "forwarded" AND push the WS
// request frame to the sink inside the same critical section
// (review round 2 C1). The closure runs while the entry's inner
@@ -717,6 +752,18 @@ async fn forward_one(
});
}
};
info!(
diagnostic = true,
event = "tool.dispatch.response",
session = %sid,
method = ?job.method,
browser = %session.browser_id,
browser_generation = client.generation,
ws_rpc_id = %rpc_id,
elapsed_ms = started.elapsed().as_millis() as u64,
response_is_error = matches!(&response.body, ResponseBody::Err(_)),
"diagnostic tool dispatch"
);
match response.body {
ResponseBody::Ok(v) => Ok(v),
ResponseBody::Err(err) => Err(err),
+68 -2
View File
@@ -105,12 +105,22 @@ impl SessionRegistry {
pub fn insert(&self, session: Session) {
let session_id = session.id.clone();
let browser_id = session.browser_id.clone();
let agent_window_id = session.agent_window_id;
let mut sessions = self.inner.lock().expect("session registry poisoned");
sessions.insert(session_id.clone(), session);
self.last_activity
.lock()
.expect("session activity registry poisoned")
.insert(session_id, Instant::now());
.insert(session_id.clone(), Instant::now());
tracing::info!(
diagnostic = true,
event = "session.registry.insert",
session = %session_id,
browser = %browser_id,
?agent_window_id,
"diagnostic session registry mutation"
);
}
/// Reserve a fresh, collision-free [`SessionId`] under the registry
@@ -152,6 +162,13 @@ impl SessionRegistry {
.lock()
.expect("session activity registry poisoned")
.insert(candidate.clone(), Instant::now());
tracing::info!(
diagnostic = true,
event = "session.registry.reserve",
session = %candidate,
browser = %browser_id,
"diagnostic session registry mutation"
);
return Some(candidate);
}
None
@@ -169,13 +186,24 @@ impl SessionRegistry {
let mut guard = self.inner.lock().expect("session registry poisoned");
let session = guard.get_mut(session_id)?;
session.agent_window_id = agent_window_id;
tracing::info!(
diagnostic = true,
event = "session.registry.commit",
session = %session_id,
browser = %session.browser_id,
?agent_window_id,
"diagnostic session registry mutation"
);
Some(session.clone())
}
/// Drop a placeholder reservation, used on extension error/timeout
/// paths so failed reservations do not accumulate.
#[track_caller]
pub fn cancel_reservation(&self, session_id: &SessionId) {
self.inner
let caller = std::panic::Location::caller();
let removed = self
.inner
.lock()
.expect("session registry poisoned")
.remove(session_id);
@@ -183,9 +211,20 @@ impl SessionRegistry {
.lock()
.expect("session activity registry poisoned")
.remove(session_id);
tracing::info!(
diagnostic = true,
event = "session.registry.cancel_reservation",
session = %session_id,
existed = removed.is_some(),
caller_file = caller.file(),
caller_line = caller.line(),
"diagnostic session registry mutation"
);
}
#[track_caller]
pub fn remove(&self, id: &SessionId) -> Option<Session> {
let caller = std::panic::Location::caller();
let removed = self
.inner
.lock()
@@ -195,6 +234,17 @@ impl SessionRegistry {
.lock()
.expect("session activity registry poisoned")
.remove(id);
tracing::info!(
diagnostic = true,
event = "session.registry.remove",
session = %id,
existed = removed.is_some(),
browser = removed.as_ref().map(|session| session.browser_id.0.as_str()),
agent_window_id = removed.as_ref().and_then(|session| session.agent_window_id),
caller_file = caller.file(),
caller_line = caller.line(),
"diagnostic session registry mutation"
);
removed
}
@@ -243,7 +293,9 @@ impl SessionRegistry {
}
/// Drop all sessions owned by `browser_id` (e.g. on disconnect).
#[track_caller]
pub fn purge_browser(&self, browser_id: &BrowserId) -> Vec<Session> {
let caller = std::panic::Location::caller();
let mut guard = self.inner.lock().expect("session registry poisoned");
let drained: Vec<Session> = guard
.values()
@@ -260,6 +312,20 @@ impl SessionRegistry {
for session in &drained {
activity.remove(&session.id);
}
let session_ids: Vec<&str> = drained
.iter()
.map(|session| session.id.0.as_str())
.collect();
tracing::info!(
diagnostic = true,
event = "session.registry.purge_browser",
browser = %browser_id,
sessions = ?session_ids,
count = drained.len(),
caller_file = caller.file(),
caller_line = caller.line(),
"diagnostic session registry mutation"
);
drained
}
+11 -2
View File
@@ -40,6 +40,8 @@ pub(crate) const DAEMONIZED_ENV: &str = "BSK_DAEMONIZED";
/// daemon lock, IPC socket, and WS port — before taking over.
pub(crate) const DAEMON_REPLACEMENT_WAIT_ENV: &str = "BSK_DAEMON_REPLACES_PID";
const LUMI_DIAGNOSTIC_BUILD: &str = "lumi-session-loss-v2";
/// Concrete daemon configuration resolved from CLI flags / defaults.
#[derive(Debug, Clone)]
pub struct DaemonConfig {
@@ -251,6 +253,7 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
pid = info.pid,
ws_port = info.ws_port,
sock = %sock_path.display(),
diagnostic_build = LUMI_DIAGNOSTIC_BUILD,
"daemon ready"
);
@@ -588,7 +591,9 @@ pub(crate) fn spawn_update_check_task(
// The session gate is read after the fetch, as late
// as possible before the binary gets replaced.
let active_sessions = state.sessions.len();
let auto_update = update::auto_update_enabled() && exe_path.is_some();
// Keep this local diagnostic build in place until a
// reproduction has uploaded its extension-side evidence.
let auto_update = false;
update::auto_update_step(
candidate.as_ref(),
auto_update,
@@ -732,7 +737,11 @@ fn init_tracing() -> Option<tracing_appender::non_blocking::WorkerGuard> {
use tracing_subscriber::prelude::*;
let log_dir = paths::log_dir().ok();
let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
let env_filter = EnvFilter::try_from_default_env()
.unwrap_or_else(|_| EnvFilter::new("info"))
// Lumi commonly runs with RUST_LOG=warn. This local build must retain
// BrowserSkill's INFO diagnostics even under that host-wide setting.
.add_directive("bsk=info".parse().expect("valid diagnostic log directive"));
let (file_layer, guard) = if let Some(dir) = log_dir {
let appender = tracing_appender::rolling::daily(&dir, "daemon.log");
+65 -6
View File
@@ -233,6 +233,14 @@ async fn drive_connection(
.ok_or_else(|| anyhow!("handshake missing params"))?;
let params: HandshakeParams = serde_json::from_value(params_raw)
.map_err(|err| anyhow!("invalid HandshakeParams: {err}"))?;
info!(
diagnostic = true,
event = "browser.handshake.received",
browser = %params.instance_id,
extension_version = %params.version,
extension_protocol = %params.protocol_version,
"diagnostic browser handshake"
);
// Version compatibility (design §10, M10.4): protocol_version only.
let our_app_version: Version = env!("CARGO_PKG_VERSION")
@@ -350,7 +358,7 @@ async fn drive_connection(
// Pump loop: outbound frames from sink → ws; inbound from ws → resolve.
let pump_state = Arc::clone(&state);
let pump_browser = Arc::clone(&client);
let result_outcome: anyhow::Result<()> = async {
let result_outcome: anyhow::Result<String> = async {
loop {
tokio::select! {
outbound = rx.recv() => {
@@ -359,7 +367,7 @@ async fn drive_connection(
let json = serde_json::to_string(&frame)?;
writer.send(Message::Text(json)).await?;
}
None => break,
None => break Ok("outbound_channel_closed".to_string()),
}
}
msg = reader.next() => {
@@ -381,18 +389,37 @@ async fn drive_connection(
pump_browser.touch();
}
Some(Ok(Message::Frame(_))) => {}
Some(Ok(Message::Close(_))) | None => break,
Some(Ok(Message::Close(frame))) => {
let reason = frame
.map(|frame| format!("peer_close code={:?} reason={}", frame.code, frame.reason))
.unwrap_or_else(|| "peer_close_without_frame".to_string());
break Ok(reason);
}
None => break Ok("peer_eof".to_string()),
Some(Err(err)) => {
warn!(%err, "ws read error");
break;
break Ok(format!("read_error: {err}"));
}
}
}
}
}
Ok(())
}
.await;
let pump_exit_reason = match &result_outcome {
Ok(reason) => reason.clone(),
Err(err) => format!("pump_error: {err:#}"),
};
info!(
diagnostic = true,
event = "browser.websocket.pump_exit",
id = %browser_id,
generation,
reason = %pump_exit_reason,
last_seen_idle_ms = client.idle_for().as_millis() as u64,
session_ids = ?state.sessions.ids_for_browser(&browser_id),
"diagnostic browser websocket exit"
);
// Cleanup: drop browser + purge its sessions, but only if the
// registry still holds *this* generation. If a reconnect already
@@ -416,7 +443,7 @@ async fn drive_connection(
"browser disconnected; newer reconnect already owns this id, leaving sessions intact"
);
}
result_outcome
result_outcome.map(|_| ())
}
async fn handle_inbound_text(state: &Arc<DaemonState>, client: &Arc<BrowserClient>, text: &str) {
@@ -442,6 +469,38 @@ async fn handle_inbound_text(state: &Arc<DaemonState>, client: &Arc<BrowserClien
client.mark_heartbeat_seen();
debug!(id = %client.id, "heartbeat");
}
bsk_protocol::EventKind::BrowserDiagnostic => {
let entries = ev.payload.get("entries").and_then(|value| value.as_array());
info!(
diagnostic = true,
event = "extension.diagnostic.batch",
browser = %client.id,
generation = client.generation,
worker_boot_id = ev.payload.get("worker_boot_id").and_then(|value| value.as_str()).unwrap_or("unknown"),
count = entries.map_or(0, Vec::len),
"received extension diagnostic batch"
);
if let Some(entries) = entries {
for entry in entries {
info!(
diagnostic = true,
event = "extension.diagnostic.entry",
browser = %client.id,
generation = client.generation,
entry = %entry,
"extension diagnostic"
);
}
} else {
warn!(
diagnostic = true,
event = "extension.diagnostic.invalid",
browser = %client.id,
payload = %ev.payload,
"extension diagnostic payload missing entries"
);
}
}
bsk_protocol::EventKind::SessionWindowClosed => {
handle_session_window_closed(state, &client.id, &ev.payload);
}
-23
View File
@@ -188,29 +188,6 @@ fn parses_click_count_alias() {
assert_eq!(args.click_count, 2);
}
#[test]
fn parses_surface_capture_image_coordinates() {
let cli = parse(&[
"bsk",
"click",
"@e1",
"--session",
"s1",
"--capture",
"sc_test",
"--image-x",
"160.5",
"--image-y",
"-1",
]);
let Command::Click(args) = cli.command else {
panic!("expected click command");
};
assert_eq!(args.capture.as_deref(), Some("sc_test"));
assert_eq!(args.image_x, Some(160.5));
assert_eq!(args.image_y, Some(-1.0));
}
#[test]
fn parses_hover_with_settle() {
let cli = parse(&[
-2
View File
@@ -238,7 +238,6 @@ async fn screenshot_returns_image_base64_with_dimensions() {
height: 600,
format: "png".into(),
tab_id: 7,
capture: None,
dialogs: vec![],
})
.unwrap(),
@@ -287,7 +286,6 @@ async fn screenshot_forwards_ref_to_extension() {
height: 60,
format: "png".into(),
tab_id: 7,
capture: None,
dialogs: vec![],
})
.unwrap(),
-9
View File
@@ -354,9 +354,6 @@ async fn click_round_trips_ref_and_modifiers() {
used_selector: None,
x: 12.5,
y: 34.0,
capture_id: None,
image_x: None,
image_y: None,
dialogs: vec![],
})
.unwrap(),
@@ -376,9 +373,6 @@ async fn click_round_trips_ref_and_modifiers() {
click_count: Some(1),
modifiers: Some(vec![KeyModifier::Ctrl, KeyModifier::Shift]),
timeout_ms: Some(5_000),
capture_id: None,
image_x: None,
image_y: None,
},
)
.await
@@ -616,9 +610,6 @@ async fn m7_tools_propagate_extension_errors() {
click_count: None,
modifiers: None,
timeout_ms: Some(5_000),
capture_id: None,
image_x: None,
image_y: None,
},
)
.await
@@ -16,13 +16,6 @@
}
]
},
"capture_id": {
"description": "Short-lived capture id returned by a visual Surface screenshot.",
"type": [
"string",
"null"
]
},
"click_count": {
"description": "Number of consecutive mouse presses (double-click = 2).",
"type": [
@@ -32,22 +25,6 @@
"format": "uint32",
"minimum": 1.0
},
"image_x": {
"description": "X coordinate in the capture image's pixel coordinate space.",
"type": [
"number",
"null"
],
"format": "double"
},
"image_y": {
"description": "Y coordinate in the capture image's pixel coordinate space.",
"type": [
"number",
"null"
],
"format": "double"
},
"modifiers": {
"type": [
"array",
@@ -8,12 +8,6 @@
"y"
],
"properties": {
"capture_id": {
"type": [
"string",
"null"
]
},
"dialogs": {
"description": "Native JS dialogs observed and auto-handled during this call.",
"type": "array",
@@ -21,20 +15,6 @@
"$ref": "#/definitions/JavaScriptDialogInfo"
}
},
"image_x": {
"type": [
"number",
"null"
],
"format": "double"
},
"image_y": {
"type": [
"number",
"null"
],
"format": "double"
},
"tab_id": {
"type": "integer",
"format": "int64"
@@ -8,16 +8,6 @@
"tab_id"
],
"properties": {
"capture": {
"anyOf": [
{
"$ref": "#/definitions/SurfaceCaptureInfo"
},
{
"type": "null"
}
]
},
"dialogs": {
"type": "array",
"items": {
@@ -119,31 +109,6 @@
"prompt",
"beforeunload"
]
},
"SurfaceCaptureInfo": {
"type": "object",
"required": [
"coordinate_space",
"expires_at",
"id",
"surface_ref"
],
"properties": {
"coordinate_space": {
"type": "string"
},
"expires_at": {
"type": "integer",
"format": "uint64",
"minimum": 0.0
},
"id": {
"type": "string"
},
"surface_ref": {
"type": "string"
}
}
}
}
}
+25
View File
@@ -58,6 +58,10 @@ pub enum EventKind {
BrowserDisconnected,
#[serde(rename = "browser.connected")]
BrowserConnected,
/// Structured diagnostics emitted by locally instrumented extension
/// builds. Production peers may ignore this event safely.
#[serde(rename = "browser.diagnostic")]
BrowserDiagnostic,
}
#[derive(Debug, Clone, PartialEq)]
@@ -256,6 +260,27 @@ mod tests {
assert_eq!(frame.event, EventKind::SystemHeartbeat);
}
#[test]
fn browser_diagnostic_event_frame_round_trips_from_extension_shape() {
let wire = serde_json::json!({
"event": "browser.diagnostic",
"payload": {
"schema_version": 1,
"worker_boot_id": "worker-1",
"entries": [{ "event": "extension.worker.started" }]
}
});
let frame: EventFrame = serde_json::from_value(wire).unwrap();
assert_eq!(frame.event, EventKind::BrowserDiagnostic);
assert_eq!(
frame
.payload
.get("worker_boot_id")
.and_then(|value| value.as_str()),
Some("worker-1")
);
}
#[test]
fn session_user_interrupt_serialises_as_snake_case() {
let v = serde_json::to_value(EventKind::SessionUserInterrupt).unwrap();
@@ -67,15 +67,6 @@ pub struct ClickParams {
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 1))]
pub timeout_ms: Option<u32>,
/// Short-lived capture id returned by a visual Surface screenshot.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub capture_id: Option<String>,
/// X coordinate in the capture image's pixel coordinate space.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub image_x: Option<f64>,
/// Y coordinate in the capture image's pixel coordinate space.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub image_y: Option<f64>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
@@ -89,12 +80,6 @@ pub struct ClickResult {
/// agents can correlate with a follow-up `tool.screenshot`.
pub x: f64,
pub y: f64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub capture_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub image_x: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub image_y: Option<f64>,
/// Native JS dialogs observed and auto-handled during this call.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
@@ -303,9 +288,6 @@ mod tests {
click_count: Some(1),
modifiers: Some(vec![KeyModifier::Ctrl]),
timeout_ms: Some(5_000),
capture_id: Some("sc_test".into()),
image_x: Some(160.0),
image_y: Some(90.0),
};
let v = serde_json::to_value(&p).unwrap();
assert_eq!(v.get("ref").and_then(|v| v.as_str()), Some("@e3"));
@@ -208,20 +208,10 @@ pub struct ScreenshotResult {
/// Always `"png"` in v0.1; reserved for future JPEG / WebP support.
pub format: String,
pub tab_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub capture: Option<SurfaceCaptureInfo>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct SurfaceCaptureInfo {
pub id: String,
pub surface_ref: String,
pub coordinate_space: String,
pub expires_at: u64,
}
#[cfg(test)]
mod tests {
use super::*;
@@ -303,12 +293,6 @@ mod tests {
height: 600,
format: "png".into(),
tab_id: 7,
capture: Some(SurfaceCaptureInfo {
id: "sc_test".into(),
surface_ref: "@e3".into(),
coordinate_space: "capture-image-pixel".into(),
expires_at: 1_700_000_030_000,
}),
dialogs: vec![],
};
let v = serde_json::to_value(&r).unwrap();
+5 -69
View File
@@ -489,9 +489,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
? "Capture an indented aria-tree snapshot of the session's active tab. Interactive elements " +
"carry @eN refs for browser_interact actions. Prefer browser_inspect action=observe for a richer semantic view."
: "Produce a semantic VOM observation of the session's active tab (roles, states, perception " +
"probes) with @eN refs for browser_interact actions. Visual-only surfaces require image " +
"understanding and their refs are screenshot-only; keep using any DOM/AX semantics rendered " +
"alongside them. Read-only: never submits input.";
"probes) with @eN refs for browser_interact actions. Read-only: never submits input.";
register(
defineTool({
name,
@@ -563,8 +561,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
name: "interact.click",
description:
"Click an element in the session's active tab. Target is a snapshot ref (@e3) from the " +
"last browser_inspect observation, or a CSS selector. A visual Surface requires the " +
"captureId and image coordinates returned by browser_inspect action=screenshot.",
"last browser_inspect observation, or a CSS selector.",
parameters: {
target: {
type: "string",
@@ -581,24 +578,6 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
type: "integer",
description: "Number of consecutive presses (double-click = 2).",
},
modifiers: {
type: "array",
items: { type: "string", enum: ["alt", "ctrl", "meta", "shift"] },
description: "Keyboard modifiers held during the click.",
},
captureId: {
type: "string",
description:
"Short-lived Surface capture id returned by browser_inspect action=screenshot.",
},
imageX: {
type: "number",
description: "X coordinate in capture-image pixels; requires captureId and imageY.",
},
imageY: {
type: "number",
description: "Y coordinate in capture-image pixels; requires captureId and imageX.",
},
},
output: {
schema: {
@@ -609,7 +588,6 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
tabId: { type: "integer", required: true },
x: { type: "number", required: true },
y: { type: "number", required: true },
captureId: { type: "string" },
},
},
render: (_args, value) => [
@@ -621,44 +599,17 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
},
async execute(args, exec) {
if (args.target.trim().length === 0) throw new Error("target must be a non-empty string");
const pointArgCount =
Number(args.captureId !== undefined) +
Number(args.imageX !== undefined) +
Number(args.imageY !== undefined);
if (pointArgCount !== 0 && pointArgCount !== 3) {
throw new Error("captureId, imageX, and imageY must be given together");
}
const sessionId = registry.resolve(args.session, "browser_interact(action=click)");
const cmdArgs = ["click", "--session", sessionId];
if (args.button !== undefined) cmdArgs.push("--button", args.button);
if (args.clickCount !== undefined) cmdArgs.push("--click-count", String(args.clickCount));
if (args.modifiers !== undefined && args.modifiers.length > 0) {
cmdArgs.push("--modifiers", args.modifiers.join(","));
}
if (args.captureId !== undefined) {
cmdArgs.push(
"--capture",
args.captureId,
"--image-x",
String(args.imageX),
"--image-y",
String(args.imageY),
);
}
cmdArgs.push(args.target);
const reply = (await runBsk(deps, exec, cmdArgs, "click", sessionId)) as {
tab_id: number;
x: number;
y: number;
capture_id?: string;
};
return {
session: sessionId,
tabId: reply.tab_id,
x: reply.x,
y: reply.y,
...(reply.capture_id !== undefined ? { captureId: reply.capture_id } : {}),
};
return { session: sessionId, tabId: reply.tab_id, x: reply.x, y: reply.y };
},
presentCall: (args) => ({
card: "terminal",
@@ -801,17 +752,12 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
name: "inspect.screenshot",
description:
"Capture a PNG screenshot of the session's active tab, or crop to a snapshot ref element. " +
"This is the supported operation for visual surface refs emitted by inspect.observe. Only use " +
"it to interpret a visual-only surface when you can inspect image output. If you lack multimodal " +
"or image-reading capability, tell the user to switch to a model with image understanding; do " +
"not guess content or coordinates. " +
"Returns the image itself when the deployment supports image input, otherwise a file path.",
parameters: {
session: SESSION_PARAM,
ref: {
type: "string",
description:
"Ref (@e3) from the last snapshot/observe; crops to that DOM element or visual surface.",
description: "Snapshot ref (@e3) from the last snapshot/observe; crops to that element.",
},
},
output: {
@@ -825,8 +771,6 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
width: { type: "integer", required: true },
height: { type: "integer", required: true },
byteSize: { type: "integer", required: true },
captureId: { type: "string" },
captureExpiresAt: { type: "integer" },
image: {
type: "object",
additionalProperties: false,
@@ -850,11 +794,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
value.image !== undefined
? `[session ${value.session}] screenshot of tab ${value.tabId} (${value.width}x${value.height}px)`
: `[session ${value.session}] screenshot saved to ${value.path} (${value.width}x${value.height}px, ${value.byteSize} bytes) — this deployment cannot inline images; read the file to view it`;
const captureText =
value.captureId !== undefined
? ` Surface capture ${value.captureId} uses capture-image-pixel coordinates and expires at ${value.captureExpiresAt}.`
: "";
const blocks: ContentBlock[] = [{ type: "text", text: text + captureText }];
const blocks: ContentBlock[] = [{ type: "text", text }];
if (value.image !== undefined) {
blocks.push({
type: "image",
@@ -893,7 +833,6 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
height: number;
path: string;
byte_size: number;
capture?: { id: string; expires_at: number };
};
writtenPath = reply.path;
const data = await readFile(reply.path);
@@ -907,9 +846,6 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
width: reply.width,
height: reply.height,
byteSize: reply.byte_size,
...(reply.capture !== undefined
? { captureId: reply.capture.id, captureExpiresAt: reply.capture.expires_at }
: {}),
...(ref !== undefined
? {
image: {
@@ -478,12 +478,12 @@ describe("interaction tools", () => {
emulate: { tab_id: 7, cleared: false },
};
it("interact.click maps target, button, click count, and modifiers", async () => {
it("interact.click maps target, button, and click count", async () => {
const { tools, calls } = setup(responses);
await startSession(tools);
const click = tools.get("interact.click");
const value = (await click?.execute(
{ target: "@e1", button: "right", clickCount: 2, modifiers: ["ctrl", "shift"] },
{ target: "@e1", button: "right", clickCount: 2 },
makeExec(),
)) as { session: string; x: number; y: number };
expect(value).toMatchObject({ session: "s1", x: 10, y: 20 });
@@ -495,43 +495,6 @@ describe("interaction tools", () => {
"right",
"--click-count",
"2",
"--modifiers",
"ctrl,shift",
"@e1",
]);
});
it("interact.click forwards a complete Surface capture coordinate transaction", async () => {
const { tools, calls } = setup({
...responses,
click: {
tab_id: 7,
used_ref: "e1",
x: 110,
y: 70,
capture_id: "sc_test",
image_x: 200,
image_y: 100,
},
});
await startSession(tools);
const click = tools.get("interact.click");
const value = await click?.execute(
{ target: "@e1", captureId: "sc_test", imageX: 200, imageY: 100 },
makeExec(),
);
expect(value).toMatchObject({ captureId: "sc_test", x: 110, y: 70 });
expect(calls[1].args).toEqual([
"click",
"--session",
"s1",
"--capture",
"sc_test",
"--image-x",
"200",
"--image-y",
"100",
"@e1",
]);
});
@@ -960,32 +923,6 @@ describe("inspect.screenshot", () => {
expect(rendered?.every((block) => block.type === "text")).toBe(true);
});
it("returns Surface capture metadata alongside the screenshot", async () => {
const path = pngFile();
const { tools } = setup({
"session start": START_REPLY("s1"),
screenshot: {
tab_id: 7,
width: 800,
height: 600,
format: "png",
path,
byte_size: 8,
capture: { id: "sc_test", expires_at: 1700000030000 },
},
});
await startSession(tools);
const screenshot = tools.get("inspect.screenshot");
const value = await screenshot?.execute({ ref: "@e1" }, makeExec());
expect(value).toMatchObject({
captureId: "sc_test",
captureExpiresAt: 1700000030000,
});
const rendered = screenshot?.output.render({ session: "s1" }, value as never);
expect(rendered?.[0]).toMatchObject({ type: "text", text: expect.stringContaining("sc_test") });
});
it("commits the image through the attachment store on an image-capable route", async () => {
const path = pngFile();
const services = {
-60
View File
@@ -29,66 +29,6 @@ function coreRefs(refs: ReturnType<typeof renderVom>["refs"]) {
}
describe("renderVom single-layer page", () => {
it("renders visual surfaces as screenshot-only refs", () => {
const input = scene([
node({ id: 1, role: "RootWebArea", frameId: "main" }),
node({ id: 2, parentId: 1, role: "Iframe", frameId: "main", backendNodeId: 20 }),
]);
input.visualSurfaces = [
{
parentId: 2,
backendNodeId: 99,
frameId: "child",
renderingKind: "canvas",
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
label: "Data grid",
memberCount: 2,
},
];
const out = renderVom(input);
expect(out.text).toContain(
'@e1 surface "Data grid" [rendering=canvas; layers=2; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
);
expect(out.refs).toContainEqual(
expect.objectContaining({
ref: "e1",
backendNodeId: 99,
kind: "surface",
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
capabilities: ["screenshot"],
}),
);
});
it("renders an unnamed visual surface with a stable generic label", () => {
const input = scene([node({ id: 1, role: "RootWebArea", frameId: "main" })]);
input.visualSurfaces = [
{
parentId: 1,
backendNodeId: 99,
frameId: "main",
renderingKind: "canvas",
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
memberCount: 1,
},
];
const out = renderVom(input);
expect(out.text).toContain(
'@e1 surface "canvas visual surface" [rendering=canvas; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
);
expect(out.refs).toContainEqual(
expect.objectContaining({
ref: "e1",
name: "canvas visual surface",
kind: "surface",
capabilities: ["screenshot"],
}),
);
});
it("does not derive handle context across frame scopes", () => {
const out = renderVom(
scene([
-2
View File
@@ -15,8 +15,6 @@ export type {
VomNode,
VomOptions,
VomRef,
VomRefCapability,
VomResult,
VomScene,
VomVisualSurface,
} from "./types";
+1 -65
View File
@@ -9,7 +9,6 @@ import type {
VomOptions,
VomResult,
VomScene,
VomVisualSurface,
} from "./types";
const SKIP_ROLES = new Set(["generic", "none", "presentation", "inlinetextbox"]);
@@ -738,53 +737,6 @@ interface RenderState {
duplicateRefNames: Set<string>;
surfaceMap: Map<number, CondSurface>;
scopeMap: Map<number, ActiveScopeBlock>;
visualSurfaces: Map<number | null, VomVisualSurface[]>;
}
function groupedByParent<T extends { parentId: number | null }>(
items: T[],
): Map<number | null, T[]> {
const grouped = new Map<number | null, T[]>();
for (const item of items) {
const siblings = grouped.get(item.parentId) ?? [];
siblings.push(item);
grouped.set(item.parentId, siblings);
}
return grouped;
}
function emitVisualEntries(parentId: number | null, depth: number, state: RenderState): void {
if (state.stopped) return;
for (const surface of state.visualSurfaces.get(parentId) ?? []) {
if (depth > state.maxDepth) {
state.truncated = true;
continue;
}
const ref = `e${state.nextRef}`;
const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`;
const layers = surface.memberCount > 1 ? `; layers=${surface.memberCount}` : "";
const line = `${" ".repeat(depth)}@${ref} surface ${JSON.stringify(label)} [rendering=${surface.renderingKind}${layers}; visual-only; requires=image-understanding; use: bsk screenshot --ref @${ref}]`;
const nextTokens = state.tokens + estimateTokens(line);
if (nextTokens > state.maxTokens) {
state.truncated = true;
state.stopped = true;
return;
}
state.lines.push(line);
state.tokens = nextTokens;
state.refs.push({
ref,
backendNodeId: surface.backendNodeId,
frameId: surface.frameId,
visibleRect: surface.visibleRect,
role: "surface",
name: label,
kind: "surface",
capabilities: ["screenshot"],
line: state.lines.length - 1,
});
state.nextRef += 1;
}
}
function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: RenderState): void {
@@ -860,8 +812,6 @@ function renderTree(
...(node.role ? { role: node.role } : {}),
...(name ? { name } : {}),
...(context.length > 0 ? { ctx: context.join(" > ") } : {}),
kind: "dom",
capabilities: ["interact", "screenshot"],
line: state.lines.length - 1,
});
state.nextRef += 1;
@@ -871,12 +821,10 @@ function renderTree(
if (scope) emitActiveScopeBlock(scope, depth + 1, state);
if ((ref && shouldSkipRedundantRefChildren(node)) || shouldSkipRedundantChildren(node, state)) {
emitVisualEntries(node.id, depth + 1, state);
continue;
}
renderTree(children, node.id, depth + 1, state);
}
emitVisualEntries(parentId, depth, state);
}
function renderNodes(
@@ -885,7 +833,6 @@ function renderNodes(
initialLines: string[],
surfaces: CondSurface[] = [],
activeScopeBlocks: ActiveScopeBlock[] = [],
visualSurfaces: VomVisualSurface[] = [],
): RenderState {
const children = buildChildren(nodes);
const state: RenderState = {
@@ -904,7 +851,6 @@ function renderNodes(
duplicateRefNames: duplicateReferenceNames(nodes),
surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])),
scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])),
visualSurfaces: groupedByParent(visualSurfaces),
};
renderTree(children, null, 1, state);
@@ -1110,16 +1056,7 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
"@layers 2 focus=L1",
`L1 ${layer.kind} cover=${Math.round(layer.coverage * 100)}%`,
];
const visibleSurface = (parentId: number | null): boolean =>
parentId === null || included.has(parentId);
const state = renderNodes(
visibleNodes,
options,
header,
scene.surfaces,
scene.activeScopeBlocks,
scene.visualSurfaces?.filter((surface) => visibleSurface(surface.parentId)),
);
const state = renderNodes(visibleNodes, options, header, scene.surfaces, scene.activeScopeBlocks);
state.lines.push(renderPageOcclusionLine(hiddenCount));
return {
@@ -1147,7 +1084,6 @@ export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult
],
scene.surfaces,
scene.activeScopeBlocks,
scene.visualSurfaces,
);
return {
-18
View File
@@ -61,8 +61,6 @@ export interface VomScene {
rootFrameId?: string;
surfaces?: CondSurface[];
activeScopeBlocks?: ActiveScopeBlock[];
/** Non-semantic rendered regions discovered alongside the AX/DOM tree. */
visualSurfaces?: VomVisualSurface[];
}
export interface VomOptions {
@@ -96,22 +94,6 @@ export interface VomRef {
ref: string;
backendNodeId: number;
frameId?: string;
/** Observation-time top-viewport crop for screenshot-only surfaces. */
visibleRect?: Rect;
kind?: "dom" | "surface";
capabilities?: VomRefCapability[];
}
export type VomRefCapability = "interact" | "screenshot";
export interface VomVisualSurface {
parentId: number | null;
backendNodeId: number;
frameId: string;
renderingKind: "canvas";
visibleRect: Rect;
label?: string;
memberCount: number;
}
export interface RenderedRef extends VomRef {
+2 -12
View File
@@ -80,16 +80,6 @@ Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` /
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
When `bsk observe` renders `@eN surface ... [visual-only; requires=image-understanding; ...]`, the ref identifies rendered canvas content that is not represented by the text observation, not an interactable DOM control. Keep using any reliable DOM/AX text and controls that appear alongside it. If you lack multimodal or image-reading capability, do not take a screenshot and pretend to know its contents, and never guess coordinates; tell the user that they need to switch to a model with image-understanding capability.
If you can actually inspect image output, use `bsk screenshot --ref @eN --session <id>` to obtain the visible crop and its short-lived, single-use Surface capture id. Only when the task requires a point action and the image provides a clear target, bind that exact screenshot to the same general click command:
```bash
bsk click @eN --capture <capture-id> --image-x <px> --image-y <px> --session <id>
```
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/press workflow. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/hover/select reject them.
## Observation priority
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
@@ -184,7 +174,7 @@ bsk emulate --session <id> --off
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional); Surface crops return a short-lived capture id |
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
### Console & network debugging (read-only; require `--session`)
@@ -210,7 +200,7 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
| Command | Summary |
|---------|---------|
| `bsk click <ref-or-selector>` | Click a DOM element (`--button`, `--click-count`, `--modifiers`), or perform one screenshot-bound Surface point click with `--capture`, `--image-x`, and `--image-y` |
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |