mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ee620acf64 | ||
|
|
05a36075b8 |
@@ -1,7 +1,6 @@
|
|||||||
import { i18n } from "@browser-skill/i18n";
|
import { i18n } from "@browser-skill/i18n";
|
||||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||||
import { ConnectionController } from "@/lib/connection-controller";
|
import { ConnectionController } from "@/lib/connection-controller";
|
||||||
import { DiagnosticLogger } from "@/lib/diagnostics";
|
|
||||||
import { startHeartbeat } from "@/lib/heartbeat";
|
import { startHeartbeat } from "@/lib/heartbeat";
|
||||||
import {
|
import {
|
||||||
getConnectionEnabled,
|
getConnectionEnabled,
|
||||||
@@ -47,20 +46,9 @@ import type { Transport } from "@/transport/transport";
|
|||||||
import { WSTransport } from "@/transport/ws-transport";
|
import { WSTransport } from "@/transport/ws-transport";
|
||||||
|
|
||||||
export default defineBackground(() => {
|
export default defineBackground(() => {
|
||||||
const diagnostics = new DiagnosticLogger();
|
const controller = new ConnectionController();
|
||||||
diagnostics.record("extension.worker.started", {
|
const transport = new WSTransport({ url: __BSK_DAEMON_WS_URL__ });
|
||||||
diagnostic_build: "lumi-session-loss-v2",
|
const sessions = new SessionManager();
|
||||||
extension_version: __BSK_EXT_VERSION__,
|
|
||||||
extension_runtime_id: chrome.runtime.id,
|
|
||||||
daemon_ws_url: __BSK_DAEMON_WS_URL__,
|
|
||||||
});
|
|
||||||
const controller = new ConnectionController(diagnostics.sink);
|
|
||||||
const transport = new WSTransport({
|
|
||||||
url: __BSK_DAEMON_WS_URL__,
|
|
||||||
diagnostics: diagnostics.sink,
|
|
||||||
});
|
|
||||||
diagnostics.bindTransport((frame) => transport.send(frame));
|
|
||||||
const sessions = new SessionManager({ diagnostics: diagnostics.sink });
|
|
||||||
const cdp = new ChromiumCdp();
|
const cdp = new ChromiumCdp();
|
||||||
const sessionsLive = attachSessionsLiveFlag({ manager: sessions });
|
const sessionsLive = attachSessionsLiveFlag({ manager: sessions });
|
||||||
let overlayGeneration = 0;
|
let overlayGeneration = 0;
|
||||||
@@ -121,9 +109,6 @@ export default defineBackground(() => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function onOverlaySessionStateChanged(): void {
|
function onOverlaySessionStateChanged(): void {
|
||||||
diagnostics.record("background.sessions.changed", {
|
|
||||||
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
|
|
||||||
});
|
|
||||||
void sessionsLive.syncFromManager();
|
void sessionsLive.syncFromManager();
|
||||||
const liveSessionIds = new Set(sessions.list().map((ctx) => ctx.sessionId));
|
const liveSessionIds = new Set(sessions.list().map((ctx) => ctx.sessionId));
|
||||||
for (const sessionId of controlModes.keys()) {
|
for (const sessionId of controlModes.keys()) {
|
||||||
@@ -161,7 +146,6 @@ export default defineBackground(() => {
|
|||||||
onSessionsChanged: () => {
|
onSessionsChanged: () => {
|
||||||
void sessionsLive.syncFromManager();
|
void sessionsLive.syncFromManager();
|
||||||
},
|
},
|
||||||
diagnostics: diagnostics.sink,
|
|
||||||
});
|
});
|
||||||
const recordDeps = {
|
const recordDeps = {
|
||||||
tabsApi: chrome.tabs,
|
tabsApi: chrome.tabs,
|
||||||
@@ -209,7 +193,6 @@ export default defineBackground(() => {
|
|||||||
title: i18n.t("helpRequest.notificationTitle", { ns: "extension" }),
|
title: i18n.t("helpRequest.notificationTitle", { ns: "extension" }),
|
||||||
body: "",
|
body: "",
|
||||||
}),
|
}),
|
||||||
diagnostics: diagnostics.sink,
|
|
||||||
});
|
});
|
||||||
dispatcher.start();
|
dispatcher.start();
|
||||||
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
|
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
|
||||||
@@ -242,7 +225,6 @@ export default defineBackground(() => {
|
|||||||
transport,
|
transport,
|
||||||
cdp,
|
cdp,
|
||||||
onSessionsChanged: onOverlaySessionStateChanged,
|
onSessionsChanged: onOverlaySessionStateChanged,
|
||||||
diagnostics: diagnostics.sink,
|
|
||||||
});
|
});
|
||||||
|
|
||||||
// MV3 service worker keepalive + reconnect supervisor (review M4/M5
|
// MV3 service worker keepalive + reconnect supervisor (review M4/M5
|
||||||
@@ -254,7 +236,6 @@ export default defineBackground(() => {
|
|||||||
startKeepalive({
|
startKeepalive({
|
||||||
transport,
|
transport,
|
||||||
shouldConnect: () => controller.isConnectionEnabled,
|
shouldConnect: () => controller.isConnectionEnabled,
|
||||||
diagnostics: diagnostics.sink,
|
|
||||||
});
|
});
|
||||||
|
|
||||||
// Application-level heartbeat (Chrome 116+): while the post-handshake
|
// Application-level heartbeat (Chrome 116+): while the post-handshake
|
||||||
@@ -268,19 +249,6 @@ export default defineBackground(() => {
|
|||||||
controller.subscribe((snap) =>
|
controller.subscribe((snap) =>
|
||||||
cb(snap.state === "connected" || snap.state === "version_skew"),
|
cb(snap.state === "connected" || snap.state === "version_skew"),
|
||||||
),
|
),
|
||||||
diagnostics: diagnostics.sink,
|
|
||||||
});
|
|
||||||
|
|
||||||
controller.subscribe((snapshot) => {
|
|
||||||
const ready = snapshot.state === "connected" || snapshot.state === "version_skew";
|
|
||||||
diagnostics.setTransportReady(ready);
|
|
||||||
diagnostics.record("background.controller.snapshot", {
|
|
||||||
state: snapshot.state,
|
|
||||||
connection_enabled: snapshot.connectionEnabled,
|
|
||||||
instance_id: snapshot.instanceId,
|
|
||||||
last_error: snapshot.lastError,
|
|
||||||
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|
||||||
// Wake-driven reconnect (best effort). An MV3 service worker is killed
|
// Wake-driven reconnect (best effort). An MV3 service worker is killed
|
||||||
@@ -289,12 +257,7 @@ export default defineBackground(() => {
|
|||||||
// the worker and let us reconnect immediately instead of waiting for
|
// the worker and let us reconnect immediately instead of waiting for
|
||||||
// the next 30s alarm tick. They only help when the daemon is actually
|
// the next 30s alarm tick. They only help when the daemon is actually
|
||||||
// running; a cold daemon is (re)spawned by the next `bsk` command.
|
// running; a cold daemon is (re)spawned by the next `bsk` command.
|
||||||
const reconnectIfNeeded = (source: string) => {
|
const reconnectIfNeeded = () => {
|
||||||
diagnostics.record("background.wake_reconnect.requested", {
|
|
||||||
source,
|
|
||||||
controller_enabled: controller.isConnectionEnabled,
|
|
||||||
transport_state: transport.state,
|
|
||||||
});
|
|
||||||
if (!controller.isConnectionEnabled) return;
|
if (!controller.isConnectionEnabled) return;
|
||||||
if (transport.state === "connected") return;
|
if (transport.state === "connected") return;
|
||||||
void transport.connect().catch((err) => {
|
void transport.connect().catch((err) => {
|
||||||
@@ -302,14 +265,13 @@ export default defineBackground(() => {
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
if (typeof chrome.runtime?.onStartup?.addListener === "function") {
|
if (typeof chrome.runtime?.onStartup?.addListener === "function") {
|
||||||
chrome.runtime.onStartup.addListener(() => reconnectIfNeeded("runtime.onStartup"));
|
chrome.runtime.onStartup.addListener(reconnectIfNeeded);
|
||||||
}
|
}
|
||||||
if (typeof chrome.idle?.onStateChanged?.addListener === "function") {
|
if (typeof chrome.idle?.onStateChanged?.addListener === "function") {
|
||||||
chrome.idle.onStateChanged.addListener((state) => {
|
chrome.idle.onStateChanged.addListener((state) => {
|
||||||
diagnostics.record("chrome.idle.state_changed", { state });
|
|
||||||
// "active" fires when the user returns from idle/locked — the most
|
// "active" fires when the user returns from idle/locked — the most
|
||||||
// reliable "machine just woke" signal we get.
|
// reliable "machine just woke" signal we get.
|
||||||
if (state === "active") reconnectIfNeeded("idle.active");
|
if (state === "active") reconnectIfNeeded();
|
||||||
});
|
});
|
||||||
// Treat >60s of no input as idle so the active transition is timely.
|
// Treat >60s of no input as idle so the active transition is timely.
|
||||||
chrome.idle.setDetectionInterval?.(60);
|
chrome.idle.setDetectionInterval?.(60);
|
||||||
@@ -317,21 +279,14 @@ export default defineBackground(() => {
|
|||||||
|
|
||||||
void (async () => {
|
void (async () => {
|
||||||
const connectionEnabled = await getConnectionEnabled();
|
const connectionEnabled = await getConnectionEnabled();
|
||||||
diagnostics.record("background.controller.attach_started", { connectionEnabled });
|
|
||||||
await controller.attach(transport, detectBrowserMeta(), connectionEnabled, {
|
await controller.attach(transport, detectBrowserMeta(), connectionEnabled, {
|
||||||
beforeDisconnect: async () => {
|
beforeDisconnect: async () => {
|
||||||
diagnostics.record("background.before_disconnect_cleanup", {
|
|
||||||
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
|
|
||||||
});
|
|
||||||
const report = await cleanupAfterDisconnect();
|
const report = await cleanupAfterDisconnect();
|
||||||
if (report.failures.length > 0) {
|
if (report.failures.length > 0) {
|
||||||
console.warn("[browser-skill] session cleanup before disconnect was incomplete", report);
|
console.warn("[browser-skill] session cleanup before disconnect was incomplete", report);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
onDisconnected: async () => {
|
onDisconnected: async () => {
|
||||||
diagnostics.record("background.after_disconnect_cleanup", {
|
|
||||||
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
|
|
||||||
});
|
|
||||||
const report = await cleanupAfterDisconnect();
|
const report = await cleanupAfterDisconnect();
|
||||||
if (report.failures.length > 0) {
|
if (report.failures.length > 0) {
|
||||||
console.warn("[browser-skill] session cleanup after disconnect was incomplete", report);
|
console.warn("[browser-skill] session cleanup after disconnect was incomplete", report);
|
||||||
@@ -339,7 +294,6 @@ export default defineBackground(() => {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
})().catch((err) => {
|
})().catch((err) => {
|
||||||
diagnostics.record("background.controller.attach_failed", { error: err });
|
|
||||||
console.error("[browser-skill] controller failed to attach", err);
|
console.error("[browser-skill] controller failed to attach", err);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -420,15 +374,6 @@ export default defineBackground(() => {
|
|||||||
dbg.__bhDispatcher = dispatcher;
|
dbg.__bhDispatcher = dispatcher;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (typeof chrome.runtime?.onSuspend?.addListener === "function") {
|
|
||||||
chrome.runtime.onSuspend.addListener(() => {
|
|
||||||
diagnostics.record("extension.worker.suspending", {
|
|
||||||
live_session_ids: sessions.list().map((ctx) => ctx.sessionId),
|
|
||||||
transport_state: transport.state,
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
console.info("[browser-skill] background worker initialised");
|
console.info("[browser-skill] background worker initialised");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -1,85 +0,0 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
|
||||||
import { DiagnosticLogger } from "../diagnostics";
|
|
||||||
|
|
||||||
class MemoryStorage {
|
|
||||||
value: Record<string, unknown> = {};
|
|
||||||
|
|
||||||
async get(key: string): Promise<Record<string, unknown>> {
|
|
||||||
return { [key]: this.value[key] };
|
|
||||||
}
|
|
||||||
|
|
||||||
async set(items: Record<string, unknown>): Promise<void> {
|
|
||||||
this.value = { ...this.value, ...items };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
describe("DiagnosticLogger", () => {
|
|
||||||
it("persists evidence before transport readiness and flushes it after reconnect", async () => {
|
|
||||||
const storage = new MemoryStorage();
|
|
||||||
const send = vi.fn();
|
|
||||||
const logger = new DiagnosticLogger(storage);
|
|
||||||
logger.bindTransport(send);
|
|
||||||
|
|
||||||
logger.record("transport.socket.closed", { code: 1006 });
|
|
||||||
await logger.flush();
|
|
||||||
expect(send).not.toHaveBeenCalled();
|
|
||||||
|
|
||||||
logger.setTransportReady(true);
|
|
||||||
await logger.flush();
|
|
||||||
|
|
||||||
expect(send).toHaveBeenCalledOnce();
|
|
||||||
expect(send.mock.calls[0]?.[0]).toMatchObject({
|
|
||||||
event: "browser.diagnostic",
|
|
||||||
payload: {
|
|
||||||
schema_version: 1,
|
|
||||||
entries: [
|
|
||||||
{
|
|
||||||
event: "transport.socket.closed",
|
|
||||||
fields: { code: 1006 },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
});
|
|
||||||
expect(storage.value.bsk_diagnostic_events_v1).toEqual([]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("keeps persisted evidence when the transport rejects the flush", async () => {
|
|
||||||
const storage = new MemoryStorage();
|
|
||||||
const logger = new DiagnosticLogger(storage);
|
|
||||||
logger.bindTransport(() => {
|
|
||||||
throw new Error("socket closed");
|
|
||||||
});
|
|
||||||
logger.record("session.stop.requested", { session_id: "abcd" });
|
|
||||||
|
|
||||||
logger.setTransportReady(true);
|
|
||||||
await logger.flush();
|
|
||||||
|
|
||||||
expect(storage.value.bsk_diagnostic_events_v1).toMatchObject([
|
|
||||||
{ event: "session.stop.requested", fields: { session_id: "abcd" } },
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("does not resend an acknowledged event when send records a new event", async () => {
|
|
||||||
const storage = new MemoryStorage();
|
|
||||||
const batches: Array<{ payload?: { entries?: Array<{ event: string }> } }> = [];
|
|
||||||
const logger = new DiagnosticLogger(storage);
|
|
||||||
logger.record("dispatcher.rpc.received", { rpc_id: "one" });
|
|
||||||
await logger.flush();
|
|
||||||
logger.bindTransport((frame) => {
|
|
||||||
batches.push(frame as { payload?: { entries?: Array<{ event: string }> } });
|
|
||||||
if (batches.length === 1) {
|
|
||||||
logger.record("dispatcher.rpc.response_sent", { rpc_id: "one" });
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
logger.setTransportReady(true);
|
|
||||||
await logger.flush();
|
|
||||||
await logger.flush();
|
|
||||||
|
|
||||||
expect(batches.map((batch) => batch.payload?.entries?.map((entry) => entry.event))).toEqual([
|
|
||||||
["dispatcher.rpc.received"],
|
|
||||||
["dispatcher.rpc.response_sent"],
|
|
||||||
]);
|
|
||||||
expect(storage.value.bsk_diagnostic_events_v1).toEqual([]);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -6,7 +6,6 @@ import {
|
|||||||
} from "../transport/handshake";
|
} from "../transport/handshake";
|
||||||
import type { Transport } from "../transport/transport";
|
import type { Transport } from "../transport/transport";
|
||||||
import type { ConnectionState, HandshakeResult } from "../transport/types";
|
import type { ConnectionState, HandshakeResult } from "../transport/types";
|
||||||
import type { DiagnosticSink } from "./diagnostics";
|
|
||||||
import { getLabel, getOrCreateInstanceId } from "./instance-id";
|
import { getLabel, getOrCreateInstanceId } from "./instance-id";
|
||||||
import { compareProtocol, parseProtocolMajor } from "./semver";
|
import { compareProtocol, parseProtocolMajor } from "./semver";
|
||||||
|
|
||||||
@@ -52,11 +51,6 @@ export class ConnectionController {
|
|||||||
private handshakeAbort: AbortController | null = null;
|
private handshakeAbort: AbortController | null = null;
|
||||||
private handshakeRetryTimer: ReturnType<typeof setTimeout> | null = null;
|
private handshakeRetryTimer: ReturnType<typeof setTimeout> | null = null;
|
||||||
private suppressDisconnectRecovery = false;
|
private suppressDisconnectRecovery = false;
|
||||||
private readonly diagnostics?: DiagnosticSink;
|
|
||||||
|
|
||||||
constructor(diagnostics?: DiagnosticSink) {
|
|
||||||
this.diagnostics = diagnostics;
|
|
||||||
}
|
|
||||||
|
|
||||||
get isConnectionEnabled(): boolean {
|
get isConnectionEnabled(): boolean {
|
||||||
return this.connectionEnabled;
|
return this.connectionEnabled;
|
||||||
@@ -95,20 +89,8 @@ export class ConnectionController {
|
|||||||
this.lifecycleHooks = lifecycleHooks;
|
this.lifecycleHooks = lifecycleHooks;
|
||||||
this.instanceId = await getOrCreateInstanceId();
|
this.instanceId = await getOrCreateInstanceId();
|
||||||
this.label = await getLabel();
|
this.label = await getLabel();
|
||||||
this.diagnostics?.("controller.attach", {
|
|
||||||
instance_id: this.instanceId,
|
|
||||||
connection_enabled: connectionEnabled,
|
|
||||||
transport_state: transport.state,
|
|
||||||
});
|
|
||||||
|
|
||||||
transport.onConnectionStateChange((s) => {
|
transport.onConnectionStateChange((s) => {
|
||||||
this.diagnostics?.("controller.transport_state", {
|
|
||||||
transport_state: s,
|
|
||||||
controller_state: this.currentState,
|
|
||||||
connection_generation: this.connectionGeneration,
|
|
||||||
suppress_disconnect_recovery: this.suppressDisconnectRecovery,
|
|
||||||
recovery_inflight: this.disconnectRecovery !== null,
|
|
||||||
});
|
|
||||||
if (s === "disconnected") {
|
if (s === "disconnected") {
|
||||||
// An in-flight handshake belongs to the dead connection — cancel it so
|
// An in-flight handshake belongs to the dead connection — cancel it so
|
||||||
// a late resolution can never clobber the next attempt.
|
// a late resolution can never clobber the next attempt.
|
||||||
@@ -121,9 +103,6 @@ export class ConnectionController {
|
|||||||
// handshake bounce); the caller owns the reconnect policy there.
|
// handshake bounce); the caller owns the reconnect policy there.
|
||||||
this.suppressDisconnectRecovery = false;
|
this.suppressDisconnectRecovery = false;
|
||||||
} else {
|
} else {
|
||||||
this.diagnostics?.("controller.disconnect_recovery.requested", {
|
|
||||||
connection_generation: this.connectionGeneration,
|
|
||||||
});
|
|
||||||
void this.recoverFromDisconnect();
|
void this.recoverFromDisconnect();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -182,12 +161,6 @@ export class ConnectionController {
|
|||||||
const generation = ++this.connectionGeneration;
|
const generation = ++this.connectionGeneration;
|
||||||
const abort = new AbortController();
|
const abort = new AbortController();
|
||||||
this.handshakeAbort = abort;
|
this.handshakeAbort = abort;
|
||||||
this.diagnostics?.("controller.handshake.started", {
|
|
||||||
connection_generation: generation,
|
|
||||||
browser_name: browser.name,
|
|
||||||
browser_version: browser.version,
|
|
||||||
instance_id: this.instanceId,
|
|
||||||
});
|
|
||||||
void this.runHandshake(browser, generation, abort.signal);
|
void this.runHandshake(browser, generation, abort.signal);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -224,21 +197,11 @@ export class ConnectionController {
|
|||||||
}
|
}
|
||||||
this.clearHandshakeRetry();
|
this.clearHandshakeRetry();
|
||||||
this.lastError = null;
|
this.lastError = null;
|
||||||
this.diagnostics?.("controller.handshake.completed", {
|
|
||||||
connection_generation: generation,
|
|
||||||
verdict: verdict.kind,
|
|
||||||
daemon_version: outcome.result.version,
|
|
||||||
daemon_protocol: outcome.result.protocol_version,
|
|
||||||
});
|
|
||||||
this.setState(verdict.kind);
|
this.setState(verdict.kind);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (generation !== this.connectionGeneration || signal.aborted || isAbortError(err)) return;
|
if (generation !== this.connectionGeneration || signal.aborted || isAbortError(err)) return;
|
||||||
this.handshake = null;
|
this.handshake = null;
|
||||||
this.lastError = err instanceof Error ? err.message : String(err);
|
this.lastError = err instanceof Error ? err.message : String(err);
|
||||||
this.diagnostics?.("controller.handshake.failed", {
|
|
||||||
connection_generation: generation,
|
|
||||||
error: err,
|
|
||||||
});
|
|
||||||
// Bounce the half-open socket deliberately; the retry timer below owns
|
// Bounce the half-open socket deliberately; the retry timer below owns
|
||||||
// the reconnect, so skip unexpected-loss recovery for this disconnect.
|
// the reconnect, so skip unexpected-loss recovery for this disconnect.
|
||||||
this.suppressDisconnectRecovery = true;
|
this.suppressDisconnectRecovery = true;
|
||||||
@@ -265,57 +228,23 @@ export class ConnectionController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private recoverFromDisconnect(): Promise<void> {
|
private recoverFromDisconnect(): Promise<void> {
|
||||||
if (this.disconnectRecovery) {
|
if (this.disconnectRecovery) return this.disconnectRecovery;
|
||||||
this.diagnostics?.("controller.disconnect_recovery.coalesced", {
|
|
||||||
connection_generation: this.connectionGeneration,
|
|
||||||
});
|
|
||||||
return this.disconnectRecovery;
|
|
||||||
}
|
|
||||||
const recovery = (async () => {
|
const recovery = (async () => {
|
||||||
const recoveryGeneration = this.connectionGeneration;
|
|
||||||
this.diagnostics?.("controller.disconnect_recovery.started", {
|
|
||||||
recovery_generation: recoveryGeneration,
|
|
||||||
});
|
|
||||||
// WSTransport schedules its reconnect timer immediately after notifying
|
// WSTransport schedules its reconnect timer immediately after notifying
|
||||||
// state listeners. Yield once, then disconnect explicitly so that timer
|
// state listeners. Yield once, then disconnect explicitly so that timer
|
||||||
// is cancelled before local session teardown begins.
|
// is cancelled before local session teardown begins.
|
||||||
await Promise.resolve();
|
await Promise.resolve();
|
||||||
if (!this.connectionEnabled || !this.transport) {
|
if (!this.connectionEnabled || !this.transport) return;
|
||||||
this.diagnostics?.("controller.disconnect_recovery.skipped", {
|
|
||||||
recovery_generation: recoveryGeneration,
|
|
||||||
reason: "disabled_or_missing_transport",
|
|
||||||
});
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
// Another path already reconnected while we yielded — nothing to do.
|
// Another path already reconnected while we yielded — nothing to do.
|
||||||
if (this.currentState !== "disconnected") {
|
if (this.currentState !== "disconnected") return;
|
||||||
this.diagnostics?.("controller.disconnect_recovery.skipped", {
|
|
||||||
recovery_generation: recoveryGeneration,
|
|
||||||
reason: "state_already_changed",
|
|
||||||
current_state: this.currentState,
|
|
||||||
});
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
await this.transport.disconnect().catch(() => {});
|
await this.transport.disconnect().catch(() => {});
|
||||||
try {
|
try {
|
||||||
this.diagnostics?.("controller.disconnect_cleanup.started", {
|
|
||||||
recovery_generation: recoveryGeneration,
|
|
||||||
current_generation: this.connectionGeneration,
|
|
||||||
});
|
|
||||||
await this.lifecycleHooks.onDisconnected?.();
|
await this.lifecycleHooks.onDisconnected?.();
|
||||||
this.diagnostics?.("controller.disconnect_cleanup.completed", {
|
|
||||||
recovery_generation: recoveryGeneration,
|
|
||||||
current_generation: this.connectionGeneration,
|
|
||||||
});
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.warn("[browser-skill] session cleanup after disconnect failed", err);
|
console.warn("[browser-skill] session cleanup after disconnect failed", err);
|
||||||
}
|
}
|
||||||
if (!this.connectionEnabled) return;
|
if (!this.connectionEnabled) return;
|
||||||
try {
|
try {
|
||||||
this.diagnostics?.("controller.disconnect_reconnect.started", {
|
|
||||||
recovery_generation: recoveryGeneration,
|
|
||||||
current_generation: this.connectionGeneration,
|
|
||||||
});
|
|
||||||
await this.transport.connect();
|
await this.transport.connect();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
this.lastError = err instanceof Error ? err.message : String(err);
|
this.lastError = err instanceof Error ? err.message : String(err);
|
||||||
@@ -323,39 +252,22 @@ export class ConnectionController {
|
|||||||
}
|
}
|
||||||
})();
|
})();
|
||||||
this.disconnectRecovery = recovery.finally(() => {
|
this.disconnectRecovery = recovery.finally(() => {
|
||||||
this.diagnostics?.("controller.disconnect_recovery.completed", {
|
|
||||||
connection_generation: this.connectionGeneration,
|
|
||||||
current_state: this.currentState,
|
|
||||||
});
|
|
||||||
this.disconnectRecovery = null;
|
this.disconnectRecovery = null;
|
||||||
});
|
});
|
||||||
return this.disconnectRecovery;
|
return this.disconnectRecovery;
|
||||||
}
|
}
|
||||||
|
|
||||||
private cancelHandshake(): void {
|
private cancelHandshake(): void {
|
||||||
const previousGeneration = this.connectionGeneration;
|
|
||||||
this.connectionGeneration += 1;
|
this.connectionGeneration += 1;
|
||||||
this.diagnostics?.("controller.handshake.cancelled", {
|
|
||||||
previous_generation: previousGeneration,
|
|
||||||
next_generation: this.connectionGeneration,
|
|
||||||
had_abort_controller: this.handshakeAbort !== null,
|
|
||||||
});
|
|
||||||
this.handshakeAbort?.abort();
|
this.handshakeAbort?.abort();
|
||||||
this.handshakeAbort = null;
|
this.handshakeAbort = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
private scheduleHandshakeRetry(browser: { name: string; version: string }): void {
|
private scheduleHandshakeRetry(browser: { name: string; version: string }): void {
|
||||||
if (this.handshakeRetryTimer || !this.connectionEnabled) return;
|
if (this.handshakeRetryTimer || !this.connectionEnabled) return;
|
||||||
this.diagnostics?.("controller.handshake_retry.scheduled", {
|
|
||||||
delay_ms: HANDSHAKE_RETRY_DELAY_MS,
|
|
||||||
connection_generation: this.connectionGeneration,
|
|
||||||
});
|
|
||||||
this.handshakeRetryTimer = setTimeout(() => {
|
this.handshakeRetryTimer = setTimeout(() => {
|
||||||
this.handshakeRetryTimer = null;
|
this.handshakeRetryTimer = null;
|
||||||
if (!this.connectionEnabled || !this.transport) return;
|
if (!this.connectionEnabled || !this.transport) return;
|
||||||
this.diagnostics?.("controller.handshake_retry.timer_fired", {
|
|
||||||
connection_generation: this.connectionGeneration,
|
|
||||||
});
|
|
||||||
void this.transport.connect().catch((err) => {
|
void this.transport.connect().catch((err) => {
|
||||||
this.lastError = err instanceof Error ? err.message : String(err);
|
this.lastError = err instanceof Error ? err.message : String(err);
|
||||||
this.setState("disconnected");
|
this.setState("disconnected");
|
||||||
|
|||||||
@@ -1,178 +0,0 @@
|
|||||||
import type { ProtocolFrame } from "@/transport/types";
|
|
||||||
|
|
||||||
const STORAGE_KEY = "bsk_diagnostic_events_v1";
|
|
||||||
const MAX_ENTRIES = 500;
|
|
||||||
|
|
||||||
export type DiagnosticFields = Record<string, unknown>;
|
|
||||||
export type DiagnosticSink = (event: string, fields?: DiagnosticFields) => void;
|
|
||||||
|
|
||||||
export interface DiagnosticEntry {
|
|
||||||
id: string;
|
|
||||||
sequence: number;
|
|
||||||
timestamp: string;
|
|
||||||
monotonic_ms: number;
|
|
||||||
worker_boot_id: string;
|
|
||||||
event: string;
|
|
||||||
fields: DiagnosticFields;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface DiagnosticStorage {
|
|
||||||
get(key: string): Promise<Record<string, unknown>>;
|
|
||||||
set(items: Record<string, unknown>): Promise<void>;
|
|
||||||
}
|
|
||||||
|
|
||||||
function defaultStorage(): DiagnosticStorage | null {
|
|
||||||
if (typeof chrome === "undefined" || !chrome.storage?.local) return null;
|
|
||||||
return {
|
|
||||||
get: (key) => chrome.storage.local.get(key),
|
|
||||||
set: (items) => chrome.storage.local.set(items),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function makeBootId(): string {
|
|
||||||
try {
|
|
||||||
return crypto.randomUUID();
|
|
||||||
} catch {
|
|
||||||
return `worker-${Date.now()}-${Math.random().toString(16).slice(2)}`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function normalise(value: unknown): unknown {
|
|
||||||
if (value instanceof Error) {
|
|
||||||
return { name: value.name, message: value.message, stack: value.stack };
|
|
||||||
}
|
|
||||||
if (typeof value === "bigint") return value.toString();
|
|
||||||
if (Array.isArray(value)) return value.map(normalise);
|
|
||||||
if (value && typeof value === "object") {
|
|
||||||
return Object.fromEntries(
|
|
||||||
Object.entries(value as Record<string, unknown>).map(([key, child]) => [
|
|
||||||
key,
|
|
||||||
normalise(child),
|
|
||||||
]),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Persistent diagnostics for the MV3 background worker.
|
|
||||||
*
|
|
||||||
* Events are first written to chrome.storage.local so a worker termination or
|
|
||||||
* dead WebSocket does not erase the evidence. Once the post-handshake link is
|
|
||||||
* ready, pending entries are forwarded to the daemon as one
|
|
||||||
* `browser.diagnostic` event and removed from the local queue.
|
|
||||||
*/
|
|
||||||
export class DiagnosticLogger {
|
|
||||||
readonly workerBootId: string;
|
|
||||||
readonly sink: DiagnosticSink;
|
|
||||||
|
|
||||||
private sequence = 0;
|
|
||||||
private storageQueue: Promise<void> = Promise.resolve();
|
|
||||||
private flushPromise: Promise<void> | null = null;
|
|
||||||
private flushRequested = false;
|
|
||||||
private send: ((frame: ProtocolFrame) => void) | null = null;
|
|
||||||
private transportReady = false;
|
|
||||||
private readonly storage: DiagnosticStorage | null;
|
|
||||||
|
|
||||||
constructor(storage: DiagnosticStorage | null = defaultStorage()) {
|
|
||||||
this.workerBootId = makeBootId();
|
|
||||||
this.storage = storage;
|
|
||||||
this.sink = (event, fields = {}) => this.record(event, fields);
|
|
||||||
}
|
|
||||||
|
|
||||||
bindTransport(send: (frame: ProtocolFrame) => void): void {
|
|
||||||
this.send = send;
|
|
||||||
}
|
|
||||||
|
|
||||||
setTransportReady(ready: boolean): void {
|
|
||||||
this.transportReady = ready;
|
|
||||||
if (ready) void this.flush();
|
|
||||||
}
|
|
||||||
|
|
||||||
record(event: string, fields: DiagnosticFields = {}): void {
|
|
||||||
const sequence = ++this.sequence;
|
|
||||||
const entry: DiagnosticEntry = {
|
|
||||||
id: `${this.workerBootId}:${sequence}`,
|
|
||||||
sequence,
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
monotonic_ms: Math.round(performance.now()),
|
|
||||||
worker_boot_id: this.workerBootId,
|
|
||||||
event,
|
|
||||||
fields: normalise(fields) as DiagnosticFields,
|
|
||||||
};
|
|
||||||
console.info("[bsk diagnostic]", entry);
|
|
||||||
this.storageQueue = this.storageQueue
|
|
||||||
.then(async () => {
|
|
||||||
if (!this.storage) return;
|
|
||||||
const stored = await this.storage.get(STORAGE_KEY);
|
|
||||||
const previous = Array.isArray(stored[STORAGE_KEY])
|
|
||||||
? (stored[STORAGE_KEY] as DiagnosticEntry[])
|
|
||||||
: [];
|
|
||||||
await this.storage.set({
|
|
||||||
[STORAGE_KEY]: [...previous, entry].slice(-MAX_ENTRIES),
|
|
||||||
});
|
|
||||||
})
|
|
||||||
.catch((error) => {
|
|
||||||
console.warn("[bsk diagnostic] failed to persist event", error);
|
|
||||||
});
|
|
||||||
if (this.transportReady) void this.flush();
|
|
||||||
}
|
|
||||||
|
|
||||||
flush(): Promise<void> {
|
|
||||||
if (this.flushPromise) {
|
|
||||||
this.flushRequested = true;
|
|
||||||
return this.flushPromise;
|
|
||||||
}
|
|
||||||
const running = (async () => {
|
|
||||||
await this.storageQueue;
|
|
||||||
if (!this.transportReady || !this.send || !this.storage) return;
|
|
||||||
const stored = await this.storage.get(STORAGE_KEY);
|
|
||||||
const entries = Array.isArray(stored[STORAGE_KEY])
|
|
||||||
? (stored[STORAGE_KEY] as DiagnosticEntry[])
|
|
||||||
: [];
|
|
||||||
if (entries.length === 0) return;
|
|
||||||
try {
|
|
||||||
this.send({
|
|
||||||
event: "browser.diagnostic",
|
|
||||||
payload: {
|
|
||||||
schema_version: 1,
|
|
||||||
worker_boot_id: this.workerBootId,
|
|
||||||
entries,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
} catch (error) {
|
|
||||||
console.warn("[bsk diagnostic] daemon flush failed", error);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const sentIds = new Set(entries.map((entry) => entry.id));
|
|
||||||
// Serialize acknowledgement behind every record write that may have
|
|
||||||
// started while send() was in progress. Without this, a concurrent
|
|
||||||
// record can read the pre-ack queue and write an already-sent entry
|
|
||||||
// back into storage, causing the next batch to duplicate it.
|
|
||||||
this.storageQueue = this.storageQueue
|
|
||||||
.then(async () => {
|
|
||||||
const current = await this.storage?.get(STORAGE_KEY);
|
|
||||||
const currentEntries = Array.isArray(current?.[STORAGE_KEY])
|
|
||||||
? (current[STORAGE_KEY] as DiagnosticEntry[])
|
|
||||||
: [];
|
|
||||||
await this.storage?.set({
|
|
||||||
[STORAGE_KEY]: currentEntries.filter((entry) => !sentIds.has(entry.id)),
|
|
||||||
});
|
|
||||||
})
|
|
||||||
.catch((error) => {
|
|
||||||
console.warn("[bsk diagnostic] failed to acknowledge event batch", error);
|
|
||||||
});
|
|
||||||
await this.storageQueue;
|
|
||||||
})().catch((error) => {
|
|
||||||
console.warn("[bsk diagnostic] flush failed", error);
|
|
||||||
});
|
|
||||||
this.flushPromise = running.finally(() => {
|
|
||||||
this.flushPromise = null;
|
|
||||||
if (this.flushRequested && this.transportReady) {
|
|
||||||
this.flushRequested = false;
|
|
||||||
void this.flush();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
return this.flushPromise;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -21,7 +21,6 @@
|
|||||||
* after the handshake has landed.
|
* after the handshake has landed.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import type { DiagnosticSink } from "@/lib/diagnostics";
|
|
||||||
import type { ProtocolFrame } from "@/transport/types";
|
import type { ProtocolFrame } from "@/transport/types";
|
||||||
|
|
||||||
export const HEARTBEAT_EVENT = "system.heartbeat";
|
export const HEARTBEAT_EVENT = "system.heartbeat";
|
||||||
@@ -47,7 +46,6 @@ export interface HeartbeatOptions {
|
|||||||
intervalMs?: number;
|
intervalMs?: number;
|
||||||
/** Inject fake timers (tests). */
|
/** Inject fake timers (tests). */
|
||||||
timers?: HeartbeatTimers;
|
timers?: HeartbeatTimers;
|
||||||
diagnostics?: DiagnosticSink;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface HeartbeatHandle {
|
export interface HeartbeatHandle {
|
||||||
@@ -76,9 +74,7 @@ export function startHeartbeat(options: HeartbeatOptions): HeartbeatHandle {
|
|||||||
const beat = () => {
|
const beat = () => {
|
||||||
try {
|
try {
|
||||||
options.send({ event: HEARTBEAT_EVENT, payload: {} });
|
options.send({ event: HEARTBEAT_EVENT, payload: {} });
|
||||||
options.diagnostics?.("heartbeat.sent");
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
options.diagnostics?.("heartbeat.send_failed", { error: err });
|
|
||||||
// The socket raced a close between the state change and this
|
// The socket raced a close between the state change and this
|
||||||
// tick. The transport's own reconnect path recovers; swallow so
|
// tick. The transport's own reconnect path recovers; swallow so
|
||||||
// the interval keeps running (it is cleared on the disconnect
|
// the interval keeps running (it is cleared on the disconnect
|
||||||
@@ -91,14 +87,12 @@ export function startHeartbeat(options: HeartbeatOptions): HeartbeatHandle {
|
|||||||
if (timer !== null) {
|
if (timer !== null) {
|
||||||
timers.clearInterval(timer);
|
timers.clearInterval(timer);
|
||||||
timer = null;
|
timer = null;
|
||||||
options.diagnostics?.("heartbeat.timer.stopped");
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const start = () => {
|
const start = () => {
|
||||||
if (timer !== null) return;
|
if (timer !== null) return;
|
||||||
timer = timers.setInterval(beat, intervalMs);
|
timer = timers.setInterval(beat, intervalMs);
|
||||||
options.diagnostics?.("heartbeat.timer.started", { interval_ms: intervalMs });
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const unsubscribe = options.onActiveChange((active) => {
|
const unsubscribe = options.onActiveChange((active) => {
|
||||||
|
|||||||
@@ -20,7 +20,6 @@
|
|||||||
* SW idle threshold.
|
* SW idle threshold.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import type { DiagnosticSink } from "@/lib/diagnostics";
|
|
||||||
import type { Transport } from "@/transport/transport";
|
import type { Transport } from "@/transport/transport";
|
||||||
|
|
||||||
export const KEEPALIVE_ALARM_NAME = "bh-keepalive";
|
export const KEEPALIVE_ALARM_NAME = "bh-keepalive";
|
||||||
@@ -60,7 +59,6 @@ export interface KeepaliveOptions {
|
|||||||
alarmName?: string;
|
alarmName?: string;
|
||||||
/** Override the alarm period (tests). */
|
/** Override the alarm period (tests). */
|
||||||
periodMin?: number;
|
periodMin?: number;
|
||||||
diagnostics?: DiagnosticSink;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface KeepaliveHandle {
|
export interface KeepaliveHandle {
|
||||||
@@ -81,24 +79,11 @@ export function startKeepalive(options: KeepaliveOptions): KeepaliveHandle {
|
|||||||
const period = options.periodMin ?? KEEPALIVE_PERIOD_MIN;
|
const period = options.periodMin ?? KEEPALIVE_PERIOD_MIN;
|
||||||
|
|
||||||
const tick = async () => {
|
const tick = async () => {
|
||||||
if (options.shouldConnect && !options.shouldConnect()) {
|
if (options.shouldConnect && !options.shouldConnect()) return;
|
||||||
options.diagnostics?.("keepalive.tick.skipped", { reason: "connection_disabled" });
|
if (options.transport.state === "connected") return;
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (options.transport.state === "connected") {
|
|
||||||
options.diagnostics?.("keepalive.tick.healthy", { transport_state: "connected" });
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
options.diagnostics?.("keepalive.reconnect.started", {
|
|
||||||
transport_state: options.transport.state,
|
|
||||||
});
|
|
||||||
try {
|
try {
|
||||||
await options.transport.connect();
|
await options.transport.connect();
|
||||||
options.diagnostics?.("keepalive.reconnect.completed", {
|
|
||||||
transport_state: options.transport.state,
|
|
||||||
});
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
options.diagnostics?.("keepalive.reconnect.failed", { error: err });
|
|
||||||
console.debug("[bsk keepalive] connect attempt failed", err);
|
console.debug("[bsk keepalive] connect attempt failed", err);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
import type { DiagnosticSink } from "@/lib/diagnostics";
|
|
||||||
import { handleSessionStop, type SessionStopDeps } from "@/tools/session";
|
import { handleSessionStop, type SessionStopDeps } from "@/tools/session";
|
||||||
import type { SessionManager } from "./manager";
|
import type { SessionManager } from "./manager";
|
||||||
|
|
||||||
@@ -16,7 +15,6 @@ export interface DisconnectCleanupOptions {
|
|||||||
manager: SessionManager;
|
manager: SessionManager;
|
||||||
sessionStopDeps?: SessionStopDeps;
|
sessionStopDeps?: SessionStopDeps;
|
||||||
onSessionsChanged?: () => void;
|
onSessionsChanged?: () => void;
|
||||||
diagnostics?: DiagnosticSink;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -31,15 +29,7 @@ export function createDisconnectCleanup(options: DisconnectCleanupOptions) {
|
|||||||
let inFlight: Promise<DisconnectCleanupReport> | null = null;
|
let inFlight: Promise<DisconnectCleanupReport> | null = null;
|
||||||
|
|
||||||
return (): Promise<DisconnectCleanupReport> => {
|
return (): Promise<DisconnectCleanupReport> => {
|
||||||
if (inFlight) {
|
if (inFlight) return inFlight;
|
||||||
options.diagnostics?.("disconnect_cleanup.coalesced", {
|
|
||||||
live_session_ids: options.manager.list().map((ctx) => ctx.sessionId),
|
|
||||||
});
|
|
||||||
return inFlight;
|
|
||||||
}
|
|
||||||
options.diagnostics?.("disconnect_cleanup.scheduled", {
|
|
||||||
live_session_ids: options.manager.list().map((ctx) => ctx.sessionId),
|
|
||||||
});
|
|
||||||
inFlight = cleanupSessions(options).finally(() => {
|
inFlight = cleanupSessions(options).finally(() => {
|
||||||
inFlight = null;
|
inFlight = null;
|
||||||
});
|
});
|
||||||
@@ -52,17 +42,8 @@ async function cleanupSessions(
|
|||||||
): Promise<DisconnectCleanupReport> {
|
): Promise<DisconnectCleanupReport> {
|
||||||
const stoppedSessionIds: string[] = [];
|
const stoppedSessionIds: string[] = [];
|
||||||
const failures: DisconnectCleanupFailure[] = [];
|
const failures: DisconnectCleanupFailure[] = [];
|
||||||
const capturedSessions = options.manager.list();
|
|
||||||
options.diagnostics?.("disconnect_cleanup.started", {
|
|
||||||
captured_session_ids: capturedSessions.map((ctx) => ctx.sessionId),
|
|
||||||
});
|
|
||||||
|
|
||||||
for (const ctx of capturedSessions) {
|
for (const ctx of options.manager.list()) {
|
||||||
options.diagnostics?.("disconnect_cleanup.session_stop.started", {
|
|
||||||
session_id: ctx.sessionId,
|
|
||||||
agent_window_id: ctx.agentWindowId,
|
|
||||||
still_registered: options.manager.has(ctx.sessionId),
|
|
||||||
});
|
|
||||||
try {
|
try {
|
||||||
const result = await handleSessionStop(
|
const result = await handleSessionStop(
|
||||||
options.manager,
|
options.manager,
|
||||||
@@ -81,26 +62,14 @@ async function cleanupSessions(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
stoppedSessionIds.push(ctx.sessionId);
|
stoppedSessionIds.push(ctx.sessionId);
|
||||||
options.diagnostics?.("disconnect_cleanup.session_stop.completed", {
|
|
||||||
session_id: ctx.sessionId,
|
|
||||||
});
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
failures.push({
|
failures.push({
|
||||||
sessionId: ctx.sessionId,
|
sessionId: ctx.sessionId,
|
||||||
message: err instanceof Error ? err.message : String(err),
|
message: err instanceof Error ? err.message : String(err),
|
||||||
});
|
});
|
||||||
options.diagnostics?.("disconnect_cleanup.session_stop.failed", {
|
|
||||||
session_id: ctx.sessionId,
|
|
||||||
error: err,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
options.onSessionsChanged?.();
|
options.onSessionsChanged?.();
|
||||||
options.diagnostics?.("disconnect_cleanup.completed", {
|
|
||||||
stopped_session_ids: stoppedSessionIds,
|
|
||||||
failures,
|
|
||||||
remaining_session_ids: options.manager.list().map((ctx) => ctx.sessionId),
|
|
||||||
});
|
|
||||||
return { stoppedSessionIds, failures };
|
return { stoppedSessionIds, failures };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
import type { DiagnosticSink } from "@/lib/diagnostics";
|
|
||||||
import type { Transport } from "@/transport/transport";
|
import type { Transport } from "@/transport/transport";
|
||||||
import type { EventFrame } from "@/transport/types";
|
import type { EventFrame } from "@/transport/types";
|
||||||
import type { SessionManager } from "./manager";
|
import type { SessionManager } from "./manager";
|
||||||
@@ -25,7 +24,6 @@ export interface SessionEventHandlerOptions {
|
|||||||
* `chrome.storage.session` "sessions live" flag (review M4/M5 I3).
|
* `chrome.storage.session` "sessions live" flag (review M4/M5 I3).
|
||||||
*/
|
*/
|
||||||
onSessionsChanged?: () => void;
|
onSessionsChanged?: () => void;
|
||||||
diagnostics?: DiagnosticSink;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function chromeWindowEvents(): WindowRemovedListener {
|
function chromeWindowEvents(): WindowRemovedListener {
|
||||||
@@ -52,11 +50,6 @@ export function attachSessionEventHandler(options: SessionEventHandlerOptions):
|
|||||||
|
|
||||||
const onRemoved = (windowId: number): void => {
|
const onRemoved = (windowId: number): void => {
|
||||||
const ctx = manager.findByWindowId(windowId);
|
const ctx = manager.findByWindowId(windowId);
|
||||||
options.diagnostics?.("chrome.window.removed", {
|
|
||||||
window_id: windowId,
|
|
||||||
matched_session_id: ctx?.sessionId ?? null,
|
|
||||||
live_session_ids: manager.list().map((entry) => entry.sessionId),
|
|
||||||
});
|
|
||||||
if (!ctx) return;
|
if (!ctx) return;
|
||||||
const returnFailures = Array.from(ctx.borrowedTabs.keys()).map((tabId) => ({
|
const returnFailures = Array.from(ctx.borrowedTabs.keys()).map((tabId) => ({
|
||||||
tab_id: tabId,
|
tab_id: tabId,
|
||||||
@@ -88,16 +81,7 @@ export function attachSessionEventHandler(options: SessionEventHandlerOptions):
|
|||||||
};
|
};
|
||||||
try {
|
try {
|
||||||
transport.send(event);
|
transport.send(event);
|
||||||
options.diagnostics?.("session.window_closed.sent", {
|
|
||||||
session_id: ctx.sessionId,
|
|
||||||
window_id: windowId,
|
|
||||||
});
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
options.diagnostics?.("session.window_closed.send_failed", {
|
|
||||||
session_id: ctx.sessionId,
|
|
||||||
window_id: windowId,
|
|
||||||
error: err,
|
|
||||||
});
|
|
||||||
console.warn("[bh] could not push session.window_closed event", err);
|
console.warn("[bh] could not push session.window_closed event", err);
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
import type { DiagnosticSink } from "@/lib/diagnostics";
|
|
||||||
import { AGENT_WINDOW_HOME, type AgentWindowApi, chromeAgentWindowApi } from "./agent-window";
|
import { AGENT_WINDOW_HOME, type AgentWindowApi, chromeAgentWindowApi } from "./agent-window";
|
||||||
import { RefStore } from "./ref-store";
|
import { RefStore } from "./ref-store";
|
||||||
|
|
||||||
@@ -24,7 +23,6 @@ export interface BorrowReservation {
|
|||||||
export interface SessionManagerOptions {
|
export interface SessionManagerOptions {
|
||||||
agentWindow?: AgentWindowApi;
|
agentWindow?: AgentWindowApi;
|
||||||
now?: () => number;
|
now?: () => number;
|
||||||
diagnostics?: DiagnosticSink;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Options for starting a session's Agent Window. */
|
/** Options for starting a session's Agent Window. */
|
||||||
@@ -85,12 +83,10 @@ export class SessionManager {
|
|||||||
private readonly borrowReservations = new Map<number, string>();
|
private readonly borrowReservations = new Map<number, string>();
|
||||||
private readonly agentWindow: AgentWindowApi;
|
private readonly agentWindow: AgentWindowApi;
|
||||||
private readonly now: () => number;
|
private readonly now: () => number;
|
||||||
private readonly diagnostics?: DiagnosticSink;
|
|
||||||
|
|
||||||
constructor(options: SessionManagerOptions = {}) {
|
constructor(options: SessionManagerOptions = {}) {
|
||||||
this.agentWindow = options.agentWindow ?? chromeAgentWindowApi;
|
this.agentWindow = options.agentWindow ?? chromeAgentWindowApi;
|
||||||
this.now = options.now ?? Date.now;
|
this.now = options.now ?? Date.now;
|
||||||
this.diagnostics = options.diagnostics;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
has(sessionId: string): boolean {
|
has(sessionId: string): boolean {
|
||||||
@@ -173,13 +169,6 @@ export class SessionManager {
|
|||||||
* daemon in the `tool.session_start` reply.
|
* daemon in the `tool.session_start` reply.
|
||||||
*/
|
*/
|
||||||
async start(sessionId: string, opts: SessionStartOptions = {}): Promise<SessionContext> {
|
async start(sessionId: string, opts: SessionStartOptions = {}): Promise<SessionContext> {
|
||||||
this.diagnostics?.("session_manager.start.requested", {
|
|
||||||
session_id: sessionId,
|
|
||||||
existing_session_ids: this.list().map((ctx) => ctx.sessionId),
|
|
||||||
focused: opts.focused ?? null,
|
|
||||||
size: opts.size ?? null,
|
|
||||||
signal_aborted: opts.signal?.aborted ?? false,
|
|
||||||
});
|
|
||||||
if (this.sessions.has(sessionId)) {
|
if (this.sessions.has(sessionId)) {
|
||||||
throw new Error(`[bh] session ${sessionId} already exists`);
|
throw new Error(`[bh] session ${sessionId} already exists`);
|
||||||
}
|
}
|
||||||
@@ -189,10 +178,6 @@ export class SessionManager {
|
|||||||
try {
|
try {
|
||||||
const { signal: _signal, ...createOptions } = opts;
|
const { signal: _signal, ...createOptions } = opts;
|
||||||
windowId = await this.agentWindow.create(AGENT_WINDOW_HOME, createOptions);
|
windowId = await this.agentWindow.create(AGENT_WINDOW_HOME, createOptions);
|
||||||
this.diagnostics?.("session_manager.window.created", {
|
|
||||||
session_id: sessionId,
|
|
||||||
agent_window_id: windowId,
|
|
||||||
});
|
|
||||||
throwIfSessionStartAborted(opts.signal);
|
throwIfSessionStartAborted(opts.signal);
|
||||||
await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
|
await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
|
||||||
throwIfSessionStartAborted(opts.signal);
|
throwIfSessionStartAborted(opts.signal);
|
||||||
@@ -206,18 +191,8 @@ export class SessionManager {
|
|||||||
};
|
};
|
||||||
this.sessions.set(sessionId, ctx);
|
this.sessions.set(sessionId, ctx);
|
||||||
this.windowIndex.set(windowId, sessionId);
|
this.windowIndex.set(windowId, sessionId);
|
||||||
this.diagnostics?.("session_manager.start.committed", {
|
|
||||||
session_id: sessionId,
|
|
||||||
agent_window_id: windowId,
|
|
||||||
live_session_ids: this.list().map((entry) => entry.sessionId),
|
|
||||||
});
|
|
||||||
return ctx;
|
return ctx;
|
||||||
} catch (startupError) {
|
} catch (startupError) {
|
||||||
this.diagnostics?.("session_manager.start.failed", {
|
|
||||||
session_id: sessionId,
|
|
||||||
agent_window_id: windowId,
|
|
||||||
error: startupError,
|
|
||||||
});
|
|
||||||
if (windowId !== null) {
|
if (windowId !== null) {
|
||||||
try {
|
try {
|
||||||
await this.agentWindow.remove(windowId);
|
await this.agentWindow.remove(windowId);
|
||||||
@@ -241,28 +216,12 @@ export class SessionManager {
|
|||||||
options: { dropOnly?: boolean } = {},
|
options: { dropOnly?: boolean } = {},
|
||||||
): Promise<SessionContext | null> {
|
): Promise<SessionContext | null> {
|
||||||
const ctx = this.sessions.get(sessionId);
|
const ctx = this.sessions.get(sessionId);
|
||||||
this.diagnostics?.("session_manager.stop.requested", {
|
|
||||||
session_id: sessionId,
|
|
||||||
drop_only: options.dropOnly ?? false,
|
|
||||||
existed: ctx !== undefined,
|
|
||||||
live_session_ids: this.list().map((entry) => entry.sessionId),
|
|
||||||
});
|
|
||||||
if (!ctx) return null;
|
if (!ctx) return null;
|
||||||
if (!options.dropOnly) {
|
if (!options.dropOnly) {
|
||||||
await this.agentWindow.remove(ctx.agentWindowId);
|
await this.agentWindow.remove(ctx.agentWindowId);
|
||||||
this.diagnostics?.("session_manager.window.removed", {
|
|
||||||
session_id: sessionId,
|
|
||||||
agent_window_id: ctx.agentWindowId,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
this.sessions.delete(sessionId);
|
this.sessions.delete(sessionId);
|
||||||
this.windowIndex.delete(ctx.agentWindowId);
|
this.windowIndex.delete(ctx.agentWindowId);
|
||||||
this.diagnostics?.("session_manager.stop.committed", {
|
|
||||||
session_id: sessionId,
|
|
||||||
agent_window_id: ctx.agentWindowId,
|
|
||||||
drop_only: options.dropOnly ?? false,
|
|
||||||
live_session_ids: this.list().map((entry) => entry.sessionId),
|
|
||||||
});
|
|
||||||
return ctx;
|
return ctx;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -456,30 +456,6 @@ describe("handleClick", () => {
|
|||||||
expect(res).toMatchObject({ code: "not_found", data: { reason: "selector_not_found" } });
|
expect(res).toMatchObject({ code: "not_found", data: { reason: "selector_not_found" } });
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns invalid_selector when Chrome rejects non-CSS selector syntax", async () => {
|
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
|
||||||
await sm.start("aa11");
|
|
||||||
const fake = makeFakeCdp({
|
|
||||||
"DOM.getDocument": () => ({ root: { nodeId: 1 } }),
|
|
||||||
"DOM.querySelector": () => {
|
|
||||||
throw new Error('{"code":-32000,"message":"DOM Error while querying"}');
|
|
||||||
},
|
|
||||||
});
|
|
||||||
const res = await handleClick(
|
|
||||||
sm,
|
|
||||||
{ session_id: "aa11", selector: "text=下午 12~18点" },
|
|
||||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
|
||||||
);
|
|
||||||
expect(res).toMatchObject({
|
|
||||||
code: "invalid_params",
|
|
||||||
message: "selector is not valid CSS: text=下午 12~18点",
|
|
||||||
data: {
|
|
||||||
reason: "invalid_selector",
|
|
||||||
selector: "text=下午 12~18点",
|
|
||||||
},
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it("respects the AbortSignal", async () => {
|
it("respects the AbortSignal", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
const ctx = await sm.start("aa11");
|
const ctx = await sm.start("aa11");
|
||||||
|
|||||||
@@ -1904,36 +1904,41 @@ describe("buildVomScene", () => {
|
|||||||
backendDOMNodeId: 20,
|
backendDOMNodeId: 20,
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
const scene = buildVomScene(axNodes, {
|
const scene = buildVomScene(
|
||||||
viewport: { width: 1000, height: 800 },
|
axNodes,
|
||||||
iframeNodes: new Map(),
|
{
|
||||||
excludedBackendNodeIds: new Set(),
|
viewport: { width: 1000, height: 800 },
|
||||||
surfaceProbes: [
|
iframeNodes: new Map(),
|
||||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
excludedBackendNodeIds: new Set(),
|
||||||
],
|
nodes: [
|
||||||
nodes: [
|
{
|
||||||
{
|
backendNodeId: 10,
|
||||||
backendNodeId: 10,
|
parentBackendNodeId: null,
|
||||||
parentBackendNodeId: null,
|
tag: "body",
|
||||||
tag: "body",
|
attrs: {},
|
||||||
attrs: {},
|
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
paintOrder: 0,
|
||||||
paintOrder: 0,
|
position: "static",
|
||||||
position: "static",
|
pointerEvents: "auto",
|
||||||
pointerEvents: "auto",
|
},
|
||||||
},
|
{
|
||||||
{
|
backendNodeId: 20,
|
||||||
backendNodeId: 20,
|
parentBackendNodeId: 10,
|
||||||
parentBackendNodeId: 10,
|
tag: "button",
|
||||||
tag: "button",
|
attrs: {},
|
||||||
attrs: {},
|
rect: { x: 20, y: 20, w: 120, h: 40 },
|
||||||
rect: { x: 20, y: 20, w: 120, h: 40 },
|
paintOrder: 1,
|
||||||
paintOrder: 1,
|
position: "static",
|
||||||
position: "static",
|
pointerEvents: "auto",
|
||||||
pointerEvents: "auto",
|
},
|
||||||
},
|
],
|
||||||
],
|
},
|
||||||
});
|
{
|
||||||
|
surfaceProbes: [
|
||||||
|
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
expect(scene.surfaces).toEqual([
|
expect(scene.surfaces).toEqual([
|
||||||
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||||
@@ -1957,46 +1962,51 @@ describe("buildVomScene", () => {
|
|||||||
backendDOMNodeId: 21,
|
backendDOMNodeId: 21,
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
const scene = buildVomScene(axNodes, {
|
const scene = buildVomScene(
|
||||||
viewport: { width: 1000, height: 800 },
|
axNodes,
|
||||||
iframeNodes: new Map(),
|
{
|
||||||
excludedBackendNodeIds: new Set(),
|
viewport: { width: 1000, height: 800 },
|
||||||
surfaceProbes: [
|
iframeNodes: new Map(),
|
||||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
excludedBackendNodeIds: new Set(),
|
||||||
],
|
nodes: [
|
||||||
nodes: [
|
{
|
||||||
{
|
backendNodeId: 10,
|
||||||
backendNodeId: 10,
|
parentBackendNodeId: null,
|
||||||
parentBackendNodeId: null,
|
tag: "body",
|
||||||
tag: "body",
|
attrs: {},
|
||||||
attrs: {},
|
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
paintOrder: 0,
|
||||||
paintOrder: 0,
|
position: "static",
|
||||||
position: "static",
|
pointerEvents: "auto",
|
||||||
pointerEvents: "auto",
|
},
|
||||||
},
|
{
|
||||||
{
|
backendNodeId: 20,
|
||||||
backendNodeId: 20,
|
parentBackendNodeId: 10,
|
||||||
parentBackendNodeId: 10,
|
tag: "div",
|
||||||
tag: "div",
|
attrs: { class: "tg-avatar" },
|
||||||
attrs: { class: "tg-avatar" },
|
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
paintOrder: 1,
|
||||||
paintOrder: 1,
|
position: "static",
|
||||||
position: "static",
|
pointerEvents: "auto",
|
||||||
pointerEvents: "auto",
|
},
|
||||||
},
|
{
|
||||||
{
|
backendNodeId: 21,
|
||||||
backendNodeId: 21,
|
parentBackendNodeId: 20,
|
||||||
parentBackendNodeId: 20,
|
tag: "div",
|
||||||
tag: "div",
|
attrs: { class: "tg-avatar__inner" },
|
||||||
attrs: { class: "tg-avatar__inner" },
|
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
paintOrder: 2,
|
||||||
paintOrder: 2,
|
position: "static",
|
||||||
position: "static",
|
pointerEvents: "auto",
|
||||||
pointerEvents: "auto",
|
},
|
||||||
},
|
],
|
||||||
],
|
},
|
||||||
});
|
{
|
||||||
|
surfaceProbes: [
|
||||||
|
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
expect(renderVom(scene).text).toContain('@e1 button "image" [hover first: My profile]');
|
expect(renderVom(scene).text).toContain('@e1 button "image" [hover first: My profile]');
|
||||||
});
|
});
|
||||||
@@ -2017,48 +2027,53 @@ describe("buildVomScene", () => {
|
|||||||
backendDOMNodeId: 21,
|
backendDOMNodeId: 21,
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
const scene = buildVomScene(axNodes, {
|
const scene = buildVomScene(
|
||||||
viewport: { width: 1000, height: 800 },
|
axNodes,
|
||||||
iframeNodes: new Map(),
|
{
|
||||||
excludedBackendNodeIds: new Set(),
|
viewport: { width: 1000, height: 800 },
|
||||||
surfaceProbes: [
|
iframeNodes: new Map(),
|
||||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
excludedBackendNodeIds: new Set(),
|
||||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] },
|
nodes: [
|
||||||
{ triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] },
|
{
|
||||||
],
|
backendNodeId: 10,
|
||||||
nodes: [
|
parentBackendNodeId: null,
|
||||||
{
|
tag: "body",
|
||||||
backendNodeId: 10,
|
attrs: {},
|
||||||
parentBackendNodeId: null,
|
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||||
tag: "body",
|
paintOrder: 0,
|
||||||
attrs: {},
|
position: "static",
|
||||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
pointerEvents: "auto",
|
||||||
paintOrder: 0,
|
},
|
||||||
position: "static",
|
{
|
||||||
pointerEvents: "auto",
|
backendNodeId: 20,
|
||||||
},
|
parentBackendNodeId: 10,
|
||||||
{
|
tag: "div",
|
||||||
backendNodeId: 20,
|
attrs: { class: "tg-avatar" },
|
||||||
parentBackendNodeId: 10,
|
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||||
tag: "div",
|
paintOrder: 1,
|
||||||
attrs: { class: "tg-avatar" },
|
position: "static",
|
||||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
pointerEvents: "auto",
|
||||||
paintOrder: 1,
|
},
|
||||||
position: "static",
|
{
|
||||||
pointerEvents: "auto",
|
backendNodeId: 21,
|
||||||
},
|
parentBackendNodeId: 20,
|
||||||
{
|
tag: "div",
|
||||||
backendNodeId: 21,
|
attrs: { class: "tg-avatar__inner" },
|
||||||
parentBackendNodeId: 20,
|
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||||
tag: "div",
|
paintOrder: 2,
|
||||||
attrs: { class: "tg-avatar__inner" },
|
position: "static",
|
||||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
pointerEvents: "auto",
|
||||||
paintOrder: 2,
|
},
|
||||||
position: "static",
|
],
|
||||||
pointerEvents: "auto",
|
},
|
||||||
},
|
{
|
||||||
],
|
surfaceProbes: [
|
||||||
});
|
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||||
|
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] },
|
||||||
|
{ triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
expect(scene.surfaces).toEqual([
|
expect(scene.surfaces).toEqual([
|
||||||
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile"] },
|
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile"] },
|
||||||
@@ -2081,54 +2096,59 @@ describe("buildVomScene", () => {
|
|||||||
backendDOMNodeId: 21,
|
backendDOMNodeId: 21,
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
const scene = buildVomScene(axNodes, {
|
const scene = buildVomScene(
|
||||||
viewport: { width: 1000, height: 800 },
|
axNodes,
|
||||||
iframeNodes: new Map(),
|
{
|
||||||
excludedBackendNodeIds: new Set(),
|
viewport: { width: 1000, height: 800 },
|
||||||
surfaceProbes: [
|
iframeNodes: new Map(),
|
||||||
{
|
excludedBackendNodeIds: new Set(),
|
||||||
triggerBackendNodeId: 20,
|
nodes: [
|
||||||
triggerPoint: { x: 915, y: 25 },
|
{
|
||||||
triggerAction: "hover",
|
backendNodeId: 10,
|
||||||
subItems: ["My profile", "Sign out"],
|
parentBackendNodeId: null,
|
||||||
},
|
tag: "body",
|
||||||
],
|
attrs: {},
|
||||||
nodes: [
|
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||||
{
|
paintOrder: 0,
|
||||||
backendNodeId: 10,
|
position: "static",
|
||||||
parentBackendNodeId: null,
|
pointerEvents: "auto",
|
||||||
tag: "body",
|
cursor: "auto",
|
||||||
attrs: {},
|
},
|
||||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
{
|
||||||
paintOrder: 0,
|
backendNodeId: 20,
|
||||||
position: "static",
|
parentBackendNodeId: 10,
|
||||||
pointerEvents: "auto",
|
tag: "div",
|
||||||
cursor: "auto",
|
attrs: { class: "tg-avatar" },
|
||||||
},
|
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||||
{
|
paintOrder: 1,
|
||||||
backendNodeId: 20,
|
position: "static",
|
||||||
parentBackendNodeId: 10,
|
pointerEvents: "auto",
|
||||||
tag: "div",
|
cursor: "pointer",
|
||||||
attrs: { class: "tg-avatar" },
|
},
|
||||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
{
|
||||||
paintOrder: 1,
|
backendNodeId: 21,
|
||||||
position: "static",
|
parentBackendNodeId: 20,
|
||||||
pointerEvents: "auto",
|
tag: "div",
|
||||||
cursor: "pointer",
|
attrs: { class: "tg-avatar__inner" },
|
||||||
},
|
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||||
{
|
paintOrder: 2,
|
||||||
backendNodeId: 21,
|
position: "static",
|
||||||
parentBackendNodeId: 20,
|
pointerEvents: "auto",
|
||||||
tag: "div",
|
cursor: "pointer",
|
||||||
attrs: { class: "tg-avatar__inner" },
|
},
|
||||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
],
|
||||||
paintOrder: 2,
|
},
|
||||||
position: "static",
|
{
|
||||||
pointerEvents: "auto",
|
surfaceProbes: [
|
||||||
cursor: "pointer",
|
{
|
||||||
},
|
triggerBackendNodeId: 20,
|
||||||
],
|
triggerPoint: { x: 915, y: 25 },
|
||||||
});
|
triggerAction: "hover",
|
||||||
|
subItems: ["My profile", "Sign out"],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
expect(scene.surfaces).toEqual([
|
expect(scene.surfaces).toEqual([
|
||||||
{ triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
|
{ triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
|
||||||
@@ -2154,41 +2174,46 @@ describe("buildVomScene", () => {
|
|||||||
backendDOMNodeId: 42,
|
backendDOMNodeId: 42,
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
const scene = buildVomScene(axNodes, {
|
const scene = buildVomScene(
|
||||||
viewport: { width: 1000, height: 800 },
|
axNodes,
|
||||||
iframeNodes: new Map(),
|
{
|
||||||
excludedBackendNodeIds: new Set(),
|
viewport: { width: 1000, height: 800 },
|
||||||
surfaceProbes: [
|
iframeNodes: new Map(),
|
||||||
{
|
excludedBackendNodeIds: new Set(),
|
||||||
triggerBackendNodeId: 999,
|
nodes: [
|
||||||
triggerPoint: { x: 900, y: 20 },
|
{
|
||||||
triggerAction: "hover",
|
backendNodeId: 10,
|
||||||
subItems: ["My profile"],
|
parentBackendNodeId: null,
|
||||||
},
|
tag: "body",
|
||||||
],
|
attrs: {},
|
||||||
nodes: [
|
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||||
{
|
paintOrder: 0,
|
||||||
backendNodeId: 10,
|
position: "static",
|
||||||
parentBackendNodeId: null,
|
pointerEvents: "auto",
|
||||||
tag: "body",
|
},
|
||||||
attrs: {},
|
{
|
||||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
backendNodeId: 42,
|
||||||
paintOrder: 0,
|
parentBackendNodeId: 10,
|
||||||
position: "static",
|
tag: "a",
|
||||||
pointerEvents: "auto",
|
attrs: {},
|
||||||
},
|
rect: { x: 860, y: 120, w: 40, h: 30 },
|
||||||
{
|
paintOrder: 1,
|
||||||
backendNodeId: 42,
|
position: "static",
|
||||||
parentBackendNodeId: 10,
|
pointerEvents: "auto",
|
||||||
tag: "a",
|
},
|
||||||
attrs: {},
|
],
|
||||||
rect: { x: 860, y: 120, w: 40, h: 30 },
|
},
|
||||||
paintOrder: 1,
|
{
|
||||||
position: "static",
|
surfaceProbes: [
|
||||||
pointerEvents: "auto",
|
{
|
||||||
},
|
triggerBackendNodeId: 999,
|
||||||
],
|
triggerPoint: { x: 900, y: 20 },
|
||||||
});
|
triggerAction: "hover",
|
||||||
|
subItems: ["My profile"],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
expect(scene.surfaces).toBeUndefined();
|
expect(scene.surfaces).toBeUndefined();
|
||||||
expect(renderVom(scene).text).not.toContain("[hover first:");
|
expect(renderVom(scene).text).not.toContain("[hover first:");
|
||||||
@@ -2606,7 +2631,106 @@ describe("handleSnapshot", () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("enables conditional surface probing for semantic observe", async () => {
|
it("hovers the page only after the accessibility tree has been captured", async () => {
|
||||||
|
// Hovering can open menus and reflow the page. Probing between the DOM and
|
||||||
|
// AX captures would leave the two halves of one observation describing the
|
||||||
|
// page on either side of that change.
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
await sm.start("aa11");
|
||||||
|
const root: CdpAxNode = {
|
||||||
|
nodeId: "1",
|
||||||
|
role: { type: "role", value: "RootWebArea" },
|
||||||
|
name: { type: "computedString", value: "Example" },
|
||||||
|
backendDOMNodeId: 100,
|
||||||
|
childIds: ["2"],
|
||||||
|
};
|
||||||
|
const button: CdpAxNode = {
|
||||||
|
nodeId: "2",
|
||||||
|
parentId: "1",
|
||||||
|
role: { type: "role", value: "button" },
|
||||||
|
name: { type: "computedString", value: "Products" },
|
||||||
|
backendDOMNodeId: 200,
|
||||||
|
};
|
||||||
|
const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"];
|
||||||
|
const i = (s: string) => strings.indexOf(s);
|
||||||
|
const methodOrder: string[] = [];
|
||||||
|
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||||
|
methodOrder.push(method);
|
||||||
|
if (method === "Accessibility.enable") return {};
|
||||||
|
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
||||||
|
}
|
||||||
|
if (method === "DOMSnapshot.enable") return {};
|
||||||
|
if (method === "DOMSnapshot.captureSnapshot") {
|
||||||
|
return {
|
||||||
|
strings,
|
||||||
|
documents: [
|
||||||
|
{
|
||||||
|
nodes: {
|
||||||
|
parentIndex: [-1, 0],
|
||||||
|
nodeName: [i("body"), i("button")],
|
||||||
|
backendNodeId: [100, 200],
|
||||||
|
attributes: [[], []],
|
||||||
|
},
|
||||||
|
layout: {
|
||||||
|
nodeIndex: [0, 1],
|
||||||
|
styles: [
|
||||||
|
[i("static"), i("auto"), i("auto")],
|
||||||
|
[i("static"), i("auto"), i("auto")],
|
||||||
|
],
|
||||||
|
bounds: [
|
||||||
|
[0, 0, 1000, 800],
|
||||||
|
[20, 20, 120, 40],
|
||||||
|
],
|
||||||
|
paintOrders: [0, 1],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (method === "Input.dispatchMouseEvent") return {};
|
||||||
|
if (method === "Runtime.evaluate") {
|
||||||
|
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
|
||||||
|
// Report the button as a `:hover` rule target so a real hover fires.
|
||||||
|
if (expression.includes("document.styleSheets")) {
|
||||||
|
return { result: { value: [{ x: 80, y: 40 }] } };
|
||||||
|
}
|
||||||
|
return { result: { value: [] } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected CDP method ${method}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
const res = await handleObserve(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", probe_hover: true },
|
||||||
|
{
|
||||||
|
cdp: {
|
||||||
|
send: send as unknown as <T = unknown>(
|
||||||
|
tabId: number,
|
||||||
|
method: string,
|
||||||
|
params?: object,
|
||||||
|
) => Promise<T>,
|
||||||
|
trackSessionTab: vi.fn(),
|
||||||
|
},
|
||||||
|
tabsApi: {
|
||||||
|
get: vi.fn(
|
||||||
|
async (tabId: number) =>
|
||||||
|
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||||
|
),
|
||||||
|
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||||
|
const firstHover = methodOrder.indexOf("Input.dispatchMouseEvent");
|
||||||
|
expect(firstHover).toBeGreaterThan(-1);
|
||||||
|
expect(methodOrder.lastIndexOf("DOMSnapshot.captureSnapshot")).toBeLessThan(firstHover);
|
||||||
|
expect(methodOrder.lastIndexOf("Accessibility.getFullAXTree")).toBeLessThan(firstHover);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("runs conditional surface probing only when observe opts in", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
await sm.start("aa11");
|
await sm.start("aa11");
|
||||||
const root: CdpAxNode = {
|
const root: CdpAxNode = {
|
||||||
@@ -2662,30 +2786,46 @@ describe("handleSnapshot", () => {
|
|||||||
if (method === "Runtime.evaluate") return { result: { value: [] } };
|
if (method === "Runtime.evaluate") return { result: { value: [] } };
|
||||||
throw new Error(`unexpected CDP method ${method}`);
|
throw new Error(`unexpected CDP method ${method}`);
|
||||||
});
|
});
|
||||||
const res = await handleObserve(
|
const deps = {
|
||||||
|
cdp: {
|
||||||
|
send: send as unknown as <T = unknown>(
|
||||||
|
tabId: number,
|
||||||
|
method: string,
|
||||||
|
params?: object,
|
||||||
|
) => Promise<T>,
|
||||||
|
trackSessionTab: vi.fn(),
|
||||||
|
},
|
||||||
|
tabsApi: {
|
||||||
|
get: vi.fn(
|
||||||
|
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||||
|
),
|
||||||
|
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const withoutProbe = await handleObserve(
|
||||||
sm,
|
sm,
|
||||||
{ session_id: "aa11", debug_surfaces: true },
|
{ session_id: "aa11", debug_surfaces: true },
|
||||||
{
|
deps,
|
||||||
cdp: {
|
);
|
||||||
send: send as unknown as <T = unknown>(
|
if ("code" in withoutProbe) {
|
||||||
tabId: number,
|
throw new Error(`unexpected error: ${JSON.stringify(withoutProbe)}`);
|
||||||
method: string,
|
}
|
||||||
params?: object,
|
expect(withoutProbe.hover_probe).toBeUndefined();
|
||||||
) => Promise<T>,
|
expect(send).not.toHaveBeenCalledWith(
|
||||||
trackSessionTab: vi.fn(),
|
4,
|
||||||
},
|
"Runtime.evaluate",
|
||||||
tabsApi: {
|
expect.objectContaining({ returnByValue: true }),
|
||||||
get: vi.fn(
|
|
||||||
async (tabId: number) =>
|
|
||||||
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
|
||||||
),
|
|
||||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
);
|
);
|
||||||
|
|
||||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
const withProbe = await handleObserve(
|
||||||
expect(res.debug).toEqual({ surface_probes: [] });
|
sm,
|
||||||
|
{ session_id: "aa11", debug_surfaces: true, probe_hover: true },
|
||||||
|
deps,
|
||||||
|
);
|
||||||
|
if ("code" in withProbe) throw new Error(`unexpected error: ${JSON.stringify(withProbe)}`);
|
||||||
|
expect(withProbe.debug).toEqual({ surface_probes: [] });
|
||||||
|
expect(withProbe.hover_probe).toEqual({ performed: true, revealed_content: false });
|
||||||
expect(send).toHaveBeenCalledWith(
|
expect(send).toHaveBeenCalledWith(
|
||||||
4,
|
4,
|
||||||
"Runtime.evaluate",
|
"Runtime.evaluate",
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
import type { DiagnosticSink } from "@/lib/diagnostics";
|
|
||||||
import { OVERLAY_AUTOMATION_BYPASS } from "@/lib/overlay-bridge";
|
import { OVERLAY_AUTOMATION_BYPASS } from "@/lib/overlay-bridge";
|
||||||
import type { SessionManager } from "@/session-manager/manager";
|
import type { SessionManager } from "@/session-manager/manager";
|
||||||
import type { Transport } from "@/transport/transport";
|
import type { Transport } from "@/transport/transport";
|
||||||
@@ -119,7 +118,6 @@ export interface DispatcherDeps {
|
|||||||
approveBorrow?: BorrowConfirmationApprover;
|
approveBorrow?: BorrowConfirmationApprover;
|
||||||
/** i18n notification copy for `tool.request_help` (resolved per-call). */
|
/** i18n notification copy for `tool.request_help` (resolved per-call). */
|
||||||
helpNotificationCopy?: () => { title: string; body: string };
|
helpNotificationCopy?: () => { title: string; body: string };
|
||||||
diagnostics?: DiagnosticSink;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -147,7 +145,6 @@ export class ToolDispatcher {
|
|||||||
private readonly onBrowserControlResumed?: (sessionId: string) => void;
|
private readonly onBrowserControlResumed?: (sessionId: string) => void;
|
||||||
private readonly approveBorrow?: BorrowConfirmationApprover;
|
private readonly approveBorrow?: BorrowConfirmationApprover;
|
||||||
private readonly helpNotificationCopy?: () => { title: string; body: string };
|
private readonly helpNotificationCopy?: () => { title: string; body: string };
|
||||||
private readonly diagnostics?: DiagnosticSink;
|
|
||||||
private subscription: { dispose(): void } | null = null;
|
private subscription: { dispose(): void } | null = null;
|
||||||
private readonly hoverBypassTabs = new Map<number, string>();
|
private readonly hoverBypassTabs = new Map<number, string>();
|
||||||
private readonly hoverLatches = new Map<number, HoverLatch>();
|
private readonly hoverLatches = new Map<number, HoverLatch>();
|
||||||
@@ -168,7 +165,6 @@ export class ToolDispatcher {
|
|||||||
this.onBrowserControlResumed = deps.onBrowserControlResumed;
|
this.onBrowserControlResumed = deps.onBrowserControlResumed;
|
||||||
this.approveBorrow = deps.approveBorrow;
|
this.approveBorrow = deps.approveBorrow;
|
||||||
this.helpNotificationCopy = deps.helpNotificationCopy;
|
this.helpNotificationCopy = deps.helpNotificationCopy;
|
||||||
this.diagnostics = deps.diagnostics;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
start(): void {
|
start(): void {
|
||||||
@@ -196,16 +192,6 @@ export class ToolDispatcher {
|
|||||||
private async dispatch(msg: ProtocolFrame): Promise<void> {
|
private async dispatch(msg: ProtocolFrame): Promise<void> {
|
||||||
if (!isRequestFrame(msg)) return;
|
if (!isRequestFrame(msg)) return;
|
||||||
const req = msg as RequestFrame;
|
const req = msg as RequestFrame;
|
||||||
const requestSessionId = (req.params as { session_id?: unknown } | undefined)?.session_id;
|
|
||||||
const startedAt = performance.now();
|
|
||||||
this.diagnostics?.("dispatcher.rpc.received", {
|
|
||||||
rpc_id: req.id,
|
|
||||||
method: req.method,
|
|
||||||
session_id: typeof requestSessionId === "string" ? requestSessionId : null,
|
|
||||||
transport_state: this.transport.state,
|
|
||||||
live_session_ids: this.sessions.list().map((ctx) => ctx.sessionId),
|
|
||||||
inflight_rpc_ids: Array.from(this.inflightAbortControllers.keys()),
|
|
||||||
});
|
|
||||||
|
|
||||||
// Cancel frames take a fast path: trip the matching controller
|
// Cancel frames take a fast path: trip the matching controller
|
||||||
// (if any), reply with `{cancelled}` so the daemon can answer
|
// (if any), reply with `{cancelled}` so the daemon can answer
|
||||||
@@ -214,12 +200,6 @@ export class ToolDispatcher {
|
|||||||
const params = (req.params as { rpc_id?: string } | undefined) ?? {};
|
const params = (req.params as { rpc_id?: string } | undefined) ?? {};
|
||||||
const target = typeof params.rpc_id === "string" ? params.rpc_id : "";
|
const target = typeof params.rpc_id === "string" ? params.rpc_id : "";
|
||||||
const ac = target ? this.inflightAbortControllers.get(target) : undefined;
|
const ac = target ? this.inflightAbortControllers.get(target) : undefined;
|
||||||
this.diagnostics?.("dispatcher.cancel.received", {
|
|
||||||
rpc_id: req.id,
|
|
||||||
target_rpc_id: target,
|
|
||||||
target_found: ac !== undefined,
|
|
||||||
inflight_rpc_ids: Array.from(this.inflightAbortControllers.keys()),
|
|
||||||
});
|
|
||||||
if (ac) {
|
if (ac) {
|
||||||
try {
|
try {
|
||||||
ac.abort();
|
ac.abort();
|
||||||
@@ -233,17 +213,7 @@ export class ToolDispatcher {
|
|||||||
};
|
};
|
||||||
try {
|
try {
|
||||||
this.transport.send(reply);
|
this.transport.send(reply);
|
||||||
this.diagnostics?.("dispatcher.cancel.ack_sent", {
|
|
||||||
rpc_id: req.id,
|
|
||||||
target_rpc_id: target,
|
|
||||||
cancelled: ac !== undefined,
|
|
||||||
});
|
|
||||||
} catch (sendErr) {
|
} catch (sendErr) {
|
||||||
this.diagnostics?.("dispatcher.cancel.ack_failed", {
|
|
||||||
rpc_id: req.id,
|
|
||||||
target_rpc_id: target,
|
|
||||||
error: sendErr,
|
|
||||||
});
|
|
||||||
console.warn("[bsk dispatcher] failed to ack cancel", sendErr);
|
console.warn("[bsk dispatcher] failed to ack cancel", sendErr);
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
@@ -268,13 +238,6 @@ export class ToolDispatcher {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
this.diagnostics?.("dispatcher.rpc.invoke_failed", {
|
|
||||||
rpc_id: req.id,
|
|
||||||
method: req.method,
|
|
||||||
session_id: typeof requestSessionId === "string" ? requestSessionId : null,
|
|
||||||
abort_signal: ac.signal.aborted,
|
|
||||||
error: err,
|
|
||||||
});
|
|
||||||
if (isAbortLikeError(err)) {
|
if (isAbortLikeError(err)) {
|
||||||
body = {
|
body = {
|
||||||
id: req.id,
|
id: req.id,
|
||||||
@@ -295,27 +258,8 @@ export class ToolDispatcher {
|
|||||||
let sent = true;
|
let sent = true;
|
||||||
try {
|
try {
|
||||||
this.transport.send(body);
|
this.transport.send(body);
|
||||||
this.diagnostics?.("dispatcher.rpc.response_sent", {
|
|
||||||
rpc_id: req.id,
|
|
||||||
method: req.method,
|
|
||||||
session_id: typeof requestSessionId === "string" ? requestSessionId : null,
|
|
||||||
response_kind: "error" in body ? "error" : "ok",
|
|
||||||
error_code: "error" in body ? body.error.code : null,
|
|
||||||
elapsed_ms: Math.round(performance.now() - startedAt),
|
|
||||||
transport_state: this.transport.state,
|
|
||||||
live_session_ids: this.sessions.list().map((ctx) => ctx.sessionId),
|
|
||||||
});
|
|
||||||
} catch (sendErr) {
|
} catch (sendErr) {
|
||||||
sent = false;
|
sent = false;
|
||||||
this.diagnostics?.("dispatcher.rpc.response_send_failed", {
|
|
||||||
rpc_id: req.id,
|
|
||||||
method: req.method,
|
|
||||||
session_id: typeof requestSessionId === "string" ? requestSessionId : null,
|
|
||||||
started_session_id: startedSession,
|
|
||||||
elapsed_ms: Math.round(performance.now() - startedAt),
|
|
||||||
transport_state: this.transport.state,
|
|
||||||
error: sendErr,
|
|
||||||
});
|
|
||||||
// Transport is dead by the time we want to reply. Drop the link
|
// Transport is dead by the time we want to reply. Drop the link
|
||||||
// proactively so the alarm-driven keepalive reconnects sooner
|
// proactively so the alarm-driven keepalive reconnects sooner
|
||||||
// and the daemon's pending RPC times out cleanly instead of
|
// and the daemon's pending RPC times out cleanly instead of
|
||||||
@@ -334,23 +278,12 @@ export class ToolDispatcher {
|
|||||||
try {
|
try {
|
||||||
const ctx = await this.sessions.stop(startedSession);
|
const ctx = await this.sessions.stop(startedSession);
|
||||||
if (ctx) {
|
if (ctx) {
|
||||||
this.diagnostics?.("dispatcher.session_start.rolled_back", {
|
|
||||||
rpc_id: req.id,
|
|
||||||
session_id: startedSession,
|
|
||||||
agent_window_id: ctx.agentWindowId,
|
|
||||||
reason: "response_send_failed",
|
|
||||||
});
|
|
||||||
console.warn(
|
console.warn(
|
||||||
"[bsk dispatcher] rolled back orphan session after send failure",
|
"[bsk dispatcher] rolled back orphan session after send failure",
|
||||||
startedSession,
|
startedSession,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
} catch (rollbackErr) {
|
} catch (rollbackErr) {
|
||||||
this.diagnostics?.("dispatcher.session_start.rollback_failed", {
|
|
||||||
rpc_id: req.id,
|
|
||||||
session_id: startedSession,
|
|
||||||
error: rollbackErr,
|
|
||||||
});
|
|
||||||
console.warn("[bsk dispatcher] session rollback after send failure failed", rollbackErr);
|
console.warn("[bsk dispatcher] session rollback after send failure failed", rollbackErr);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -450,7 +383,11 @@ export class ToolDispatcher {
|
|||||||
? {
|
? {
|
||||||
cdp: this.cdp,
|
cdp: this.cdp,
|
||||||
tabsApi: chromeTabsCaptureApi,
|
tabsApi: chromeTabsCaptureApi,
|
||||||
conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
|
// Active hover probing is opt-in. A held hover latch still
|
||||||
|
// suppresses it, because probing would move the cursor off
|
||||||
|
// the element the caller is deliberately holding.
|
||||||
|
conditionalSurfaceProbe:
|
||||||
|
params.probe_hover === true && !this.hasHoverLatchForScope(hoverScope),
|
||||||
hoverProbeBypassOverlay: bypassOverlay,
|
hoverProbeBypassOverlay: bypassOverlay,
|
||||||
}
|
}
|
||||||
: undefined,
|
: undefined,
|
||||||
|
|||||||
@@ -106,14 +106,6 @@ function isAbortLikeError(err: unknown): boolean {
|
|||||||
return err instanceof DOMException && err.name === "AbortError";
|
return err instanceof DOMException && err.name === "AbortError";
|
||||||
}
|
}
|
||||||
|
|
||||||
function isInvalidCssSelectorError(err: unknown): boolean {
|
|
||||||
const message = err instanceof Error ? err.message : String(err);
|
|
||||||
return (
|
|
||||||
message.includes("DOM Error while querying") ||
|
|
||||||
/invalid selector|not a valid selector|querySelector.*failed/i.test(message)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function wait(ms: number, signal?: AbortSignal): Promise<void> {
|
async function wait(ms: number, signal?: AbortSignal): Promise<void> {
|
||||||
if (signal?.aborted) throw new DOMException("aborted", "AbortError");
|
if (signal?.aborted) throw new DOMException("aborted", "AbortError");
|
||||||
await new Promise<void>((resolve, reject) => {
|
await new Promise<void>((resolve, reject) => {
|
||||||
@@ -216,14 +208,6 @@ async function resolveBackendNode(
|
|||||||
}
|
}
|
||||||
return { backendNodeId, cdpTarget: { tabId: target.tabId }, usedSelector: params.selector };
|
return { backendNodeId, cdpTarget: { tabId: target.tabId }, usedSelector: params.selector };
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (isInvalidCssSelectorError(err)) {
|
|
||||||
return rpcError(
|
|
||||||
"invalid_params",
|
|
||||||
"invalid_selector",
|
|
||||||
`selector is not valid CSS: ${params.selector}`,
|
|
||||||
{ selector: params.selector },
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return {
|
return {
|
||||||
code: "cdp_failed",
|
code: "cdp_failed",
|
||||||
message: err instanceof Error ? err.message : String(err),
|
message: err instanceof Error ? err.message : String(err),
|
||||||
|
|||||||
@@ -49,11 +49,14 @@ import {
|
|||||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||||
import {
|
import {
|
||||||
type CapturedNode,
|
type CapturedNode,
|
||||||
|
type CapturedSurfaceProbe,
|
||||||
type CapturedViewModel,
|
type CapturedViewModel,
|
||||||
captureViewModel,
|
captureViewModel,
|
||||||
collectOverlayExcludedBackendIds,
|
collectOverlayExcludedBackendIds,
|
||||||
|
probeHoverSurfaces,
|
||||||
} from "./vom/capture";
|
} from "./vom/capture";
|
||||||
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
|
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
|
||||||
|
import { withOverlayBypass } from "./vom/hover-perception";
|
||||||
import { probeTooltipNames } from "./vom/name-enrichment";
|
import { probeTooltipNames } from "./vom/name-enrichment";
|
||||||
import {
|
import {
|
||||||
type CaptureVomObservationResult,
|
type CaptureVomObservationResult,
|
||||||
@@ -525,8 +528,11 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A
|
|||||||
return blocks;
|
return blocks;
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] {
|
function buildConditionalSurfaces(
|
||||||
const probes = captured.surfaceProbes ?? [];
|
nodes: VomNode[],
|
||||||
|
captured: CapturedViewModel,
|
||||||
|
probes: CapturedSurfaceProbe[],
|
||||||
|
): CondSurface[] {
|
||||||
if (probes.length === 0) return [];
|
if (probes.length === 0) return [];
|
||||||
|
|
||||||
const surfaces: CondSurface[] = [];
|
const surfaces: CondSurface[] = [];
|
||||||
@@ -649,6 +655,7 @@ function findSurfaceNodeByPoint(
|
|||||||
export interface BuildVomSceneOptions {
|
export interface BuildVomSceneOptions {
|
||||||
pageUrl?: string;
|
pageUrl?: string;
|
||||||
supplementalNames?: ReadonlyMap<string, string>;
|
supplementalNames?: ReadonlyMap<string, string>;
|
||||||
|
surfaceProbes?: CapturedSurfaceProbe[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>;
|
export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>;
|
||||||
@@ -817,18 +824,19 @@ export function buildFrameVomScene(
|
|||||||
excludedBackendNodeIds: captured.excludedBackendNodeIds,
|
excludedBackendNodeIds: captured.excludedBackendNodeIds,
|
||||||
supplementalNames: options.supplementalNames,
|
supplementalNames: options.supplementalNames,
|
||||||
});
|
});
|
||||||
return attachCapturedSceneAnnotations(scene, documents, captured);
|
return attachCapturedSceneAnnotations(scene, documents, captured, options.surfaceProbes ?? []);
|
||||||
}
|
}
|
||||||
|
|
||||||
function attachCapturedSceneAnnotations(
|
function attachCapturedSceneAnnotations(
|
||||||
scene: VomScene,
|
scene: VomScene,
|
||||||
documents: VomFrameDocument[],
|
documents: VomFrameDocument[],
|
||||||
captured: CapturedViewModel,
|
captured: CapturedViewModel,
|
||||||
|
surfaceProbes: CapturedSurfaceProbe[],
|
||||||
): VomScene {
|
): VomScene {
|
||||||
const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId);
|
const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId);
|
||||||
const signals = capturedOnlySignals(rootDocument?.domNodes ?? captured.nodes);
|
const signals = capturedOnlySignals(rootDocument?.domNodes ?? captured.nodes);
|
||||||
const activeScopeBlocks = buildActiveScopeBlocks(scene.nodes, signals);
|
const activeScopeBlocks = buildActiveScopeBlocks(scene.nodes, signals);
|
||||||
const surfaces = buildConditionalSurfaces(scene.nodes, captured);
|
const surfaces = buildConditionalSurfaces(scene.nodes, captured, surfaceProbes);
|
||||||
return {
|
return {
|
||||||
...scene,
|
...scene,
|
||||||
...(surfaces.length > 0 ? { surfaces } : {}),
|
...(surfaces.length > 0 ? { surfaces } : {}),
|
||||||
@@ -1010,17 +1018,69 @@ async function captureForVom(
|
|||||||
options: CaptureVomObservationOptions,
|
options: CaptureVomObservationOptions,
|
||||||
): Promise<CapturedViewModel> {
|
): Promise<CapturedViewModel> {
|
||||||
try {
|
try {
|
||||||
return await captureViewModel(cdp, tabId, {
|
return await captureViewModel(cdp, tabId, { signal: options.signal });
|
||||||
conditionalSurfaceProbe: options.conditionalSurfaceProbe ?? false,
|
|
||||||
hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
|
|
||||||
signal: options.signal,
|
|
||||||
});
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (isAbortError(error)) throw error;
|
if (isAbortError(error)) throw error;
|
||||||
return fallbackCapturedViewModel(cdp, tabId, options.signal);
|
return fallbackCapturedViewModel(cdp, tabId, options.signal);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface HoverProbeOutcome {
|
||||||
|
/** Whether any active hover was dispatched during this observation. */
|
||||||
|
performed: boolean;
|
||||||
|
/**
|
||||||
|
* Whether hovering surfaced content absent from the static snapshot. Cached
|
||||||
|
* tooltip names count, so this may over-report on repeat observations — it
|
||||||
|
* errs towards telling the caller the snapshot is less fresh than it looks.
|
||||||
|
*/
|
||||||
|
revealedContent: boolean;
|
||||||
|
surfaceProbes: CapturedSurfaceProbe[];
|
||||||
|
tooltipNames: Map<string, string>;
|
||||||
|
}
|
||||||
|
|
||||||
|
const NO_HOVER_PROBES: HoverProbeOutcome = {
|
||||||
|
performed: false,
|
||||||
|
revealedContent: false,
|
||||||
|
surfaceProbes: [],
|
||||||
|
tooltipNames: new Map(),
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Runs both active-hover chains back to back.
|
||||||
|
*
|
||||||
|
* Ordering matters: this happens after DOM *and* accessibility capture so a
|
||||||
|
* hover that opens a menu cannot leave one half of the observation describing
|
||||||
|
* the page before the change and the other half after it. Sharing one overlay
|
||||||
|
* bypass span also means the agent overlay toggles once per observation
|
||||||
|
* instead of once per chain.
|
||||||
|
*/
|
||||||
|
async function runHoverProbes(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
captured: CapturedViewModel,
|
||||||
|
documents: VomFrameDocument[],
|
||||||
|
staticSemantics: ReturnType<typeof resolveSemanticGraph>,
|
||||||
|
options: CaptureVomObservationOptions,
|
||||||
|
): Promise<HoverProbeOutcome> {
|
||||||
|
if (!options.conditionalSurfaceProbe) return NO_HOVER_PROBES;
|
||||||
|
|
||||||
|
return withOverlayBypass(options.hoverProbeBypassOverlay, tabId, async () => {
|
||||||
|
const surfaceProbes = await probeHoverSurfaces(cdp, tabId, captured.nodes, {
|
||||||
|
signal: options.signal,
|
||||||
|
});
|
||||||
|
throwIfAborted(options.signal, "observation");
|
||||||
|
const tooltipNames = await probeTooltipNames(cdp, tabId, documents, staticSemantics, {
|
||||||
|
signal: options.signal,
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
performed: true,
|
||||||
|
revealedContent: surfaceProbes.length > 0 || tooltipNames.size > 0,
|
||||||
|
surfaceProbes,
|
||||||
|
tooltipNames,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export interface CaptureVomObservationOptions extends VomOptions {
|
export interface CaptureVomObservationOptions extends VomOptions {
|
||||||
conditionalSurfaceProbe?: boolean;
|
conditionalSurfaceProbe?: boolean;
|
||||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||||
@@ -1051,19 +1111,26 @@ export async function captureVomObservation(
|
|||||||
excludedBackendNodeIds: captured.excludedBackendNodeIds,
|
excludedBackendNodeIds: captured.excludedBackendNodeIds,
|
||||||
});
|
});
|
||||||
const staticSemantics = resolveSemanticGraph(semanticGraph, { identifierFallback: false });
|
const staticSemantics = resolveSemanticGraph(semanticGraph, { identifierFallback: false });
|
||||||
const tooltipNames = options.conditionalSurfaceProbe
|
const hoverProbes = await runHoverProbes(
|
||||||
? await probeTooltipNames(cdp, tabId, normalizedDocuments, staticSemantics, {
|
cdp,
|
||||||
signal: options.signal,
|
tabId,
|
||||||
hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
|
captured,
|
||||||
})
|
normalizedDocuments,
|
||||||
: new Map<string, string>();
|
staticSemantics,
|
||||||
|
options,
|
||||||
|
);
|
||||||
throwIfAborted(options.signal, "observation");
|
throwIfAborted(options.signal, "observation");
|
||||||
const scene = projectSemanticGraph(
|
const scene = projectSemanticGraph(
|
||||||
normalizeSemanticStructure(
|
normalizeSemanticStructure(
|
||||||
resolveSemanticGraph(semanticGraph, { supplementalNames: tooltipNames }),
|
resolveSemanticGraph(semanticGraph, { supplementalNames: hoverProbes.tooltipNames }),
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
const decoratedScene = attachCapturedSceneAnnotations(scene, normalizedDocuments, captured);
|
const decoratedScene = attachCapturedSceneAnnotations(
|
||||||
|
scene,
|
||||||
|
normalizedDocuments,
|
||||||
|
captured,
|
||||||
|
hoverProbes.surfaceProbes,
|
||||||
|
);
|
||||||
const rendered = renderVom(decoratedScene, {
|
const rendered = renderVom(decoratedScene, {
|
||||||
maxDepth: options.maxDepth,
|
maxDepth: options.maxDepth,
|
||||||
maxTokens: options.maxTokens,
|
maxTokens: options.maxTokens,
|
||||||
@@ -1075,7 +1142,11 @@ export async function captureVomObservation(
|
|||||||
rootFrameId: captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root",
|
rootFrameId: captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root",
|
||||||
frameDocuments: normalizedDocuments,
|
frameDocuments: normalizedDocuments,
|
||||||
rendered,
|
rendered,
|
||||||
surfaceProbes: captured.surfaceProbes,
|
surfaceProbes: hoverProbes.surfaceProbes,
|
||||||
|
hoverProbe: {
|
||||||
|
performed: hoverProbes.performed,
|
||||||
|
revealedContent: hoverProbes.revealedContent,
|
||||||
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1141,6 +1212,14 @@ async function handleVomObservation(
|
|||||||
ref_count: observation.refs.length,
|
ref_count: observation.refs.length,
|
||||||
tab_id: target.tabId,
|
tab_id: target.tabId,
|
||||||
truncated: observation.truncated,
|
truncated: observation.truncated,
|
||||||
|
...(toolName === "observe" && observation.hoverProbe?.performed
|
||||||
|
? {
|
||||||
|
hover_probe: {
|
||||||
|
performed: true,
|
||||||
|
revealed_content: observation.hoverProbe.revealedContent,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
: {}),
|
||||||
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
|
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
|
||||||
? {
|
? {
|
||||||
debug: {
|
debug: {
|
||||||
@@ -1179,5 +1258,13 @@ export async function handleObserve(
|
|||||||
deps: SnapshotDeps = getDefaultDeps(),
|
deps: SnapshotDeps = getDefaultDeps(),
|
||||||
signal?: AbortSignal,
|
signal?: AbortSignal,
|
||||||
): Promise<ObserveResult | RpcError> {
|
): Promise<ObserveResult | RpcError> {
|
||||||
return handleVomObservation(manager, params, "observe", "transient_input", true, deps, signal);
|
return handleVomObservation(
|
||||||
|
manager,
|
||||||
|
params,
|
||||||
|
"observe",
|
||||||
|
"transient_input",
|
||||||
|
params.probe_hover === true,
|
||||||
|
deps,
|
||||||
|
signal,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
import { describe, expect, it, vi } from "vitest";
|
||||||
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
|
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
|
||||||
import { captureViewModel, collectOverlayExcludedBackendIds } from "../capture";
|
import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
|
||||||
|
|
||||||
// Minimal but format-accurate captureSnapshot reply: a body with one
|
// Minimal but format-accurate captureSnapshot reply: a body with one
|
||||||
// fixed full-screen overlay div carrying a password input.
|
// fixed full-screen overlay div carrying a password input.
|
||||||
@@ -251,11 +251,10 @@ describe("captureViewModel", () => {
|
|||||||
expect(input?.attrs).toEqual({ type: "password" });
|
expect(input?.attrs).toEqual({ type: "password" });
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not run conditional surface probes by default", async () => {
|
it("never hovers the page while capturing", async () => {
|
||||||
const cdp = makeCdp(fakeSnapshotReply());
|
const cdp = makeCdp(fakeSnapshotReply());
|
||||||
const result = await captureViewModel(cdp, 4);
|
await captureViewModel(cdp, 4);
|
||||||
|
|
||||||
expect(result.surfaceProbes).toEqual([]);
|
|
||||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything());
|
expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything());
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -319,9 +318,10 @@ describe("captureViewModel", () => {
|
|||||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||||
};
|
};
|
||||||
|
|
||||||
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
const captured = await captureViewModel(cdp, 4);
|
||||||
|
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||||
|
|
||||||
expect(result.surfaceProbes).toEqual([
|
expect(surfaceProbes).toEqual([
|
||||||
{
|
{
|
||||||
triggerBackendNodeId: 12,
|
triggerBackendNodeId: 12,
|
||||||
triggerPoint: { x: 956, y: 32 },
|
triggerPoint: { x: 956, y: 32 },
|
||||||
@@ -378,9 +378,10 @@ describe("captureViewModel", () => {
|
|||||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||||
};
|
};
|
||||||
|
|
||||||
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
const captured = await captureViewModel(cdp, 4);
|
||||||
|
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||||
|
|
||||||
expect(result.surfaceProbes).toEqual([
|
expect(surfaceProbes).toEqual([
|
||||||
expect.objectContaining({
|
expect.objectContaining({
|
||||||
triggerBackendNodeId: 12,
|
triggerBackendNodeId: 12,
|
||||||
subItems: ["My profile", "Sign out"],
|
subItems: ["My profile", "Sign out"],
|
||||||
@@ -422,7 +423,8 @@ describe("captureViewModel", () => {
|
|||||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||||
};
|
};
|
||||||
|
|
||||||
await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
const captured = await captureViewModel(cdp, 4);
|
||||||
|
await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||||
|
|
||||||
expect(hoverMoves).toBe(1);
|
expect(hoverMoves).toBe(1);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,84 @@
|
|||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import { ProbeBudget, resetOverlayBypassDepth, withOverlayBypass } from "../hover-perception";
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
resetOverlayBypassDepth();
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("ProbeBudget", () => {
|
||||||
|
it("refuses a candidate that cannot finish inside the budget", () => {
|
||||||
|
const budget = new ProbeBudget(500);
|
||||||
|
|
||||||
|
expect(budget.canAfford(400)).toBe(true);
|
||||||
|
expect(budget.canAfford(600)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("stops admitting candidates once the remaining budget is smaller than one", async () => {
|
||||||
|
const budget = new ProbeBudget(60);
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 40));
|
||||||
|
|
||||||
|
// A top-of-loop `elapsed > total` check would still admit this candidate
|
||||||
|
// and then overshoot by its full cost.
|
||||||
|
expect(budget.canAfford(50)).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("withOverlayBypass", () => {
|
||||||
|
it("toggles the overlay once around nested probe phases", async () => {
|
||||||
|
const bypass = vi.fn(async () => {});
|
||||||
|
|
||||||
|
await withOverlayBypass(bypass, 4, async () => {
|
||||||
|
await withOverlayBypass(bypass, 4, async () => {
|
||||||
|
expect(bypass).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
expect(bypass).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(bypass.mock.calls).toEqual([
|
||||||
|
[4, true],
|
||||||
|
[4, false],
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("restores the overlay when the probe throws", async () => {
|
||||||
|
const bypass = vi.fn(async () => {});
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
withOverlayBypass(bypass, 4, async () => {
|
||||||
|
throw new Error("probe exploded");
|
||||||
|
}),
|
||||||
|
).rejects.toThrow("probe exploded");
|
||||||
|
|
||||||
|
expect(bypass).toHaveBeenLastCalledWith(4, false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports a failed restore instead of swallowing it", async () => {
|
||||||
|
const onRestoreFailure = vi.fn();
|
||||||
|
const bypass = vi.fn(async (_tabId: number, enabled: boolean) => {
|
||||||
|
if (!enabled) throw new Error("tab is gone");
|
||||||
|
});
|
||||||
|
|
||||||
|
await withOverlayBypass(bypass, 4, async () => undefined, { onRestoreFailure });
|
||||||
|
|
||||||
|
expect(onRestoreFailure).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not pin the overlay off for the tab when a restore fails", async () => {
|
||||||
|
const failing = vi.fn(async (_tabId: number, enabled: boolean) => {
|
||||||
|
if (!enabled) throw new Error("tab is gone");
|
||||||
|
});
|
||||||
|
await withOverlayBypass(failing, 4, async () => undefined, { onRestoreFailure: () => {} });
|
||||||
|
|
||||||
|
const bypass = vi.fn(async () => {});
|
||||||
|
await withOverlayBypass(bypass, 4, async () => undefined);
|
||||||
|
|
||||||
|
expect(bypass.mock.calls).toEqual([
|
||||||
|
[4, true],
|
||||||
|
[4, false],
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("runs the probe untouched when no bypass is wired up", async () => {
|
||||||
|
await expect(withOverlayBypass(undefined, 4, async () => "done")).resolves.toBe("done");
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -132,6 +132,37 @@ describe("tooltip name enrichment", () => {
|
|||||||
expect(cdp.sendToTarget).not.toHaveBeenCalled();
|
expect(cdp.sendToTarget).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("remembers controls that revealed no tooltip so the miss is paid once", async () => {
|
||||||
|
const cdp: CdpRunner = {
|
||||||
|
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||||
|
sendToTarget: vi.fn(async (_target, method) => {
|
||||||
|
if (method === "Page.createIsolatedWorld") return { executionContextId: 31 };
|
||||||
|
if (method === "DOM.resolveNode") return { object: { objectId: "button-801" } };
|
||||||
|
if (method === "Runtime.callFunctionOn") return { result: { value: [] } };
|
||||||
|
return {};
|
||||||
|
}) as CdpRunner["sendToTarget"],
|
||||||
|
};
|
||||||
|
|
||||||
|
const documents = [
|
||||||
|
document({ tabId: 11, sessionId: "miss-session" }, 801, undefined, "miss-frame"),
|
||||||
|
];
|
||||||
|
const graph = semantics(documents);
|
||||||
|
|
||||||
|
const first = await probeTooltipNames(cdp, 11, documents, graph);
|
||||||
|
expect(first.size).toBe(0);
|
||||||
|
const hoversAfterFirst = vi
|
||||||
|
.mocked(cdp.send)
|
||||||
|
.mock.calls.filter(([, method]) => method === "Input.dispatchMouseEvent").length;
|
||||||
|
expect(hoversAfterFirst).toBeGreaterThan(0);
|
||||||
|
|
||||||
|
const second = await probeTooltipNames(cdp, 11, documents, graph);
|
||||||
|
expect(second.size).toBe(0);
|
||||||
|
expect(
|
||||||
|
vi.mocked(cdp.send).mock.calls.filter(([, method]) => method === "Input.dispatchMouseEvent")
|
||||||
|
.length,
|
||||||
|
).toBe(hoversAfterFirst);
|
||||||
|
});
|
||||||
|
|
||||||
it("keeps tooltip evidence isolated across sibling frame sessions", async () => {
|
it("keeps tooltip evidence isolated across sibling frame sessions", async () => {
|
||||||
let hoveredX = -10;
|
let hoveredX = -10;
|
||||||
const cdp: CdpRunner = {
|
const cdp: CdpRunner = {
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
|||||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||||
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
|
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||||
import type { CdpRunner } from "../shared";
|
import type { CdpRunner } from "../shared";
|
||||||
import { clearHover, waitForHover } from "./hover-perception";
|
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
|
||||||
|
|
||||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
|
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
|
||||||
const STYLE_COL = Object.fromEntries(
|
const STYLE_COL = Object.fromEntries(
|
||||||
@@ -71,14 +71,11 @@ export interface CapturedViewModel {
|
|||||||
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
|
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
|
||||||
frameParentIds?: Map<string, string>;
|
frameParentIds?: Map<string, string>;
|
||||||
rootFrameId?: string;
|
rootFrameId?: string;
|
||||||
surfaceProbes?: CapturedSurfaceProbe[];
|
|
||||||
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
|
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
|
||||||
excludedBackendNodeIds: Set<number>;
|
excludedBackendNodeIds: Set<number>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface CaptureViewModelOptions {
|
export interface CaptureViewModelOptions {
|
||||||
conditionalSurfaceProbe?: boolean;
|
|
||||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
|
||||||
signal?: AbortSignal;
|
signal?: AbortSignal;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -368,7 +365,16 @@ function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<num
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const MAX_HOVER_PROBE_MS = 2_000;
|
/**
|
||||||
|
* Ceiling for the whole hover-surface phase.
|
||||||
|
*
|
||||||
|
* The previous 2000 was only a floor: the loop checked elapsed time at the top,
|
||||||
|
* so a candidate could start with 1ms left and still run its two settle
|
||||||
|
* windows, pushing real cost to ~2.6s. This is that true ceiling, now actually
|
||||||
|
* enforced by an up-front affordability check, so the same number of candidates
|
||||||
|
* get probed under a bound that no longer lies.
|
||||||
|
*/
|
||||||
|
const MAX_HOVER_PROBE_MS = 2_600;
|
||||||
const MAX_HOVER_TRIGGERS = 6;
|
const MAX_HOVER_TRIGGERS = 6;
|
||||||
const MAX_HOVER_SURFACES = 3;
|
const MAX_HOVER_SURFACES = 3;
|
||||||
const HOVER_SETTLE_MS = 300;
|
const HOVER_SETTLE_MS = 300;
|
||||||
@@ -614,13 +620,35 @@ function confidenceForHover(
|
|||||||
return "low";
|
return "low";
|
||||||
}
|
}
|
||||||
|
|
||||||
async function probeHoverSurfaces(
|
/**
|
||||||
|
* One candidate costs two settle windows (baseline + post-hover) plus a few
|
||||||
|
* CDP round trips. Used to decide whether the next candidate still fits the
|
||||||
|
* budget before paying for it.
|
||||||
|
*/
|
||||||
|
const HOVER_CANDIDATE_COST_MS = HOVER_SETTLE_MS * 2;
|
||||||
|
|
||||||
|
export interface HoverSurfaceProbeOptions {
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Hovers a bounded set of likely menu triggers and reports the sub-items each
|
||||||
|
* one reveals.
|
||||||
|
*
|
||||||
|
* Runs after DOM *and* accessibility capture. Hovering can open menus and
|
||||||
|
* change layout, so probing between the two captures would leave the DOM half
|
||||||
|
* of an observation describing the page before the change and the AX half
|
||||||
|
* describing it after.
|
||||||
|
*
|
||||||
|
* The caller owns the overlay bypass span (see `withOverlayBypass`).
|
||||||
|
*/
|
||||||
|
export async function probeHoverSurfaces(
|
||||||
cdp: CdpRunner,
|
cdp: CdpRunner,
|
||||||
tabId: number,
|
tabId: number,
|
||||||
nodes: CapturedNode[],
|
nodes: CapturedNode[],
|
||||||
options: CaptureViewModelOptions,
|
options: HoverSurfaceProbeOptions = {},
|
||||||
): Promise<CapturedSurfaceProbe[]> {
|
): Promise<CapturedSurfaceProbe[]> {
|
||||||
const started = Date.now();
|
const budget = new ProbeBudget(MAX_HOVER_PROBE_MS);
|
||||||
try {
|
try {
|
||||||
throwIfAborted(options.signal);
|
throwIfAborted(options.signal);
|
||||||
const cssScan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
const cssScan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||||
@@ -635,60 +663,54 @@ async function probeHoverSurfaces(
|
|||||||
const results: CapturedSurfaceProbe[] = [];
|
const results: CapturedSurfaceProbe[] = [];
|
||||||
const seen = new Set<number>();
|
const seen = new Set<number>();
|
||||||
throwIfAborted(options.signal);
|
throwIfAborted(options.signal);
|
||||||
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
|
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
|
||||||
try {
|
|
||||||
throwIfAborted(options.signal);
|
throwIfAborted(options.signal);
|
||||||
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
|
if (!budget.canAfford(HOVER_CANDIDATE_COST_MS)) break;
|
||||||
|
if (results.length >= MAX_HOVER_SURFACES) break;
|
||||||
|
if (seen.has(candidate.backendNodeId)) continue;
|
||||||
|
try {
|
||||||
|
await clearHover(cdp, tabId);
|
||||||
throwIfAborted(options.signal);
|
throwIfAborted(options.signal);
|
||||||
if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
|
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
||||||
if (results.length >= MAX_HOVER_SURFACES) break;
|
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||||
if (seen.has(candidate.backendNodeId)) continue;
|
expression: hoverStateExpression(),
|
||||||
try {
|
returnByValue: true,
|
||||||
await clearHover(cdp, tabId);
|
});
|
||||||
throwIfAborted(options.signal);
|
throwIfAborted(options.signal);
|
||||||
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
|
||||||
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
|
||||||
expression: hoverStateExpression(),
|
|
||||||
returnByValue: true,
|
|
||||||
});
|
|
||||||
throwIfAborted(options.signal);
|
|
||||||
const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
|
|
||||||
|
|
||||||
throwIfAborted(options.signal);
|
throwIfAborted(options.signal);
|
||||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||||
type: "mouseMoved",
|
type: "mouseMoved",
|
||||||
x: candidate.x,
|
x: candidate.x,
|
||||||
y: candidate.y,
|
y: candidate.y,
|
||||||
});
|
});
|
||||||
throwIfAborted(options.signal);
|
throwIfAborted(options.signal);
|
||||||
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
||||||
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||||
expression: hoverStateExpression(),
|
expression: hoverStateExpression(),
|
||||||
returnByValue: true,
|
returnByValue: true,
|
||||||
});
|
});
|
||||||
throwIfAborted(options.signal);
|
throwIfAborted(options.signal);
|
||||||
const subItems = diffHoverItems(
|
const subItems = diffHoverItems(
|
||||||
baselineItems,
|
baselineItems,
|
||||||
runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
|
runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
|
||||||
);
|
);
|
||||||
if (subItems.length === 0) continue;
|
if (subItems.length === 0) continue;
|
||||||
seen.add(candidate.backendNodeId);
|
seen.add(candidate.backendNodeId);
|
||||||
results.push({
|
results.push({
|
||||||
triggerBackendNodeId: candidate.backendNodeId,
|
triggerBackendNodeId: candidate.backendNodeId,
|
||||||
triggerPoint: { x: candidate.x, y: candidate.y },
|
triggerPoint: { x: candidate.x, y: candidate.y },
|
||||||
triggerAction: "hover",
|
triggerAction: "hover",
|
||||||
subItems,
|
subItems,
|
||||||
confidence: confidenceForHover(candidate, subItems),
|
confidence: confidenceForHover(candidate, subItems),
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (isAbortError(error)) throw error;
|
if (isAbortError(error)) throw error;
|
||||||
continue;
|
continue;
|
||||||
} finally {
|
} finally {
|
||||||
await clearHover(cdp, tabId);
|
await clearHover(cdp, tabId);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} finally {
|
|
||||||
await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
|
|
||||||
}
|
}
|
||||||
return results;
|
return results;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -1042,7 +1064,6 @@ export async function captureViewModel(
|
|||||||
nodes: [],
|
nodes: [],
|
||||||
viewport,
|
viewport,
|
||||||
iframeNodes: new Map(),
|
iframeNodes: new Map(),
|
||||||
surfaceProbes: [],
|
|
||||||
excludedBackendNodeIds: new Set(),
|
excludedBackendNodeIds: new Set(),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -1071,11 +1092,6 @@ export async function captureViewModel(
|
|||||||
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()], options.signal);
|
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()], options.signal);
|
||||||
throwIfAborted(options.signal);
|
throwIfAborted(options.signal);
|
||||||
|
|
||||||
const surfaceProbes = options.conditionalSurfaceProbe
|
|
||||||
? await probeHoverSurfaces(cdp, tabId, nodes, options)
|
|
||||||
: [];
|
|
||||||
throwIfAborted(options.signal);
|
|
||||||
|
|
||||||
const frameNodes = new Map<string, CapturedNode[]>();
|
const frameNodes = new Map<string, CapturedNode[]>();
|
||||||
if (topContext.frameId) frameNodes.set(topContext.frameId, nodes);
|
if (topContext.frameId) frameNodes.set(topContext.frameId, nodes);
|
||||||
for (const iframeFrameNodes of iframeNodes.values()) {
|
for (const iframeFrameNodes of iframeNodes.values()) {
|
||||||
@@ -1096,7 +1112,6 @@ export async function captureViewModel(
|
|||||||
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
|
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
|
||||||
frameParentIds: frameParsed.frameParentIds,
|
frameParentIds: frameParsed.frameParentIds,
|
||||||
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
|
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
|
||||||
surfaceProbes,
|
|
||||||
excludedBackendNodeIds,
|
excludedBackendNodeIds,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -35,3 +35,91 @@ export async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
|
|||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wall-clock budget for a probe phase.
|
||||||
|
*
|
||||||
|
* Callers ask whether the *next* candidate still fits before starting it,
|
||||||
|
* rather than checking elapsed time at the top of the loop. A top-of-loop
|
||||||
|
* check lets a candidate start with 1ms of budget left and then run to
|
||||||
|
* completion, overshooting by a full settle window.
|
||||||
|
*/
|
||||||
|
export class ProbeBudget {
|
||||||
|
private readonly startedAt = Date.now();
|
||||||
|
|
||||||
|
constructor(private readonly totalMs: number) {}
|
||||||
|
|
||||||
|
get elapsedMs(): number {
|
||||||
|
return Date.now() - this.startedAt;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** True when a candidate costing `estimatedMs` still fits in the budget. */
|
||||||
|
canAfford(estimatedMs: number): boolean {
|
||||||
|
return this.elapsedMs + estimatedMs <= this.totalMs;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export type OverlayBypass = (tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
|
||||||
|
export interface OverlayBypassScopeOptions {
|
||||||
|
/**
|
||||||
|
* Called when the overlay could not be restored. Defaults to `console.error`
|
||||||
|
* so a stuck-bypassed overlay is never silent; tests inject a spy.
|
||||||
|
*/
|
||||||
|
onRestoreFailure?: (error: unknown) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
const overlayBypassDepth = new Map<number, number>();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Runs `body` with the agent overlay bypassed, restoring it afterwards.
|
||||||
|
*
|
||||||
|
* Reference counted per tab so nested probe phases share a single bypass span
|
||||||
|
* instead of toggling the overlay once per phase. Restore failures are
|
||||||
|
* reported rather than swallowed, and the depth is released even when restore
|
||||||
|
* throws so a failure cannot pin the overlay off for the tab's lifetime.
|
||||||
|
*/
|
||||||
|
export async function withOverlayBypass<T>(
|
||||||
|
bypass: OverlayBypass | undefined,
|
||||||
|
tabId: number,
|
||||||
|
body: () => Promise<T>,
|
||||||
|
options: OverlayBypassScopeOptions = {},
|
||||||
|
): Promise<T> {
|
||||||
|
if (!bypass) return body();
|
||||||
|
|
||||||
|
const depth = overlayBypassDepth.get(tabId) ?? 0;
|
||||||
|
overlayBypassDepth.set(tabId, depth + 1);
|
||||||
|
if (depth === 0) {
|
||||||
|
try {
|
||||||
|
await bypass(tabId, true);
|
||||||
|
} catch (error) {
|
||||||
|
// Failing to hide the overlay only risks probing our own UI, so continue.
|
||||||
|
console.debug("[bsk hover] overlay bypass could not be enabled", error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await body();
|
||||||
|
} finally {
|
||||||
|
const current = overlayBypassDepth.get(tabId) ?? 1;
|
||||||
|
if (current <= 1) {
|
||||||
|
overlayBypassDepth.delete(tabId);
|
||||||
|
try {
|
||||||
|
await bypass(tabId, false);
|
||||||
|
} catch (error) {
|
||||||
|
(options.onRestoreFailure ?? defaultRestoreFailure)(error);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
overlayBypassDepth.set(tabId, current - 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function defaultRestoreFailure(error: unknown): void {
|
||||||
|
console.error("[bsk hover] agent overlay could not be restored after probing", error);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Test seam: drops any bypass depth left behind by a crashed probe. */
|
||||||
|
export function resetOverlayBypassDepth(): void {
|
||||||
|
overlayBypassDepth.clear();
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import { isVomReferenceNode } from "@browser-skill/vom";
|
import { isVomReferenceNode } from "@browser-skill/vom";
|
||||||
import { type CdpRunner, sendToCdpTarget } from "../shared";
|
import { type CdpRunner, sendToCdpTarget } from "../shared";
|
||||||
import type { FrameDocument, FrameOwnedAxNode } from "./frame-document";
|
import type { FrameDocument, FrameOwnedAxNode } from "./frame-document";
|
||||||
import { clearHover, waitForHover } from "./hover-perception";
|
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
|
||||||
import { stableIdentifierName } from "./semantic-graph/name-evidence";
|
import { stableIdentifierName } from "./semantic-graph/name-evidence";
|
||||||
import { frameBackendKey, type ResolvedSemanticGraph } from "./semantic-graph/types";
|
import { frameBackendKey, type ResolvedSemanticGraph } from "./semantic-graph/types";
|
||||||
|
|
||||||
@@ -9,11 +9,18 @@ const MAX_TOOLTIP_CANDIDATES = 8;
|
|||||||
const MAX_TOOLTIP_PROBE_MS = 2_400;
|
const MAX_TOOLTIP_PROBE_MS = 2_400;
|
||||||
const TOOLTIP_SETTLE_MS = 250;
|
const TOOLTIP_SETTLE_MS = 250;
|
||||||
const CACHE_TTL_MS = 5 * 60_000;
|
const CACHE_TTL_MS = 5 * 60_000;
|
||||||
|
/**
|
||||||
|
* Misses expire sooner than hits: a control that gains a tooltip once the app
|
||||||
|
* finishes booting should still be discoverable, but not at the cost of
|
||||||
|
* re-probing every unlabelled control on every observation.
|
||||||
|
*/
|
||||||
|
const MISS_CACHE_TTL_MS = 60_000;
|
||||||
const MAX_CACHE_ENTRIES = 500;
|
const MAX_CACHE_ENTRIES = 500;
|
||||||
const FORM_CONTROL_TAGS = new Set(["input", "select", "textarea"]);
|
const FORM_CONTROL_TAGS = new Set(["input", "select", "textarea"]);
|
||||||
|
|
||||||
interface TooltipCacheEntry {
|
interface TooltipCacheEntry {
|
||||||
name: string;
|
/** `null` records "probed, revealed no tooltip" so the miss is not re-paid. */
|
||||||
|
name: string | null;
|
||||||
expiresAt: number;
|
expiresAt: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -27,7 +34,6 @@ interface TooltipCandidate<T extends FrameOwnedAxNode> {
|
|||||||
|
|
||||||
export interface TooltipNameProbeOptions {
|
export interface TooltipNameProbeOptions {
|
||||||
signal?: AbortSignal;
|
signal?: AbortSignal;
|
||||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const tooltipCache = new Map<string, TooltipCacheEntry>();
|
const tooltipCache = new Map<string, TooltipCacheEntry>();
|
||||||
@@ -39,10 +45,11 @@ function cacheKey<T extends FrameOwnedAxNode>(
|
|||||||
return `${document.target.tabId}\u0000${document.frameId}\u0000${document.url ?? ""}\u0000${backendNodeId}`;
|
return `${document.target.tabId}\u0000${document.frameId}\u0000${document.url ?? ""}\u0000${backendNodeId}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
function cachedName<T extends FrameOwnedAxNode>(
|
/** Returns the cache entry, or `undefined` when this control was never probed. */
|
||||||
|
function cachedEntry<T extends FrameOwnedAxNode>(
|
||||||
document: FrameDocument<T>,
|
document: FrameDocument<T>,
|
||||||
backendNodeId: number,
|
backendNodeId: number,
|
||||||
): string | undefined {
|
): TooltipCacheEntry | undefined {
|
||||||
const key = cacheKey(document, backendNodeId);
|
const key = cacheKey(document, backendNodeId);
|
||||||
const entry = tooltipCache.get(key);
|
const entry = tooltipCache.get(key);
|
||||||
if (!entry) return undefined;
|
if (!entry) return undefined;
|
||||||
@@ -50,13 +57,13 @@ function cachedName<T extends FrameOwnedAxNode>(
|
|||||||
tooltipCache.delete(key);
|
tooltipCache.delete(key);
|
||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
return entry.name;
|
return entry;
|
||||||
}
|
}
|
||||||
|
|
||||||
function storeName<T extends FrameOwnedAxNode>(
|
function storeName<T extends FrameOwnedAxNode>(
|
||||||
document: FrameDocument<T>,
|
document: FrameDocument<T>,
|
||||||
backendNodeId: number,
|
backendNodeId: number,
|
||||||
name: string,
|
name: string | null,
|
||||||
): void {
|
): void {
|
||||||
if (tooltipCache.size >= MAX_CACHE_ENTRIES) {
|
if (tooltipCache.size >= MAX_CACHE_ENTRIES) {
|
||||||
const oldest = tooltipCache.keys().next().value;
|
const oldest = tooltipCache.keys().next().value;
|
||||||
@@ -64,7 +71,7 @@ function storeName<T extends FrameOwnedAxNode>(
|
|||||||
}
|
}
|
||||||
tooltipCache.set(cacheKey(document, backendNodeId), {
|
tooltipCache.set(cacheKey(document, backendNodeId), {
|
||||||
name,
|
name,
|
||||||
expiresAt: Date.now() + CACHE_TTL_MS,
|
expiresAt: Date.now() + (name === null ? MISS_CACHE_TTL_MS : CACHE_TTL_MS),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -237,22 +244,27 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
|
|||||||
const names = new Map<string, string>();
|
const names = new Map<string, string>();
|
||||||
const pending: TooltipCandidate<T>[] = [];
|
const pending: TooltipCandidate<T>[] = [];
|
||||||
for (const candidate of candidates(documents, graph)) {
|
for (const candidate of candidates(documents, graph)) {
|
||||||
const cached = cachedName(candidate.document, candidate.backendNodeId);
|
const cached = cachedEntry(candidate.document, candidate.backendNodeId);
|
||||||
if (cached) {
|
if (cached) {
|
||||||
names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), cached);
|
// A cached miss is still a decision: skip the control instead of re-probing.
|
||||||
|
if (cached.name !== null) {
|
||||||
|
names.set(
|
||||||
|
frameBackendKey(candidate.document.frameId, candidate.backendNodeId),
|
||||||
|
cached.name,
|
||||||
|
);
|
||||||
|
}
|
||||||
} else if (pending.length < MAX_TOOLTIP_CANDIDATES) {
|
} else if (pending.length < MAX_TOOLTIP_CANDIDATES) {
|
||||||
pending.push(candidate);
|
pending.push(candidate);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (pending.length === 0) return names;
|
if (pending.length === 0) return names;
|
||||||
|
|
||||||
const started = Date.now();
|
const budget = new ProbeBudget(MAX_TOOLTIP_PROBE_MS);
|
||||||
const contexts = new Map<string, number>();
|
const contexts = new Map<string, number>();
|
||||||
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
|
|
||||||
try {
|
try {
|
||||||
for (const candidate of pending) {
|
for (const candidate of pending) {
|
||||||
if (options.signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
if (options.signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
||||||
if (Date.now() - started >= MAX_TOOLTIP_PROBE_MS) break;
|
if (!budget.canAfford(TOOLTIP_SETTLE_MS)) break;
|
||||||
let objectId: string | undefined;
|
let objectId: string | undefined;
|
||||||
try {
|
try {
|
||||||
let contextId = contexts.get(candidate.document.frameId);
|
let contextId = contexts.get(candidate.document.frameId);
|
||||||
@@ -278,9 +290,9 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
|
|||||||
await waitForHover(TOOLTIP_SETTLE_MS, options.signal);
|
await waitForHover(TOOLTIP_SETTLE_MS, options.signal);
|
||||||
const after = await tooltipTexts(cdp, candidate.document, objectId);
|
const after = await tooltipTexts(cdp, candidate.document, objectId);
|
||||||
const name = cleanTooltipDiff(before, after);
|
const name = cleanTooltipDiff(before, after);
|
||||||
|
storeName(candidate.document, candidate.backendNodeId, name ?? null);
|
||||||
if (!name) continue;
|
if (!name) continue;
|
||||||
names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), name);
|
names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), name);
|
||||||
storeName(candidate.document, candidate.backendNodeId, name);
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error instanceof Error && error.name === "AbortError") throw error;
|
if (error instanceof Error && error.name === "AbortError") throw error;
|
||||||
} finally {
|
} finally {
|
||||||
@@ -290,7 +302,6 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
|
|||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
await clearHover(cdp, tabId);
|
await clearHover(cdp, tabId);
|
||||||
await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
|
|
||||||
}
|
}
|
||||||
return names;
|
return names;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -37,6 +37,17 @@ export interface CaptureVomObservationResult {
|
|||||||
frames: CaptureVomFrame[];
|
frames: CaptureVomFrame[];
|
||||||
matchNodes: CaptureVomMatchNode[];
|
matchNodes: CaptureVomMatchNode[];
|
||||||
surfaceProbes?: CapturedSurfaceProbe[];
|
surfaceProbes?: CapturedSurfaceProbe[];
|
||||||
|
/**
|
||||||
|
* Whether this observation actively hovered the page, and whether that
|
||||||
|
* revealed content the static tree does not contain. Lets callers judge how
|
||||||
|
* far the page may have drifted from the returned snapshot.
|
||||||
|
*/
|
||||||
|
hoverProbe?: HoverProbeReport;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HoverProbeReport {
|
||||||
|
performed: boolean;
|
||||||
|
revealedContent: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
function projectFrames(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomFrame[] {
|
function projectFrames(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomFrame[] {
|
||||||
@@ -75,6 +86,7 @@ export function projectRecordSafeObservation(input: {
|
|||||||
frameDocuments: CapturedFrameDocument<FrameAxNode>[];
|
frameDocuments: CapturedFrameDocument<FrameAxNode>[];
|
||||||
rendered: VomResult;
|
rendered: VomResult;
|
||||||
surfaceProbes?: CapturedSurfaceProbe[];
|
surfaceProbes?: CapturedSurfaceProbe[];
|
||||||
|
hoverProbe?: HoverProbeReport;
|
||||||
}): CaptureVomObservationResult {
|
}): CaptureVomObservationResult {
|
||||||
return {
|
return {
|
||||||
text: input.rendered.text,
|
text: input.rendered.text,
|
||||||
@@ -84,5 +96,6 @@ export function projectRecordSafeObservation(input: {
|
|||||||
frames: projectFrames(input.frameDocuments),
|
frames: projectFrames(input.frameDocuments),
|
||||||
matchNodes: projectMatchNodes(input.frameDocuments),
|
matchNodes: projectMatchNodes(input.frameDocuments),
|
||||||
surfaceProbes: input.surfaceProbes,
|
surfaceProbes: input.surfaceProbes,
|
||||||
|
...(input.hoverProbe ? { hoverProbe: input.hoverProbe } : {}),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,7 +24,6 @@ export type RpcErrorReason =
|
|||||||
| "agent_window_scope"
|
| "agent_window_scope"
|
||||||
| "element_not_visible"
|
| "element_not_visible"
|
||||||
| "ref_not_found"
|
| "ref_not_found"
|
||||||
| "invalid_selector"
|
|
||||||
| "selector_not_found"
|
| "selector_not_found"
|
||||||
| "target_not_fillable"
|
| "target_not_fillable"
|
||||||
| "target_not_select"
|
| "target_not_select"
|
||||||
@@ -319,9 +318,14 @@ export interface SnapshotResult {
|
|||||||
|
|
||||||
export interface ObserveParams extends SnapshotParams {
|
export interface ObserveParams extends SnapshotParams {
|
||||||
debug_surfaces?: boolean;
|
debug_surfaces?: boolean;
|
||||||
|
probe_hover?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ObserveResult extends SnapshotResult {
|
export interface ObserveResult extends SnapshotResult {
|
||||||
|
hover_probe?: {
|
||||||
|
performed: boolean;
|
||||||
|
revealed_content: boolean;
|
||||||
|
};
|
||||||
debug?: {
|
debug?: {
|
||||||
surface_probes?: Array<{
|
surface_probes?: Array<{
|
||||||
trigger_backend_node_id: number;
|
trigger_backend_node_id: number;
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
import type { DiagnosticSink } from "@/lib/diagnostics";
|
|
||||||
import type { ConnectionStateHandler, Disposable, FrameHandler, Transport } from "./transport";
|
import type { ConnectionStateHandler, Disposable, FrameHandler, Transport } from "./transport";
|
||||||
import type { ConnectionState, ProtocolFrame } from "./types";
|
import type { ConnectionState, ProtocolFrame } from "./types";
|
||||||
|
|
||||||
@@ -19,7 +18,6 @@ export interface WSTransportOptions {
|
|||||||
* Inject a fake WebSocket constructor in tests; defaults to `globalThis.WebSocket`.
|
* Inject a fake WebSocket constructor in tests; defaults to `globalThis.WebSocket`.
|
||||||
*/
|
*/
|
||||||
webSocketFactory?: WebSocketFactory;
|
webSocketFactory?: WebSocketFactory;
|
||||||
diagnostics?: DiagnosticSink;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
interface CloseLikeEvent {
|
interface CloseLikeEvent {
|
||||||
@@ -46,7 +44,6 @@ export class WSTransport implements Transport {
|
|||||||
private readonly factory: WebSocketFactory;
|
private readonly factory: WebSocketFactory;
|
||||||
private readonly initialDelayMs: number;
|
private readonly initialDelayMs: number;
|
||||||
private readonly maxDelayMs: number;
|
private readonly maxDelayMs: number;
|
||||||
private readonly diagnostics?: DiagnosticSink;
|
|
||||||
|
|
||||||
private socket: WebSocket | null = null;
|
private socket: WebSocket | null = null;
|
||||||
private currentState: ConnectionState = "disconnected";
|
private currentState: ConnectionState = "disconnected";
|
||||||
@@ -66,7 +63,6 @@ export class WSTransport implements Transport {
|
|||||||
this.factory = options.webSocketFactory ?? ((url: string) => new WebSocket(url));
|
this.factory = options.webSocketFactory ?? ((url: string) => new WebSocket(url));
|
||||||
this.initialDelayMs = options.reconnect?.initialDelayMs ?? DEFAULT_INITIAL_DELAY_MS;
|
this.initialDelayMs = options.reconnect?.initialDelayMs ?? DEFAULT_INITIAL_DELAY_MS;
|
||||||
this.maxDelayMs = options.reconnect?.maxDelayMs ?? DEFAULT_MAX_DELAY_MS;
|
this.maxDelayMs = options.reconnect?.maxDelayMs ?? DEFAULT_MAX_DELAY_MS;
|
||||||
this.diagnostics = options.diagnostics;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
get state(): ConnectionState {
|
get state(): ConnectionState {
|
||||||
@@ -74,13 +70,6 @@ export class WSTransport implements Transport {
|
|||||||
}
|
}
|
||||||
|
|
||||||
connect(): Promise<void> {
|
connect(): Promise<void> {
|
||||||
this.diagnostics?.("transport.connect.requested", {
|
|
||||||
state: this.currentState,
|
|
||||||
socket_generation: this.socketGeneration,
|
|
||||||
has_socket: this.socket !== null,
|
|
||||||
has_connecting_promise: this.connectingPromise !== null,
|
|
||||||
has_reconnect_timer: this.reconnectTimer !== null,
|
|
||||||
});
|
|
||||||
if (this.currentState === "connected") return Promise.resolve();
|
if (this.currentState === "connected") return Promise.resolve();
|
||||||
|
|
||||||
this.explicitlyClosed = false;
|
this.explicitlyClosed = false;
|
||||||
@@ -105,11 +94,6 @@ export class WSTransport implements Transport {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async disconnect(): Promise<void> {
|
async disconnect(): Promise<void> {
|
||||||
this.diagnostics?.("transport.disconnect.requested", {
|
|
||||||
state: this.currentState,
|
|
||||||
socket_generation: this.socketGeneration,
|
|
||||||
has_socket: this.socket !== null,
|
|
||||||
});
|
|
||||||
this.explicitlyClosed = true;
|
this.explicitlyClosed = true;
|
||||||
if (this.reconnectTimer) {
|
if (this.reconnectTimer) {
|
||||||
clearTimeout(this.reconnectTimer);
|
clearTimeout(this.reconnectTimer);
|
||||||
@@ -140,12 +124,6 @@ export class WSTransport implements Transport {
|
|||||||
|
|
||||||
send(msg: ProtocolFrame): void {
|
send(msg: ProtocolFrame): void {
|
||||||
if (!this.socket || this.socket.readyState !== 1 /* OPEN */) {
|
if (!this.socket || this.socket.readyState !== 1 /* OPEN */) {
|
||||||
this.diagnostics?.("transport.send.rejected", {
|
|
||||||
state: this.currentState,
|
|
||||||
socket_generation: this.socketGeneration,
|
|
||||||
socket_ready_state: this.socket?.readyState ?? null,
|
|
||||||
frame_kind: "method" in msg ? msg.method : "event" in msg ? msg.event : "response",
|
|
||||||
});
|
|
||||||
throw new Error("[WSTransport] cannot send while not connected");
|
throw new Error("[WSTransport] cannot send while not connected");
|
||||||
}
|
}
|
||||||
this.socket.send(JSON.stringify(msg));
|
this.socket.send(JSON.stringify(msg));
|
||||||
@@ -174,16 +152,10 @@ export class WSTransport implements Transport {
|
|||||||
const generation = ++this.socketGeneration;
|
const generation = ++this.socketGeneration;
|
||||||
const socket = this.factory(this.url);
|
const socket = this.factory(this.url);
|
||||||
this.socket = socket;
|
this.socket = socket;
|
||||||
this.diagnostics?.("transport.socket.created", {
|
|
||||||
socket_generation: generation,
|
|
||||||
reconnect_attempt: this.reconnectAttempt,
|
|
||||||
url: this.url,
|
|
||||||
});
|
|
||||||
|
|
||||||
socket.addEventListener("open", () => {
|
socket.addEventListener("open", () => {
|
||||||
if (!this.isCurrentSocket(socket, generation)) return;
|
if (!this.isCurrentSocket(socket, generation)) return;
|
||||||
this.reconnectAttempt = 0;
|
this.reconnectAttempt = 0;
|
||||||
this.diagnostics?.("transport.socket.open", { socket_generation: generation });
|
|
||||||
this.setState("connected");
|
this.setState("connected");
|
||||||
const resolve = this.resolveConnect;
|
const resolve = this.resolveConnect;
|
||||||
this.resolveConnect = null;
|
this.resolveConnect = null;
|
||||||
@@ -202,7 +174,6 @@ export class WSTransport implements Transport {
|
|||||||
});
|
});
|
||||||
|
|
||||||
socket.addEventListener("error", () => {
|
socket.addEventListener("error", () => {
|
||||||
this.diagnostics?.("transport.socket.error", { socket_generation: generation });
|
|
||||||
// The 'close' handler always fires after 'error' in browsers, so we
|
// The 'close' handler always fires after 'error' in browsers, so we
|
||||||
// just record the error and let close drive reconnect logic.
|
// just record the error and let close drive reconnect logic.
|
||||||
});
|
});
|
||||||
@@ -226,16 +197,7 @@ export class WSTransport implements Transport {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private handleClose(socket: WebSocket, generation: number, _ev: CloseLikeEvent): void {
|
private handleClose(socket: WebSocket, generation: number, _ev: CloseLikeEvent): void {
|
||||||
const isCurrent = this.isCurrentSocket(socket, generation);
|
if (!this.isCurrentSocket(socket, generation)) return;
|
||||||
this.diagnostics?.("transport.socket.close", {
|
|
||||||
socket_generation: generation,
|
|
||||||
current_socket_generation: this.socketGeneration,
|
|
||||||
is_current: isCurrent,
|
|
||||||
explicitly_closed: this.explicitlyClosed,
|
|
||||||
close_code: _ev.code ?? null,
|
|
||||||
close_reason: _ev.reason ?? "",
|
|
||||||
});
|
|
||||||
if (!isCurrent) return;
|
|
||||||
this.socket = null;
|
this.socket = null;
|
||||||
if (this.explicitlyClosed) {
|
if (this.explicitlyClosed) {
|
||||||
this.setState("disconnected");
|
this.setState("disconnected");
|
||||||
@@ -249,18 +211,9 @@ export class WSTransport implements Transport {
|
|||||||
if (this.reconnectTimer) return;
|
if (this.reconnectTimer) return;
|
||||||
const delay = Math.min(this.initialDelayMs * 2 ** this.reconnectAttempt, this.maxDelayMs);
|
const delay = Math.min(this.initialDelayMs * 2 ** this.reconnectAttempt, this.maxDelayMs);
|
||||||
this.reconnectAttempt += 1;
|
this.reconnectAttempt += 1;
|
||||||
this.diagnostics?.("transport.reconnect.scheduled", {
|
|
||||||
delay_ms: delay,
|
|
||||||
reconnect_attempt: this.reconnectAttempt,
|
|
||||||
socket_generation: this.socketGeneration,
|
|
||||||
});
|
|
||||||
this.reconnectTimer = setTimeout(() => {
|
this.reconnectTimer = setTimeout(() => {
|
||||||
this.reconnectTimer = null;
|
this.reconnectTimer = null;
|
||||||
if (this.explicitlyClosed) return;
|
if (this.explicitlyClosed) return;
|
||||||
this.diagnostics?.("transport.reconnect.timer_fired", {
|
|
||||||
reconnect_attempt: this.reconnectAttempt,
|
|
||||||
socket_generation: this.socketGeneration,
|
|
||||||
});
|
|
||||||
void this.connect().catch((err) => {
|
void this.connect().catch((err) => {
|
||||||
console.debug("[WSTransport] reconnect attempt failed", err);
|
console.debug("[WSTransport] reconnect attempt failed", err);
|
||||||
});
|
});
|
||||||
@@ -273,13 +226,7 @@ export class WSTransport implements Transport {
|
|||||||
|
|
||||||
private setState(next: ConnectionState): void {
|
private setState(next: ConnectionState): void {
|
||||||
if (this.currentState === next) return;
|
if (this.currentState === next) return;
|
||||||
const previous = this.currentState;
|
|
||||||
this.currentState = next;
|
this.currentState = next;
|
||||||
this.diagnostics?.("transport.state.changed", {
|
|
||||||
previous,
|
|
||||||
next,
|
|
||||||
socket_generation: this.socketGeneration,
|
|
||||||
});
|
|
||||||
for (const h of this.stateHandlers) {
|
for (const h of this.stateHandlers) {
|
||||||
try {
|
try {
|
||||||
h(next);
|
h(next);
|
||||||
|
|||||||
@@ -22,8 +22,7 @@ export default defineConfig({
|
|||||||
outDir: "dist",
|
outDir: "dist",
|
||||||
modules: ["@wxt-dev/module-react"],
|
modules: ["@wxt-dev/module-react"],
|
||||||
manifest: {
|
manifest: {
|
||||||
name: "BrowserSkill (Lumi Diagnostics)",
|
name: "BrowserSkill",
|
||||||
version_name: `${EXTENSION_VERSION}-lumi-session-loss-v2`,
|
|
||||||
description:
|
description:
|
||||||
"Let AI agents use your logged-in browser in a separate Agent Window—without interrupting your work. Powered by the bsk CLI.",
|
"Let AI agents use your logged-in browser in a separate Agent Window—without interrupting your work. Powered by the bsk CLI.",
|
||||||
// Flat debugger sessions are required to address out-of-process iframes.
|
// Flat debugger sessions are required to address out-of-process iframes.
|
||||||
@@ -46,7 +45,7 @@ export default defineConfig({
|
|||||||
128: "icon/logo.png",
|
128: "icon/logo.png",
|
||||||
},
|
},
|
||||||
action: {
|
action: {
|
||||||
default_title: "BrowserSkill (Lumi Diagnostics)",
|
default_title: "BrowserSkill",
|
||||||
default_icon: {
|
default_icon: {
|
||||||
16: "icon/logo.png",
|
16: "icon/logo.png",
|
||||||
32: "icon/logo.png",
|
32: "icon/logo.png",
|
||||||
|
|||||||
@@ -67,7 +67,7 @@ Write operations only affect tabs in the **Agent Window** (or tabs you **borrowe
|
|||||||
|
|
||||||
```
|
```
|
||||||
bsk navigate <url> --session <id>
|
bsk navigate <url> --session <id>
|
||||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
bsk observe --session <id> → primary semantic VOM view
|
||||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
||||||
@@ -78,16 +78,19 @@ bsk observe --session <id> → again after navigation / DOM change
|
|||||||
|
|
||||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
||||||
|
|
||||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
When VOM marks a control `[has-submenu]` or `[expanded]`, its menu items are not in the observation yet. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action. `[hover first: …]` means the same thing and additionally lists what that trigger revealed.
|
||||||
|
|
||||||
|
`bsk observe` does not hover the page on its own. Reach for `--probe-hover` when a control you have good reason to expect is absent from the observation **and** no `[has-submenu]` marker points at a trigger — that combination is what a CSS-only hover menu looks like from here. Try it before falling back to `bsk get-html` or `bsk screenshot`. It hovers a bounded set of likely triggers to surface hover-only menus and tooltips; it costs a few seconds and touches the live page, so once you know which element hides the menu, `bsk hover <ref>` is cheaper and more precise.
|
||||||
|
|
||||||
## Observation priority
|
## Observation priority
|
||||||
|
|
||||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
Start with `bsk observe` to understand page structure, text, controls, and element refs. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||||
|
|
||||||
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
1. `bsk observe` — primary semantic VOM observation
|
||||||
2. `bsk snapshot` — strict static accessibility tree fallback
|
2. `bsk observe --probe-hover` — re-run once when an expected control is missing and nothing marks a hover trigger
|
||||||
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
3. `bsk snapshot` — strict static accessibility tree fallback
|
||||||
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
4. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
||||||
|
5. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
||||||
|
|
||||||
Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page.
|
Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page.
|
||||||
|
|
||||||
@@ -172,7 +175,7 @@ bsk emulate --session <id> --off
|
|||||||
| Command | Summary |
|
| Command | Summary |
|
||||||
|---------|---------|
|
|---------|---------|
|
||||||
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
|
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
|
||||||
| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
|
| `bsk observe` | Semantic VOM observation; `--probe-hover` adds bounded active hover probing for conditional surfaces |
|
||||||
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
|
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
|
||||||
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
|
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
|
||||||
|
|
||||||
@@ -319,7 +322,7 @@ Always **`bsk session stop <id>`** in a `finally`-style path so the Agent Window
|
|||||||
2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
|
2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
|
||||||
3. **No skip stop** — always `bsk session stop <id>`; never assume idle timeout will clean up.
|
3. **No skip stop** — always `bsk session stop <id>`; never assume idle timeout will clean up.
|
||||||
4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
|
4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
|
||||||
5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics or conditional surfaces help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual.
|
5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual.
|
||||||
6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
|
6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|||||||
@@ -497,21 +497,4 @@ mod tests {
|
|||||||
.contains("CSS selector")
|
.contains("CSS selector")
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn invalid_selector_explains_css_contract_in_json() {
|
|
||||||
let cli = CliError::from_rpc(RpcError {
|
|
||||||
code: ErrorCode::InvalidParams,
|
|
||||||
message: "selector is not valid CSS: text=submit".into(),
|
|
||||||
data: Some(serde_json::json!({ "reason": "invalid_selector" })),
|
|
||||||
});
|
|
||||||
let exit = cli.exit_code();
|
|
||||||
let hint = hint_for(&cli, render_info_for(&cli).as_ref());
|
|
||||||
let json = json_error_string(&cli, exit, hint);
|
|
||||||
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
|
|
||||||
let hint = parsed.get("hint").and_then(|v| v.as_str()).unwrap();
|
|
||||||
assert!(hint.contains("CSS selector"));
|
|
||||||
assert!(hint.contains("text="));
|
|
||||||
assert!(hint.contains("@eN"));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -33,6 +33,12 @@ pub struct ObserveArgs {
|
|||||||
/// Include conditional surface probe diagnostics in JSON output.
|
/// Include conditional surface probe diagnostics in JSON output.
|
||||||
#[arg(long = "debug-surfaces")]
|
#[arg(long = "debug-surfaces")]
|
||||||
pub debug_surfaces: bool,
|
pub debug_surfaces: bool,
|
||||||
|
|
||||||
|
/// Actively hover page controls to reveal hover-only menus and tooltips.
|
||||||
|
/// Costs seconds of wall clock and touches the live page, so it is off
|
||||||
|
/// unless a static observation looks like it is missing hover content.
|
||||||
|
#[arg(long = "probe-hover")]
|
||||||
|
pub probe_hover: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn dispatch(args: ObserveArgs, format: Format) -> Result<(), CliError> {
|
pub fn dispatch(args: ObserveArgs, format: Format) -> Result<(), CliError> {
|
||||||
@@ -47,6 +53,7 @@ fn run(sock: PathBuf, args: ObserveArgs, format: Format) -> Result<(), CliError>
|
|||||||
max_depth: args.max_depth,
|
max_depth: args.max_depth,
|
||||||
max_tokens: args.max_tokens,
|
max_tokens: args.max_tokens,
|
||||||
debug_surfaces: args.debug_surfaces,
|
debug_surfaces: args.debug_surfaces,
|
||||||
|
probe_hover: args.probe_hover,
|
||||||
};
|
};
|
||||||
let reply: ObserveResult = call(sock, params)?;
|
let reply: ObserveResult = call(sock, params)?;
|
||||||
match format {
|
match format {
|
||||||
|
|||||||
@@ -41,7 +41,6 @@ pub mod reason {
|
|||||||
pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope";
|
pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope";
|
||||||
pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible";
|
pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible";
|
||||||
pub const REF_NOT_FOUND: &str = "ref_not_found";
|
pub const REF_NOT_FOUND: &str = "ref_not_found";
|
||||||
pub const INVALID_SELECTOR: &str = "invalid_selector";
|
|
||||||
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
|
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
|
||||||
pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
|
pub const TARGET_NOT_FILLABLE: &str = "target_not_fillable";
|
||||||
pub const TARGET_NOT_SELECT: &str = "target_not_select";
|
pub const TARGET_NOT_SELECT: &str = "target_not_select";
|
||||||
@@ -222,13 +221,6 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
|||||||
hint: Some("verify the CSS selector or wait for the element to appear before retrying"),
|
hint: Some("verify the CSS selector or wait for the element to appear before retrying"),
|
||||||
exit_code: base.exit_code,
|
exit_code: base.exit_code,
|
||||||
},
|
},
|
||||||
(ErrorCode::InvalidParams, reason::INVALID_SELECTOR) => RenderInfo {
|
|
||||||
summary: "selector is not valid CSS",
|
|
||||||
hint: Some(
|
|
||||||
"use a standard CSS selector or a fresh @eN ref; Playwright text= selectors and XPath are not supported",
|
|
||||||
),
|
|
||||||
exit_code: base.exit_code,
|
|
||||||
},
|
|
||||||
(ErrorCode::InvalidParams, reason::TARGET_NOT_FILLABLE) => RenderInfo {
|
(ErrorCode::InvalidParams, reason::TARGET_NOT_FILLABLE) => RenderInfo {
|
||||||
summary: "target element is not fillable",
|
summary: "target element is not fillable",
|
||||||
hint: Some(
|
hint: Some(
|
||||||
|
|||||||
@@ -223,14 +223,6 @@ impl BrowserRegistry {
|
|||||||
"browser reconnect: replacing previous registration"
|
"browser reconnect: replacing previous registration"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
tracing::info!(
|
|
||||||
diagnostic = true,
|
|
||||||
event = "browser.registry.insert",
|
|
||||||
id = %client.id,
|
|
||||||
generation = client.generation,
|
|
||||||
previous_generation = guard.get(&client.id).map(|previous| previous.generation),
|
|
||||||
"diagnostic browser registry mutation"
|
|
||||||
);
|
|
||||||
guard.insert(client.id.clone(), client);
|
guard.insert(client.id.clone(), client);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -247,31 +239,16 @@ impl BrowserRegistry {
|
|||||||
/// took over the same `id` while the old socket was tearing down
|
/// took over the same `id` while the old socket was tearing down
|
||||||
/// (review M4/M5 round 2 Important #1). Returns the removed entry
|
/// (review M4/M5 round 2 Important #1). Returns the removed entry
|
||||||
/// when a removal actually happened.
|
/// when a removal actually happened.
|
||||||
#[track_caller]
|
|
||||||
pub fn remove_if_generation_matches(
|
pub fn remove_if_generation_matches(
|
||||||
&self,
|
&self,
|
||||||
id: &BrowserId,
|
id: &BrowserId,
|
||||||
generation: u64,
|
generation: u64,
|
||||||
) -> Option<std::sync::Arc<BrowserClient>> {
|
) -> Option<std::sync::Arc<BrowserClient>> {
|
||||||
let caller = std::panic::Location::caller();
|
|
||||||
let mut guard = self.inner.lock().expect("browser registry poisoned");
|
let mut guard = self.inner.lock().expect("browser registry poisoned");
|
||||||
let current_generation = guard.get(id).map(|current| current.generation);
|
match guard.get(id) {
|
||||||
let removed = match guard.get(id) {
|
|
||||||
Some(current) if current.generation == generation => guard.remove(id),
|
Some(current) if current.generation == generation => guard.remove(id),
|
||||||
_ => None,
|
_ => None,
|
||||||
};
|
}
|
||||||
tracing::info!(
|
|
||||||
diagnostic = true,
|
|
||||||
event = "browser.registry.remove_if_generation_matches",
|
|
||||||
id = %id,
|
|
||||||
requested_generation = generation,
|
|
||||||
?current_generation,
|
|
||||||
removed = removed.is_some(),
|
|
||||||
caller_file = caller.file(),
|
|
||||||
caller_line = caller.line(),
|
|
||||||
"diagnostic browser registry mutation"
|
|
||||||
);
|
|
||||||
removed
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn get(&self, id: &BrowserId) -> Option<std::sync::Arc<BrowserClient>> {
|
pub fn get(&self, id: &BrowserId) -> Option<std::sync::Arc<BrowserClient>> {
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ use serde::{Deserialize, Serialize};
|
|||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use tokio::sync::Notify;
|
use tokio::sync::Notify;
|
||||||
use tokio::task::JoinHandle;
|
use tokio::task::JoinHandle;
|
||||||
use tracing::{debug, info, warn};
|
use tracing::{debug, warn};
|
||||||
|
|
||||||
use super::abort::AbortRegistry;
|
use super::abort::AbortRegistry;
|
||||||
use super::queue::{DEFAULT_TOOL_TIMEOUT, DispatchError};
|
use super::queue::{DEFAULT_TOOL_TIMEOUT, DispatchError};
|
||||||
@@ -197,22 +197,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
|||||||
let status = status.clone();
|
let status = status.clone();
|
||||||
let state = Arc::clone(&state);
|
let state = Arc::clone(&state);
|
||||||
Box::pin(async move {
|
Box::pin(async move {
|
||||||
let method_name = format!("{method:?}");
|
match method {
|
||||||
let diagnostic_rpc_id = rpc_id.clone();
|
|
||||||
let session_id = params
|
|
||||||
.get("session_id")
|
|
||||||
.and_then(Value::as_str)
|
|
||||||
.map(str::to_owned);
|
|
||||||
let started = Instant::now();
|
|
||||||
info!(
|
|
||||||
diagnostic = true,
|
|
||||||
event = "ipc.rpc.started",
|
|
||||||
rpc_id = %diagnostic_rpc_id,
|
|
||||||
method = %method_name,
|
|
||||||
session = session_id.as_deref().unwrap_or(""),
|
|
||||||
"diagnostic IPC RPC"
|
|
||||||
);
|
|
||||||
let body = match method {
|
|
||||||
Method::SystemPing => {
|
Method::SystemPing => {
|
||||||
let result = PingResult { pong: true };
|
let result = PingResult { pong: true };
|
||||||
ResponseBody::Ok(serde_json::to_value(result).unwrap_or(Value::Null))
|
ResponseBody::Ok(serde_json::to_value(result).unwrap_or(Value::Null))
|
||||||
@@ -276,23 +261,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
|||||||
message: format!("method not implemented yet: {other:?}"),
|
message: format!("method not implemented yet: {other:?}"),
|
||||||
data: None,
|
data: None,
|
||||||
}),
|
}),
|
||||||
};
|
}
|
||||||
let (outcome, error_code) = match &body {
|
|
||||||
ResponseBody::Ok(_) => ("ok", String::new()),
|
|
||||||
ResponseBody::Err(error) => ("error", format!("{:?}", error.code)),
|
|
||||||
};
|
|
||||||
info!(
|
|
||||||
diagnostic = true,
|
|
||||||
event = "ipc.rpc.completed",
|
|
||||||
rpc_id = %diagnostic_rpc_id,
|
|
||||||
method = %method_name,
|
|
||||||
session = session_id.as_deref().unwrap_or(""),
|
|
||||||
outcome,
|
|
||||||
error_code = %error_code,
|
|
||||||
elapsed_ms = started.elapsed().as_millis() as u64,
|
|
||||||
"diagnostic IPC RPC"
|
|
||||||
);
|
|
||||||
body
|
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -496,14 +465,8 @@ fn handle_cancel(state: &Arc<DaemonState>, params: Value) -> ResponseBody {
|
|||||||
};
|
};
|
||||||
let local = state.abort_registry.cancel(¶ms.rpc_id);
|
let local = state.abort_registry.cancel(¶ms.rpc_id);
|
||||||
let mut cancelled = local;
|
let mut cancelled = local;
|
||||||
let mut surface = if local { "abort_registry" } else { "none" };
|
|
||||||
if !local && let Some(snap) = state.tool_inflight.cancel(¶ms.rpc_id) {
|
if !local && let Some(snap) = state.tool_inflight.cancel(¶ms.rpc_id) {
|
||||||
cancelled = true;
|
cancelled = true;
|
||||||
surface = if snap.ws_rpc_id.is_some() {
|
|
||||||
"tool_inflight_forwarded"
|
|
||||||
} else {
|
|
||||||
"tool_inflight_queued"
|
|
||||||
};
|
|
||||||
// Forward the WS-side cancel only when the worker has
|
// Forward the WS-side cancel only when the worker has
|
||||||
// already promoted the entry to "forwarded"; queued entries
|
// already promoted the entry to "forwarded"; queued entries
|
||||||
// short-circuit on their own pre-flight without ever
|
// short-circuit on their own pre-flight without ever
|
||||||
@@ -521,14 +484,6 @@ fn handle_cancel(state: &Arc<DaemonState>, params: Value) -> ResponseBody {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
info!(
|
|
||||||
diagnostic = true,
|
|
||||||
event = "ipc.cancel.resolved",
|
|
||||||
target_rpc_id = %params.rpc_id,
|
|
||||||
cancelled,
|
|
||||||
surface,
|
|
||||||
"diagnostic IPC cancellation"
|
|
||||||
);
|
|
||||||
ResponseBody::Ok(serde_json::to_value(CancelResult { cancelled }).unwrap_or(Value::Null))
|
ResponseBody::Ok(serde_json::to_value(CancelResult { cancelled }).unwrap_or(Value::Null))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ use bsk_protocol::{ErrorCode, Frame, Method, RequestFrame, ResponseBody, RpcErro
|
|||||||
use rand::Rng;
|
use rand::Rng;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use tokio::sync::{mpsc, oneshot};
|
use tokio::sync::{mpsc, oneshot};
|
||||||
use tracing::{debug, info, warn};
|
use tracing::{debug, warn};
|
||||||
|
|
||||||
use super::abort::AbortToken;
|
use super::abort::AbortToken;
|
||||||
use super::browsers::BrowserRegistry;
|
use super::browsers::BrowserRegistry;
|
||||||
@@ -515,16 +515,6 @@ async fn forward_one(
|
|||||||
sessions: &Arc<SessionRegistry>,
|
sessions: &Arc<SessionRegistry>,
|
||||||
job: &ToolJob,
|
job: &ToolJob,
|
||||||
) -> Result<Value, RpcError> {
|
) -> Result<Value, RpcError> {
|
||||||
let started = std::time::Instant::now();
|
|
||||||
info!(
|
|
||||||
diagnostic = true,
|
|
||||||
event = "tool.dispatch.started",
|
|
||||||
session = %sid,
|
|
||||||
method = ?job.method,
|
|
||||||
has_inflight = job.inflight.is_some(),
|
|
||||||
has_lifecycle_cancel = job.lifecycle_cancel.is_some(),
|
|
||||||
"diagnostic tool dispatch"
|
|
||||||
);
|
|
||||||
// Pre-flight: a cancel that landed while this job was still in
|
// Pre-flight: a cancel that landed while this job was still in
|
||||||
// the per-session channel must short-circuit before any session
|
// the per-session channel must short-circuit before any session
|
||||||
// / browser resolution work, and before any WS frame leaves the
|
// / browser resolution work, and before any WS frame leaves the
|
||||||
@@ -545,13 +535,6 @@ async fn forward_one(
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
let Some(session) = sessions.get(sid) else {
|
let Some(session) = sessions.get(sid) else {
|
||||||
warn!(
|
|
||||||
diagnostic = true,
|
|
||||||
event = "tool.dispatch.session_missing",
|
|
||||||
session = %sid,
|
|
||||||
method = ?job.method,
|
|
||||||
"diagnostic tool dispatch"
|
|
||||||
);
|
|
||||||
return Err(RpcError {
|
return Err(RpcError {
|
||||||
code: ErrorCode::NotFound,
|
code: ErrorCode::NotFound,
|
||||||
message: format!("session {sid} no longer exists"),
|
message: format!("session {sid} no longer exists"),
|
||||||
@@ -559,14 +542,6 @@ async fn forward_one(
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
let Some(client) = browsers.get(&session.browser_id) else {
|
let Some(client) = browsers.get(&session.browser_id) else {
|
||||||
warn!(
|
|
||||||
diagnostic = true,
|
|
||||||
event = "tool.dispatch.browser_missing",
|
|
||||||
session = %sid,
|
|
||||||
method = ?job.method,
|
|
||||||
browser = %session.browser_id,
|
|
||||||
"diagnostic tool dispatch"
|
|
||||||
);
|
|
||||||
return Err(RpcError {
|
return Err(RpcError {
|
||||||
code: ErrorCode::NotFound,
|
code: ErrorCode::NotFound,
|
||||||
message: "owning browser is no longer connected".into(),
|
message: "owning browser is no longer connected".into(),
|
||||||
@@ -583,16 +558,6 @@ async fn forward_one(
|
|||||||
method: job.method.clone(),
|
method: job.method.clone(),
|
||||||
params: Some(job.params.clone()),
|
params: Some(job.params.clone()),
|
||||||
});
|
});
|
||||||
info!(
|
|
||||||
diagnostic = true,
|
|
||||||
event = "tool.dispatch.forwarding",
|
|
||||||
session = %sid,
|
|
||||||
method = ?job.method,
|
|
||||||
browser = %session.browser_id,
|
|
||||||
browser_generation = client.generation,
|
|
||||||
ws_rpc_id = %rpc_id,
|
|
||||||
"diagnostic tool dispatch"
|
|
||||||
);
|
|
||||||
// Promote the inflight entry to "forwarded" AND push the WS
|
// Promote the inflight entry to "forwarded" AND push the WS
|
||||||
// request frame to the sink inside the same critical section
|
// request frame to the sink inside the same critical section
|
||||||
// (review round 2 C1). The closure runs while the entry's inner
|
// (review round 2 C1). The closure runs while the entry's inner
|
||||||
@@ -752,18 +717,6 @@ async fn forward_one(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
info!(
|
|
||||||
diagnostic = true,
|
|
||||||
event = "tool.dispatch.response",
|
|
||||||
session = %sid,
|
|
||||||
method = ?job.method,
|
|
||||||
browser = %session.browser_id,
|
|
||||||
browser_generation = client.generation,
|
|
||||||
ws_rpc_id = %rpc_id,
|
|
||||||
elapsed_ms = started.elapsed().as_millis() as u64,
|
|
||||||
response_is_error = matches!(&response.body, ResponseBody::Err(_)),
|
|
||||||
"diagnostic tool dispatch"
|
|
||||||
);
|
|
||||||
match response.body {
|
match response.body {
|
||||||
ResponseBody::Ok(v) => Ok(v),
|
ResponseBody::Ok(v) => Ok(v),
|
||||||
ResponseBody::Err(err) => Err(err),
|
ResponseBody::Err(err) => Err(err),
|
||||||
|
|||||||
@@ -105,22 +105,12 @@ impl SessionRegistry {
|
|||||||
|
|
||||||
pub fn insert(&self, session: Session) {
|
pub fn insert(&self, session: Session) {
|
||||||
let session_id = session.id.clone();
|
let session_id = session.id.clone();
|
||||||
let browser_id = session.browser_id.clone();
|
|
||||||
let agent_window_id = session.agent_window_id;
|
|
||||||
let mut sessions = self.inner.lock().expect("session registry poisoned");
|
let mut sessions = self.inner.lock().expect("session registry poisoned");
|
||||||
sessions.insert(session_id.clone(), session);
|
sessions.insert(session_id.clone(), session);
|
||||||
self.last_activity
|
self.last_activity
|
||||||
.lock()
|
.lock()
|
||||||
.expect("session activity registry poisoned")
|
.expect("session activity registry poisoned")
|
||||||
.insert(session_id.clone(), Instant::now());
|
.insert(session_id, Instant::now());
|
||||||
tracing::info!(
|
|
||||||
diagnostic = true,
|
|
||||||
event = "session.registry.insert",
|
|
||||||
session = %session_id,
|
|
||||||
browser = %browser_id,
|
|
||||||
?agent_window_id,
|
|
||||||
"diagnostic session registry mutation"
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Reserve a fresh, collision-free [`SessionId`] under the registry
|
/// Reserve a fresh, collision-free [`SessionId`] under the registry
|
||||||
@@ -162,13 +152,6 @@ impl SessionRegistry {
|
|||||||
.lock()
|
.lock()
|
||||||
.expect("session activity registry poisoned")
|
.expect("session activity registry poisoned")
|
||||||
.insert(candidate.clone(), Instant::now());
|
.insert(candidate.clone(), Instant::now());
|
||||||
tracing::info!(
|
|
||||||
diagnostic = true,
|
|
||||||
event = "session.registry.reserve",
|
|
||||||
session = %candidate,
|
|
||||||
browser = %browser_id,
|
|
||||||
"diagnostic session registry mutation"
|
|
||||||
);
|
|
||||||
return Some(candidate);
|
return Some(candidate);
|
||||||
}
|
}
|
||||||
None
|
None
|
||||||
@@ -186,24 +169,13 @@ impl SessionRegistry {
|
|||||||
let mut guard = self.inner.lock().expect("session registry poisoned");
|
let mut guard = self.inner.lock().expect("session registry poisoned");
|
||||||
let session = guard.get_mut(session_id)?;
|
let session = guard.get_mut(session_id)?;
|
||||||
session.agent_window_id = agent_window_id;
|
session.agent_window_id = agent_window_id;
|
||||||
tracing::info!(
|
|
||||||
diagnostic = true,
|
|
||||||
event = "session.registry.commit",
|
|
||||||
session = %session_id,
|
|
||||||
browser = %session.browser_id,
|
|
||||||
?agent_window_id,
|
|
||||||
"diagnostic session registry mutation"
|
|
||||||
);
|
|
||||||
Some(session.clone())
|
Some(session.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Drop a placeholder reservation, used on extension error/timeout
|
/// Drop a placeholder reservation, used on extension error/timeout
|
||||||
/// paths so failed reservations do not accumulate.
|
/// paths so failed reservations do not accumulate.
|
||||||
#[track_caller]
|
|
||||||
pub fn cancel_reservation(&self, session_id: &SessionId) {
|
pub fn cancel_reservation(&self, session_id: &SessionId) {
|
||||||
let caller = std::panic::Location::caller();
|
self.inner
|
||||||
let removed = self
|
|
||||||
.inner
|
|
||||||
.lock()
|
.lock()
|
||||||
.expect("session registry poisoned")
|
.expect("session registry poisoned")
|
||||||
.remove(session_id);
|
.remove(session_id);
|
||||||
@@ -211,20 +183,9 @@ impl SessionRegistry {
|
|||||||
.lock()
|
.lock()
|
||||||
.expect("session activity registry poisoned")
|
.expect("session activity registry poisoned")
|
||||||
.remove(session_id);
|
.remove(session_id);
|
||||||
tracing::info!(
|
|
||||||
diagnostic = true,
|
|
||||||
event = "session.registry.cancel_reservation",
|
|
||||||
session = %session_id,
|
|
||||||
existed = removed.is_some(),
|
|
||||||
caller_file = caller.file(),
|
|
||||||
caller_line = caller.line(),
|
|
||||||
"diagnostic session registry mutation"
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[track_caller]
|
|
||||||
pub fn remove(&self, id: &SessionId) -> Option<Session> {
|
pub fn remove(&self, id: &SessionId) -> Option<Session> {
|
||||||
let caller = std::panic::Location::caller();
|
|
||||||
let removed = self
|
let removed = self
|
||||||
.inner
|
.inner
|
||||||
.lock()
|
.lock()
|
||||||
@@ -234,17 +195,6 @@ impl SessionRegistry {
|
|||||||
.lock()
|
.lock()
|
||||||
.expect("session activity registry poisoned")
|
.expect("session activity registry poisoned")
|
||||||
.remove(id);
|
.remove(id);
|
||||||
tracing::info!(
|
|
||||||
diagnostic = true,
|
|
||||||
event = "session.registry.remove",
|
|
||||||
session = %id,
|
|
||||||
existed = removed.is_some(),
|
|
||||||
browser = removed.as_ref().map(|session| session.browser_id.0.as_str()),
|
|
||||||
agent_window_id = removed.as_ref().and_then(|session| session.agent_window_id),
|
|
||||||
caller_file = caller.file(),
|
|
||||||
caller_line = caller.line(),
|
|
||||||
"diagnostic session registry mutation"
|
|
||||||
);
|
|
||||||
removed
|
removed
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -293,9 +243,7 @@ impl SessionRegistry {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Drop all sessions owned by `browser_id` (e.g. on disconnect).
|
/// Drop all sessions owned by `browser_id` (e.g. on disconnect).
|
||||||
#[track_caller]
|
|
||||||
pub fn purge_browser(&self, browser_id: &BrowserId) -> Vec<Session> {
|
pub fn purge_browser(&self, browser_id: &BrowserId) -> Vec<Session> {
|
||||||
let caller = std::panic::Location::caller();
|
|
||||||
let mut guard = self.inner.lock().expect("session registry poisoned");
|
let mut guard = self.inner.lock().expect("session registry poisoned");
|
||||||
let drained: Vec<Session> = guard
|
let drained: Vec<Session> = guard
|
||||||
.values()
|
.values()
|
||||||
@@ -312,20 +260,6 @@ impl SessionRegistry {
|
|||||||
for session in &drained {
|
for session in &drained {
|
||||||
activity.remove(&session.id);
|
activity.remove(&session.id);
|
||||||
}
|
}
|
||||||
let session_ids: Vec<&str> = drained
|
|
||||||
.iter()
|
|
||||||
.map(|session| session.id.0.as_str())
|
|
||||||
.collect();
|
|
||||||
tracing::info!(
|
|
||||||
diagnostic = true,
|
|
||||||
event = "session.registry.purge_browser",
|
|
||||||
browser = %browser_id,
|
|
||||||
sessions = ?session_ids,
|
|
||||||
count = drained.len(),
|
|
||||||
caller_file = caller.file(),
|
|
||||||
caller_line = caller.line(),
|
|
||||||
"diagnostic session registry mutation"
|
|
||||||
);
|
|
||||||
drained
|
drained
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -40,8 +40,6 @@ pub(crate) const DAEMONIZED_ENV: &str = "BSK_DAEMONIZED";
|
|||||||
/// daemon lock, IPC socket, and WS port — before taking over.
|
/// daemon lock, IPC socket, and WS port — before taking over.
|
||||||
pub(crate) const DAEMON_REPLACEMENT_WAIT_ENV: &str = "BSK_DAEMON_REPLACES_PID";
|
pub(crate) const DAEMON_REPLACEMENT_WAIT_ENV: &str = "BSK_DAEMON_REPLACES_PID";
|
||||||
|
|
||||||
const LUMI_DIAGNOSTIC_BUILD: &str = "lumi-session-loss-v2";
|
|
||||||
|
|
||||||
/// Concrete daemon configuration resolved from CLI flags / defaults.
|
/// Concrete daemon configuration resolved from CLI flags / defaults.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct DaemonConfig {
|
pub struct DaemonConfig {
|
||||||
@@ -253,7 +251,6 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
|
|||||||
pid = info.pid,
|
pid = info.pid,
|
||||||
ws_port = info.ws_port,
|
ws_port = info.ws_port,
|
||||||
sock = %sock_path.display(),
|
sock = %sock_path.display(),
|
||||||
diagnostic_build = LUMI_DIAGNOSTIC_BUILD,
|
|
||||||
"daemon ready"
|
"daemon ready"
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -591,9 +588,7 @@ pub(crate) fn spawn_update_check_task(
|
|||||||
// The session gate is read after the fetch, as late
|
// The session gate is read after the fetch, as late
|
||||||
// as possible before the binary gets replaced.
|
// as possible before the binary gets replaced.
|
||||||
let active_sessions = state.sessions.len();
|
let active_sessions = state.sessions.len();
|
||||||
// Keep this local diagnostic build in place until a
|
let auto_update = update::auto_update_enabled() && exe_path.is_some();
|
||||||
// reproduction has uploaded its extension-side evidence.
|
|
||||||
let auto_update = false;
|
|
||||||
update::auto_update_step(
|
update::auto_update_step(
|
||||||
candidate.as_ref(),
|
candidate.as_ref(),
|
||||||
auto_update,
|
auto_update,
|
||||||
@@ -737,11 +732,7 @@ fn init_tracing() -> Option<tracing_appender::non_blocking::WorkerGuard> {
|
|||||||
use tracing_subscriber::prelude::*;
|
use tracing_subscriber::prelude::*;
|
||||||
|
|
||||||
let log_dir = paths::log_dir().ok();
|
let log_dir = paths::log_dir().ok();
|
||||||
let env_filter = EnvFilter::try_from_default_env()
|
let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
|
||||||
.unwrap_or_else(|_| EnvFilter::new("info"))
|
|
||||||
// Lumi commonly runs with RUST_LOG=warn. This local build must retain
|
|
||||||
// BrowserSkill's INFO diagnostics even under that host-wide setting.
|
|
||||||
.add_directive("bsk=info".parse().expect("valid diagnostic log directive"));
|
|
||||||
|
|
||||||
let (file_layer, guard) = if let Some(dir) = log_dir {
|
let (file_layer, guard) = if let Some(dir) = log_dir {
|
||||||
let appender = tracing_appender::rolling::daily(&dir, "daemon.log");
|
let appender = tracing_appender::rolling::daily(&dir, "daemon.log");
|
||||||
|
|||||||
@@ -233,14 +233,6 @@ async fn drive_connection(
|
|||||||
.ok_or_else(|| anyhow!("handshake missing params"))?;
|
.ok_or_else(|| anyhow!("handshake missing params"))?;
|
||||||
let params: HandshakeParams = serde_json::from_value(params_raw)
|
let params: HandshakeParams = serde_json::from_value(params_raw)
|
||||||
.map_err(|err| anyhow!("invalid HandshakeParams: {err}"))?;
|
.map_err(|err| anyhow!("invalid HandshakeParams: {err}"))?;
|
||||||
info!(
|
|
||||||
diagnostic = true,
|
|
||||||
event = "browser.handshake.received",
|
|
||||||
browser = %params.instance_id,
|
|
||||||
extension_version = %params.version,
|
|
||||||
extension_protocol = %params.protocol_version,
|
|
||||||
"diagnostic browser handshake"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Version compatibility (design §10, M10.4): protocol_version only.
|
// Version compatibility (design §10, M10.4): protocol_version only.
|
||||||
let our_app_version: Version = env!("CARGO_PKG_VERSION")
|
let our_app_version: Version = env!("CARGO_PKG_VERSION")
|
||||||
@@ -358,7 +350,7 @@ async fn drive_connection(
|
|||||||
// Pump loop: outbound frames from sink → ws; inbound from ws → resolve.
|
// Pump loop: outbound frames from sink → ws; inbound from ws → resolve.
|
||||||
let pump_state = Arc::clone(&state);
|
let pump_state = Arc::clone(&state);
|
||||||
let pump_browser = Arc::clone(&client);
|
let pump_browser = Arc::clone(&client);
|
||||||
let result_outcome: anyhow::Result<String> = async {
|
let result_outcome: anyhow::Result<()> = async {
|
||||||
loop {
|
loop {
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
outbound = rx.recv() => {
|
outbound = rx.recv() => {
|
||||||
@@ -367,7 +359,7 @@ async fn drive_connection(
|
|||||||
let json = serde_json::to_string(&frame)?;
|
let json = serde_json::to_string(&frame)?;
|
||||||
writer.send(Message::Text(json)).await?;
|
writer.send(Message::Text(json)).await?;
|
||||||
}
|
}
|
||||||
None => break Ok("outbound_channel_closed".to_string()),
|
None => break,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
msg = reader.next() => {
|
msg = reader.next() => {
|
||||||
@@ -389,37 +381,18 @@ async fn drive_connection(
|
|||||||
pump_browser.touch();
|
pump_browser.touch();
|
||||||
}
|
}
|
||||||
Some(Ok(Message::Frame(_))) => {}
|
Some(Ok(Message::Frame(_))) => {}
|
||||||
Some(Ok(Message::Close(frame))) => {
|
Some(Ok(Message::Close(_))) | None => break,
|
||||||
let reason = frame
|
|
||||||
.map(|frame| format!("peer_close code={:?} reason={}", frame.code, frame.reason))
|
|
||||||
.unwrap_or_else(|| "peer_close_without_frame".to_string());
|
|
||||||
break Ok(reason);
|
|
||||||
}
|
|
||||||
None => break Ok("peer_eof".to_string()),
|
|
||||||
Some(Err(err)) => {
|
Some(Err(err)) => {
|
||||||
warn!(%err, "ws read error");
|
warn!(%err, "ws read error");
|
||||||
break Ok(format!("read_error: {err}"));
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
.await;
|
.await;
|
||||||
let pump_exit_reason = match &result_outcome {
|
|
||||||
Ok(reason) => reason.clone(),
|
|
||||||
Err(err) => format!("pump_error: {err:#}"),
|
|
||||||
};
|
|
||||||
info!(
|
|
||||||
diagnostic = true,
|
|
||||||
event = "browser.websocket.pump_exit",
|
|
||||||
id = %browser_id,
|
|
||||||
generation,
|
|
||||||
reason = %pump_exit_reason,
|
|
||||||
last_seen_idle_ms = client.idle_for().as_millis() as u64,
|
|
||||||
session_ids = ?state.sessions.ids_for_browser(&browser_id),
|
|
||||||
"diagnostic browser websocket exit"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Cleanup: drop browser + purge its sessions, but only if the
|
// Cleanup: drop browser + purge its sessions, but only if the
|
||||||
// registry still holds *this* generation. If a reconnect already
|
// registry still holds *this* generation. If a reconnect already
|
||||||
@@ -443,7 +416,7 @@ async fn drive_connection(
|
|||||||
"browser disconnected; newer reconnect already owns this id, leaving sessions intact"
|
"browser disconnected; newer reconnect already owns this id, leaving sessions intact"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
result_outcome.map(|_| ())
|
result_outcome
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn handle_inbound_text(state: &Arc<DaemonState>, client: &Arc<BrowserClient>, text: &str) {
|
async fn handle_inbound_text(state: &Arc<DaemonState>, client: &Arc<BrowserClient>, text: &str) {
|
||||||
@@ -469,38 +442,6 @@ async fn handle_inbound_text(state: &Arc<DaemonState>, client: &Arc<BrowserClien
|
|||||||
client.mark_heartbeat_seen();
|
client.mark_heartbeat_seen();
|
||||||
debug!(id = %client.id, "heartbeat");
|
debug!(id = %client.id, "heartbeat");
|
||||||
}
|
}
|
||||||
bsk_protocol::EventKind::BrowserDiagnostic => {
|
|
||||||
let entries = ev.payload.get("entries").and_then(|value| value.as_array());
|
|
||||||
info!(
|
|
||||||
diagnostic = true,
|
|
||||||
event = "extension.diagnostic.batch",
|
|
||||||
browser = %client.id,
|
|
||||||
generation = client.generation,
|
|
||||||
worker_boot_id = ev.payload.get("worker_boot_id").and_then(|value| value.as_str()).unwrap_or("unknown"),
|
|
||||||
count = entries.map_or(0, Vec::len),
|
|
||||||
"received extension diagnostic batch"
|
|
||||||
);
|
|
||||||
if let Some(entries) = entries {
|
|
||||||
for entry in entries {
|
|
||||||
info!(
|
|
||||||
diagnostic = true,
|
|
||||||
event = "extension.diagnostic.entry",
|
|
||||||
browser = %client.id,
|
|
||||||
generation = client.generation,
|
|
||||||
entry = %entry,
|
|
||||||
"extension diagnostic"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
warn!(
|
|
||||||
diagnostic = true,
|
|
||||||
event = "extension.diagnostic.invalid",
|
|
||||||
browser = %client.id,
|
|
||||||
payload = %ev.payload,
|
|
||||||
"extension diagnostic payload missing entries"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
bsk_protocol::EventKind::SessionWindowClosed => {
|
bsk_protocol::EventKind::SessionWindowClosed => {
|
||||||
handle_session_window_closed(state, &client.id, &ev.payload);
|
handle_session_window_closed(state, &client.id, &ev.payload);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -478,6 +478,7 @@ async fn observe_returns_semantic_text_and_ref_count() {
|
|||||||
tab_id: 13,
|
tab_id: 13,
|
||||||
truncated: false,
|
truncated: false,
|
||||||
dialogs: vec![],
|
dialogs: vec![],
|
||||||
|
hover_probe: None,
|
||||||
debug: None,
|
debug: None,
|
||||||
})
|
})
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
@@ -494,6 +495,7 @@ async fn observe_returns_semantic_text_and_ref_count() {
|
|||||||
max_depth: None,
|
max_depth: None,
|
||||||
max_tokens: None,
|
max_tokens: None,
|
||||||
debug_surfaces: false,
|
debug_surfaces: false,
|
||||||
|
probe_hover: false,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
"title": "ObserveParams",
|
"title": "ObserveParams",
|
||||||
"description": "Parameters for `tool.observe` — produce a semantic VOM observation for one tab plus a fresh `@e<N>` ref-store on the extension side.\n\nUnlike `tool.snapshot`, this path may run bounded perception probes such as hover-surface discovery. It must not submit input, click, navigate, or otherwise commit page state.",
|
"description": "Parameters for `tool.observe` — produce a semantic VOM observation for one tab plus a fresh `@e<N>` ref-store on the extension side.\n\nUnlike `tool.snapshot`, this path may run bounded perception probes such as hover-surface discovery, but only when `probe_hover` asks for them. It must not submit input, click, navigate, or otherwise commit page state.",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
"required": [
|
||||||
"session_id"
|
"session_id"
|
||||||
@@ -30,6 +30,11 @@
|
|||||||
"format": "uint32",
|
"format": "uint32",
|
||||||
"minimum": 0.0
|
"minimum": 0.0
|
||||||
},
|
},
|
||||||
|
"probe_hover": {
|
||||||
|
"description": "Opt in to active hover probing: the extension moves the real cursor over a bounded set of controls to discover hover-only menus and tooltips.\n\nOff by default. Probing costs seconds of wall clock, mutates the live page, and pays off only on pages whose content is genuinely hidden behind hover. Request it when a static observation looks like it is missing hover-revealed structure.",
|
||||||
|
"default": false,
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
"session_id": {
|
"session_id": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -24,6 +24,17 @@
|
|||||||
"$ref": "#/definitions/JavaScriptDialogInfo"
|
"$ref": "#/definitions/JavaScriptDialogInfo"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"hover_probe": {
|
||||||
|
"description": "Present only when `probe_hover` was requested and probing actually ran.",
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"$ref": "#/definitions/HoverProbeReport"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
"ref_count": {
|
"ref_count": {
|
||||||
"description": "Number of `@e<N>` refs registered for this session by this observation. Equivalent to the size of the new ref-store map.",
|
"description": "Number of `@e<N>` refs registered for this session by this observation. Equivalent to the size of the new ref-store map.",
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
@@ -46,6 +57,24 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"definitions": {
|
"definitions": {
|
||||||
|
"HoverProbeReport": {
|
||||||
|
"description": "Reports what active hover probing did during an observation, so the agent can judge how far the live page may have drifted from the returned text.",
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"performed",
|
||||||
|
"revealed_content"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"performed": {
|
||||||
|
"description": "The real cursor was moved over page content during this observation.",
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"revealed_content": {
|
||||||
|
"description": "Hovering surfaced content the static tree does not contain, so the page reacts to the cursor and may not have fully reverted.",
|
||||||
|
"type": "boolean"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"JavaScriptDialogHandledAction": {
|
"JavaScriptDialogHandledAction": {
|
||||||
"description": "How the extension resolved the dialog so CDP could continue.",
|
"description": "How the extension resolved the dialog so CDP could continue.",
|
||||||
"type": "string",
|
"type": "string",
|
||||||
|
|||||||
@@ -58,10 +58,6 @@ pub enum EventKind {
|
|||||||
BrowserDisconnected,
|
BrowserDisconnected,
|
||||||
#[serde(rename = "browser.connected")]
|
#[serde(rename = "browser.connected")]
|
||||||
BrowserConnected,
|
BrowserConnected,
|
||||||
/// Structured diagnostics emitted by locally instrumented extension
|
|
||||||
/// builds. Production peers may ignore this event safely.
|
|
||||||
#[serde(rename = "browser.diagnostic")]
|
|
||||||
BrowserDiagnostic,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq)]
|
#[derive(Debug, Clone, PartialEq)]
|
||||||
@@ -260,27 +256,6 @@ mod tests {
|
|||||||
assert_eq!(frame.event, EventKind::SystemHeartbeat);
|
assert_eq!(frame.event, EventKind::SystemHeartbeat);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn browser_diagnostic_event_frame_round_trips_from_extension_shape() {
|
|
||||||
let wire = serde_json::json!({
|
|
||||||
"event": "browser.diagnostic",
|
|
||||||
"payload": {
|
|
||||||
"schema_version": 1,
|
|
||||||
"worker_boot_id": "worker-1",
|
|
||||||
"entries": [{ "event": "extension.worker.started" }]
|
|
||||||
}
|
|
||||||
});
|
|
||||||
let frame: EventFrame = serde_json::from_value(wire).unwrap();
|
|
||||||
assert_eq!(frame.event, EventKind::BrowserDiagnostic);
|
|
||||||
assert_eq!(
|
|
||||||
frame
|
|
||||||
.payload
|
|
||||||
.get("worker_boot_id")
|
|
||||||
.and_then(|value| value.as_str()),
|
|
||||||
Some("worker-1")
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn session_user_interrupt_serialises_as_snake_case() {
|
fn session_user_interrupt_serialises_as_snake_case() {
|
||||||
let v = serde_json::to_value(EventKind::SessionUserInterrupt).unwrap();
|
let v = serde_json::to_value(EventKind::SessionUserInterrupt).unwrap();
|
||||||
|
|||||||
@@ -58,8 +58,9 @@ pub struct SnapshotResult {
|
|||||||
/// for one tab plus a fresh `@e<N>` ref-store on the extension side.
|
/// for one tab plus a fresh `@e<N>` ref-store on the extension side.
|
||||||
///
|
///
|
||||||
/// Unlike `tool.snapshot`, this path may run bounded perception probes
|
/// Unlike `tool.snapshot`, this path may run bounded perception probes
|
||||||
/// such as hover-surface discovery. It must not submit input, click,
|
/// such as hover-surface discovery, but only when `probe_hover` asks for
|
||||||
/// navigate, or otherwise commit page state.
|
/// them. It must not submit input, click, navigate, or otherwise commit
|
||||||
|
/// page state.
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
pub struct ObserveParams {
|
pub struct ObserveParams {
|
||||||
pub session_id: String,
|
pub session_id: String,
|
||||||
@@ -77,6 +78,16 @@ pub struct ObserveParams {
|
|||||||
/// Include implementation diagnostics for conditional surface probes.
|
/// Include implementation diagnostics for conditional surface probes.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub debug_surfaces: bool,
|
pub debug_surfaces: bool,
|
||||||
|
/// Opt in to active hover probing: the extension moves the real cursor
|
||||||
|
/// over a bounded set of controls to discover hover-only menus and
|
||||||
|
/// tooltips.
|
||||||
|
///
|
||||||
|
/// Off by default. Probing costs seconds of wall clock, mutates the live
|
||||||
|
/// page, and pays off only on pages whose content is genuinely hidden
|
||||||
|
/// behind hover. Request it when a static observation looks like it is
|
||||||
|
/// missing hover-revealed structure.
|
||||||
|
#[serde(default)]
|
||||||
|
pub probe_hover: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
@@ -102,6 +113,17 @@ pub struct ObserveDebug {
|
|||||||
pub surface_probes: Vec<SurfaceProbeDebug>,
|
pub surface_probes: Vec<SurfaceProbeDebug>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Reports what active hover probing did during an observation, so the agent
|
||||||
|
/// can judge how far the live page may have drifted from the returned text.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct HoverProbeReport {
|
||||||
|
/// The real cursor was moved over page content during this observation.
|
||||||
|
pub performed: bool,
|
||||||
|
/// Hovering surfaced content the static tree does not contain, so the page
|
||||||
|
/// reacts to the cursor and may not have fully reverted.
|
||||||
|
pub revealed_content: bool,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
pub struct ObserveResult {
|
pub struct ObserveResult {
|
||||||
/// Semantic VOM observation text. Refs are rendered as `@e<N>` so
|
/// Semantic VOM observation text. Refs are rendered as `@e<N>` so
|
||||||
@@ -118,6 +140,9 @@ pub struct ObserveResult {
|
|||||||
pub truncated: bool,
|
pub truncated: bool,
|
||||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||||
|
/// Present only when `probe_hover` was requested and probing actually ran.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub hover_probe: Option<HoverProbeReport>,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub debug: Option<ObserveDebug>,
|
pub debug: Option<ObserveDebug>,
|
||||||
}
|
}
|
||||||
@@ -308,6 +333,7 @@ mod tests {
|
|||||||
tab_id: 42,
|
tab_id: 42,
|
||||||
truncated: false,
|
truncated: false,
|
||||||
dialogs: Vec::new(),
|
dialogs: Vec::new(),
|
||||||
|
hover_probe: None,
|
||||||
debug: None,
|
debug: None,
|
||||||
};
|
};
|
||||||
let v = serde_json::to_value(&r).unwrap();
|
let v = serde_json::to_value(&r).unwrap();
|
||||||
@@ -315,4 +341,30 @@ mod tests {
|
|||||||
let round: ObserveResult = serde_json::from_value(v).unwrap();
|
let round: ObserveResult = serde_json::from_value(v).unwrap();
|
||||||
assert_eq!(round, r);
|
assert_eq!(round, r);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn observe_params_default_to_no_hover_probing() {
|
||||||
|
let params: ObserveParams =
|
||||||
|
serde_json::from_value(serde_json::json!({ "session_id": "s1" })).unwrap();
|
||||||
|
assert!(!params.probe_hover);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn observe_result_round_trips_with_hover_probe_report() {
|
||||||
|
let r = ObserveResult {
|
||||||
|
text: "@vom 1\n".into(),
|
||||||
|
ref_count: 0,
|
||||||
|
tab_id: 42,
|
||||||
|
truncated: false,
|
||||||
|
dialogs: Vec::new(),
|
||||||
|
hover_probe: Some(HoverProbeReport {
|
||||||
|
performed: true,
|
||||||
|
revealed_content: true,
|
||||||
|
}),
|
||||||
|
debug: None,
|
||||||
|
};
|
||||||
|
let v = serde_json::to_value(&r).unwrap();
|
||||||
|
let round: ObserveResult = serde_json::from_value(v).unwrap();
|
||||||
|
assert_eq!(round, r);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+12
-9
@@ -67,7 +67,7 @@ Write operations only affect tabs in the **Agent Window** (or tabs you **borrowe
|
|||||||
|
|
||||||
```
|
```
|
||||||
bsk navigate <url> --session <id>
|
bsk navigate <url> --session <id>
|
||||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
bsk observe --session <id> → primary semantic VOM view
|
||||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
||||||
@@ -78,16 +78,19 @@ bsk observe --session <id> → again after navigation / DOM change
|
|||||||
|
|
||||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
||||||
|
|
||||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
When VOM marks a control `[has-submenu]` or `[expanded]`, its menu items are not in the observation yet. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action. `[hover first: …]` means the same thing and additionally lists what that trigger revealed.
|
||||||
|
|
||||||
|
`bsk observe` does not hover the page on its own. Reach for `--probe-hover` when a control you have good reason to expect is absent from the observation **and** no `[has-submenu]` marker points at a trigger — that combination is what a CSS-only hover menu looks like from here. Try it before falling back to `bsk get-html` or `bsk screenshot`. It hovers a bounded set of likely triggers to surface hover-only menus and tooltips; it costs a few seconds and touches the live page, so once you know which element hides the menu, `bsk hover <ref>` is cheaper and more precise.
|
||||||
|
|
||||||
## Observation priority
|
## Observation priority
|
||||||
|
|
||||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
Start with `bsk observe` to understand page structure, text, controls, and element refs. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||||
|
|
||||||
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
1. `bsk observe` — primary semantic VOM observation
|
||||||
2. `bsk snapshot` — strict static accessibility tree fallback
|
2. `bsk observe --probe-hover` — re-run once when an expected control is missing and nothing marks a hover trigger
|
||||||
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
3. `bsk snapshot` — strict static accessibility tree fallback
|
||||||
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
4. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
||||||
|
5. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
||||||
|
|
||||||
Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page.
|
Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page.
|
||||||
|
|
||||||
@@ -172,7 +175,7 @@ bsk emulate --session <id> --off
|
|||||||
| Command | Summary |
|
| Command | Summary |
|
||||||
|---------|---------|
|
|---------|---------|
|
||||||
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
|
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
|
||||||
| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
|
| `bsk observe` | Semantic VOM observation; `--probe-hover` adds bounded active hover probing for conditional surfaces |
|
||||||
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
|
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
|
||||||
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
|
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
|
||||||
|
|
||||||
@@ -319,7 +322,7 @@ Always **`bsk session stop <id>`** in a `finally`-style path so the Agent Window
|
|||||||
2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
|
2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
|
||||||
3. **No skip stop** — always `bsk session stop <id>`; never assume idle timeout will clean up.
|
3. **No skip stop** — always `bsk session stop <id>`; never assume idle timeout will clean up.
|
||||||
4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
|
4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
|
||||||
5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics or conditional surfaces help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual.
|
5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual.
|
||||||
6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
|
6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|||||||
Reference in New Issue
Block a user