diff --git a/apps/extension/src/browser-driver/__tests__/chromium-cdp.test.ts b/apps/extension/src/browser-driver/__tests__/chromium-cdp.test.ts index f6d88d3..8e09273 100644 --- a/apps/extension/src/browser-driver/__tests__/chromium-cdp.test.ts +++ b/apps/extension/src/browser-driver/__tests__/chromium-cdp.test.ts @@ -119,6 +119,88 @@ describe("ChromiumCdp", () => { expect(api.sendCommand).toHaveBeenCalledWith({ tabId: 9 }, "Page.enable", {}); }); + it("validates the attached top-frame URL for page reads", async () => { + const { api } = fakeApi(); + (api.sendCommand as ReturnType).mockImplementation( + async (_target, method: string) => { + if (method === "Page.getFrameTree") { + return { frameTree: { frame: { url: "https://example.test/page" } } }; + } + return {}; + }, + ); + const cdp = new ChromiumCdp(api); + + await expect( + cdp.ensureAttachedToUrl(9, "https://example.test/page#section"), + ).resolves.toBeUndefined(); + expect(api.detach).not.toHaveBeenCalled(); + }); + + it("reattaches when the cached CDP target points at an extension page", async () => { + const { api } = fakeApi(); + let frameReads = 0; + (api.sendCommand as ReturnType).mockImplementation( + async (_target, method: string) => { + if (method === "Page.getFrameTree") { + frameReads += 1; + return { + frameTree: { + frame: { + url: + frameReads === 1 + ? "chrome-extension://other-extension/page.html" + : "https://example.test/page", + }, + }, + }; + } + return {}; + }, + ); + const cdp = new ChromiumCdp(api); + + await expect(cdp.ensureAttachedToUrl(9, "https://example.test/page")).resolves.toBeUndefined(); + expect(api.attach).toHaveBeenCalledTimes(2); + expect(api.detach).toHaveBeenCalledWith({ tabId: 9 }); + }); + + it("rejects when reattach still points at the wrong page", async () => { + const { api } = fakeApi(); + (api.sendCommand as ReturnType).mockImplementation( + async (_target, method: string) => { + if (method === "Page.getFrameTree") { + return { frameTree: { frame: { url: "https://wrong.test/" } } }; + } + return {}; + }, + ); + const cdp = new ChromiumCdp(api); + + await expect(cdp.ensureAttachedToUrl(9, "https://example.test/")).rejects.toThrow( + /CDP target URL mismatch/, + ); + expect(api.attach).toHaveBeenCalledTimes(2); + }); + + it("clears the attach cache after an internal-page access error", async () => { + const { api } = fakeApi(); + (api.sendCommand as ReturnType).mockImplementation( + async (_target, method: string) => { + if (method === "DOM.getDocument") { + throw new Error("Cannot access a chrome-extension:// URL of different extension"); + } + return {}; + }, + ); + const cdp = new ChromiumCdp(api); + + await expect(cdp.ensureAttached(9)).resolves.toBeUndefined(); + expect(cdp.isAttached(9)).toBe(true); + await expect(cdp.send(9, "DOM.getDocument")).rejects.toThrow(/chrome-extension/); + expect(cdp.isAttached(9)).toBe(false); + }); + it("enables console capture domains best-effort during attach", async () => { const { api } = fakeApi(); (api.sendCommand as ReturnType).mockImplementation( diff --git a/apps/extension/src/browser-driver/chromium-cdp.ts b/apps/extension/src/browser-driver/chromium-cdp.ts index 3848e97..8aeec4f 100644 --- a/apps/extension/src/browser-driver/chromium-cdp.ts +++ b/apps/extension/src/browser-driver/chromium-cdp.ts @@ -93,6 +93,15 @@ const MAX_CONSOLE_STACK_FRAMES = 20; const MAX_NETWORK_BUFFER = 200; const MAX_NETWORK_FIELD_LENGTH = 4096; const MAX_NETWORK_REQUEST_META = 1024; +const CDP_INTERNAL_PROTOCOLS = new Set([ + "chrome:", + "chrome-extension:", + "devtools:", + "edge:", + "brave:", + "vivaldi:", + "opera:", +]); interface ParsedDialogOpening { type: JavaScriptDialogType; @@ -173,6 +182,31 @@ export class ChromiumCdp { await attach; } + /** + * Attach and verify Chrome's actual CDP target still points at the page URL + * Chrome tabs reported to the tool resolver. A stale debugger target can + * otherwise surface another extension/internal URL while the tab record looks + * like a normal page. + */ + async ensureAttachedToUrl(tabId: number, expectedUrl: string | undefined): Promise { + await this.ensureAttached(tabId); + if (!expectedUrl) return; + + const first = await this.readTopFrameUrl(tabId).catch((err) => { + if (isInternalTargetAccessError(err)) return null; + throw err; + }); + if (first && cdpUrlsCompatible(first, expectedUrl)) return; + + await this.forceReattach(tabId); + const second = await this.readTopFrameUrl(tabId); + if (cdpUrlsCompatible(second, expectedUrl)) return; + + throw new Error( + `CDP target URL mismatch for tab ${tabId}: expected ${expectedUrl}, attached to ${second}`, + ); + } + /** * Send a CDP command and decode the result as `T`. Throws on any * `chrome.runtime.lastError`. @@ -185,6 +219,9 @@ export class ChromiumCdp { const result = await this.api.sendCommand({ tabId }, method, params ?? {}); return result as T; } catch (err) { + if (isInternalTargetAccessError(err)) { + this.clearAttachedState(tabId); + } throw normalizeError(err); } } @@ -359,6 +396,31 @@ export class ChromiumCdp { } } + private async forceReattach(tabId: number): Promise { + this.clearAttachedState(tabId); + try { + await this.api.detach({ tabId }); + } catch { + // Chrome may already have detached; attach below is the source of truth. + } + await this.ensureAttached(tabId); + } + + private clearAttachedState(tabId: number): void { + this.attachInFlight.delete(tabId); + this.attachedTabs.delete(tabId); + this.clearDialogState(tabId); + this.clearConsoleState(tabId); + this.clearNetworkState(tabId); + } + + private async readTopFrameUrl(tabId: number): Promise { + const frameTree = await this.api.sendCommand({ tabId }, "Page.getFrameTree", {}); + const url = topFrameUrl(frameTree); + if (!url) throw new Error(`Page.getFrameTree returned no top-frame URL for tab ${tabId}`); + return url; + } + private bindDialogHandler(): void { if (this.dialogSubscription) return; const listener = (source: chrome.debugger.Debuggee, method: string, params: unknown) => { @@ -906,3 +968,49 @@ function normalizeError(err: unknown): Error { } return new Error("unknown chrome.debugger error"); } + +function isInternalTargetAccessError(err: unknown): boolean { + const message = normalizeError(err).message; + return ( + /Cannot access a chrome-extension:\/\/ URL/i.test(message) || + /Cannot access a chrome:\/\/ URL/i.test(message) || + /Cannot access contents of url/i.test(message) + ); +} + +function topFrameUrl(frameTree: unknown): string | null { + const raw = (frameTree ?? {}) as Record; + const tree = (raw.frameTree ?? {}) as Record; + const frame = (tree.frame ?? {}) as Record; + return typeof frame.url === "string" && frame.url.length > 0 ? frame.url : null; +} + +function cdpUrlsCompatible(actualUrl: string, expectedUrl: string): boolean { + const actual = normalizeComparableUrl(actualUrl); + const expected = normalizeComparableUrl(expectedUrl); + if (!actual || !expected) return false; + if (isInternalCdpUrl(actual)) return false; + return actual === expected; +} + +function normalizeComparableUrl(value: string): string | null { + let parsed: URL; + try { + parsed = new URL(value); + } catch { + return null; + } + parsed.hash = ""; + return parsed.toString(); +} + +function isInternalCdpUrl(value: string): boolean { + let parsed: URL; + try { + parsed = new URL(value); + } catch { + return false; + } + if (CDP_INTERNAL_PROTOCOLS.has(parsed.protocol)) return true; + return parsed.protocol === "about:" && parsed.pathname !== "blank"; +} diff --git a/apps/extension/src/content/ControlOverlay.tsx b/apps/extension/src/content/ControlOverlay.tsx deleted file mode 100644 index c6afc5b..0000000 --- a/apps/extension/src/content/ControlOverlay.tsx +++ /dev/null @@ -1,153 +0,0 @@ -import { useTranslation } from "@browser-skill/i18n/react"; -import { RiStopCircleLine } from "@remixicon/react"; -import { useEffect, useState } from "react"; -import logoUrl from "../../assets/logo.png"; - -export interface ControlOverlayProps { - visible: boolean; - interrupting: boolean; - automationBypass: boolean; - onInterrupt: () => void; -} - -export function ControlOverlay({ - visible, - interrupting, - automationBypass, - onInterrupt, -}: ControlOverlayProps) { - const { t } = useTranslation("extension"); - const [show, setShow] = useState(false); - - useEffect(() => { - if (visible) { - const raf = requestAnimationFrame(() => setShow(true)); - return () => cancelAnimationFrame(raf); - } - setShow(false); - }, [visible]); - - if (!visible) return null; - - const pointerEvents = automationBypass ? "none" : "auto"; - - return ( - <> - - -
- -
{ - if (automationBypass) return; - event.preventDefault(); - event.stopPropagation(); - }} - onClick={(event) => { - if (automationBypass) return; - event.preventDefault(); - event.stopPropagation(); - }} - style={{ - position: "fixed", - inset: 0, - zIndex: 2147483646, - pointerEvents, - background: "transparent", - opacity: show ? 1 : 0, - transition: "opacity 300ms ease-out", - }} - /> - -
- browser-skill - - {t("controlOverlay.status")} - - -
- - ); -} diff --git a/apps/extension/src/content/__tests__/ControlOverlay.test.tsx b/apps/extension/src/content/__tests__/ControlOverlay.test.tsx deleted file mode 100644 index d35c609..0000000 --- a/apps/extension/src/content/__tests__/ControlOverlay.test.tsx +++ /dev/null @@ -1,47 +0,0 @@ -import { cleanup, render } from "@testing-library/react"; -import { afterEach, describe, expect, it } from "vitest"; -import { ControlOverlay } from "../ControlOverlay"; - -describe("ControlOverlay", () => { - afterEach(() => { - cleanup(); - }); - - it("keeps page blocker none under automationBypass but Interrupt stays clickable", () => { - const { container } = render( - {}} - />, - ); - - const blocker = container.querySelector("[data-slot='control-overlay-blocker']"); - expect(blocker).toBeTruthy(); - expect((blocker as HTMLElement).style.pointerEvents).toBe("none"); - - const pill = container.querySelector("[data-slot='control-overlay-pill']"); - expect(pill).toBeTruthy(); - expect((pill as HTMLElement).style.pointerEvents).toBe("auto"); - - const stopBtn = container.querySelector("[data-slot='control-overlay-stop-all']"); - expect(stopBtn).toBeTruthy(); - expect((stopBtn as HTMLElement).style.pointerEvents).toBe("auto"); - }); - - it("uses pointer-events auto on blocker when automationBypass is false", () => { - const { container } = render( - {}} - />, - ); - - const blocker = container.querySelector("[data-slot='control-overlay-blocker']"); - expect(blocker).toBeTruthy(); - expect((blocker as HTMLElement).style.pointerEvents).toBe("auto"); - }); -}); diff --git a/apps/extension/src/content/__tests__/overlay-controller.test.ts b/apps/extension/src/content/__tests__/overlay-controller.test.ts index 7e8fd93..bbcde9f 100644 --- a/apps/extension/src/content/__tests__/overlay-controller.test.ts +++ b/apps/extension/src/content/__tests__/overlay-controller.test.ts @@ -116,4 +116,18 @@ describe("OverlayController", () => { expect(controller.snapshot().suppressControlAfterRecord).toBe(false); expect(controller.snapshot().controlVisible).toBe(false); }); + + it("hides control while paused and shows it again only after authoritative control mode", () => { + const controller = new OverlayController(); + + controller.applyAgentControlMode("sess-1", "paused"); + expect(controller.snapshot().activeSessionId).toBe("sess-1"); + expect(controller.snapshot().controlVisible).toBe(false); + expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false); + + controller.applyAgentControlMode("sess-1", "control"); + expect(controller.snapshot().activeSessionId).toBe("sess-1"); + expect(controller.snapshot().controlVisible).toBe(true); + expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(true); + }); }); diff --git a/apps/extension/src/content/overlay-controller.ts b/apps/extension/src/content/overlay-controller.ts index a0f6560..0ec7b67 100644 --- a/apps/extension/src/content/overlay-controller.ts +++ b/apps/extension/src/content/overlay-controller.ts @@ -1,3 +1,4 @@ +import type { OverlayMode } from "@/lib/overlay-bridge"; import type { BorrowRequestData } from "./BorrowConfirmationOverlay"; import type { HelpRequestData } from "./HelpRequestOverlay"; import type { RecordRequestData } from "./RecordOverlay"; @@ -9,6 +10,7 @@ export interface OverlayState { controlVisible: boolean; interrupting: boolean; activeSessionId: string | null; + controlMode: OverlayMode; automationBypassCount: number; /** * After record Finish/Stop clears `activeRecord`, the session is still @@ -31,6 +33,7 @@ export class OverlayController { activeRecord: null, interrupting: false, activeSessionId: null, + controlMode: "hidden", automationBypassCount: 0, suppressControlAfterRecord: false, }; @@ -44,7 +47,7 @@ export class OverlayController { } isControlVisible(): boolean { - return this.state.activeSessionId !== null; + return this.state.activeSessionId !== null && this.state.controlMode === "control"; } addBorrowRequest(request: BorrowRequestData): void { @@ -61,6 +64,25 @@ export class OverlayController { activateAgentSession(sessionId: string): void { this.state.activeSessionId = sessionId; this.state.interrupting = false; + this.state.controlMode = "control"; + } + + applyAgentControlMode(sessionId: string | null, mode: OverlayMode): void { + if (mode === "control" && sessionId) { + this.state.activeSessionId = sessionId; + this.state.interrupting = false; + this.state.controlMode = "control"; + return; + } + if ((mode === "interrupting" || mode === "paused") && sessionId) { + this.state.activeSessionId = sessionId; + this.state.interrupting = mode === "interrupting"; + this.state.controlMode = mode; + return; + } + this.state.activeSessionId = null; + this.state.interrupting = false; + this.state.controlMode = "hidden"; } setInterrupting(interrupting: boolean): void { @@ -118,6 +140,7 @@ export class OverlayController { activeRecord: null, interrupting: false, activeSessionId: null, + controlMode: "hidden", automationBypassCount: 0, suppressControlAfterRecord: false, }; diff --git a/apps/extension/src/entrypoints/background.ts b/apps/extension/src/entrypoints/background.ts index 6192811..ce3586f 100644 --- a/apps/extension/src/entrypoints/background.ts +++ b/apps/extension/src/entrypoints/background.ts @@ -9,12 +9,16 @@ import { } from "@/lib/instance-id"; import { startKeepalive } from "@/lib/keepalive"; import { + OVERLAY_AGENT_STATE, OVERLAY_AUTOMATION_BYPASS, OVERLAY_MSG_INTERRUPT, + OVERLAY_MSG_READY, OVERLAY_MSG_WHO_AM_I, + type OverlayAgentStateMessage, type OverlayInterruptRequest, type OverlayInterruptResponse, type OverlayMessage, + type OverlayMode, } from "@/lib/overlay-bridge"; import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge"; import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag"; @@ -44,6 +48,91 @@ export default defineBackground(() => { const sessions = new SessionManager(); const cdp = new ChromiumCdp(); const sessionsLive = attachSessionsLiveFlag({ manager: sessions }); + let overlayGeneration = 0; + const controlModes = new Map(); + + function setControlMode(sessionId: string, mode: OverlayMode): void { + if (controlModes.get(sessionId) === mode) return; + controlModes.set(sessionId, mode); + overlayGeneration += 1; + const ctx = sessions.get(sessionId); + if (ctx) void pushOverlayStateForWindow(ctx.agentWindowId); + } + + function overlayStateForWindow(windowId?: number): OverlayAgentStateMessage { + const ctx = typeof windowId === "number" ? sessions.findByWindowId(windowId) : null; + if (!ctx) { + return { + type: OVERLAY_AGENT_STATE, + sessionId: null, + mode: "hidden", + generation: overlayGeneration, + }; + } + return { + type: OVERLAY_AGENT_STATE, + sessionId: ctx.sessionId, + mode: controlModes.get(ctx.sessionId) ?? "control", + generation: overlayGeneration, + }; + } + + async function pushOverlayStateToTab( + tabId: number, + state: OverlayAgentStateMessage, + ): Promise { + try { + await chrome.tabs.sendMessage(tabId, state); + } catch { + // Restricted pages and not-yet-loaded content scripts cannot receive messages. + } + } + + async function pushOverlayStateForWindow(windowId: number): Promise { + const state = overlayStateForWindow(windowId); + const tabs = await chrome.tabs.query({ windowId }); + await Promise.all( + tabs.map((tab) => + typeof tab.id === "number" ? pushOverlayStateToTab(tab.id, state) : Promise.resolve(), + ), + ); + } + + function pushAllAgentOverlayStates(): void { + const windowIds = new Set(sessions.list().map((ctx) => ctx.agentWindowId)); + for (const windowId of windowIds) { + void pushOverlayStateForWindow(windowId); + } + } + + function onOverlaySessionStateChanged(): void { + void sessionsLive.syncFromManager(); + const liveSessionIds = new Set(sessions.list().map((ctx) => ctx.sessionId)); + for (const sessionId of controlModes.keys()) { + if (!liveSessionIds.has(sessionId)) controlModes.delete(sessionId); + } + overlayGeneration += 1; + pushAllAgentOverlayStates(); + } + + function onBrowserControlResumed(sessionId: string): void { + const ctx = sessions.get(sessionId); + if (!ctx) return; + setControlMode(sessionId, "control"); + } + + function pushOverlayStateForAgentWindow(windowId: number): void { + if (!sessions.findByWindowId(windowId)) return; + void pushOverlayStateForWindow(windowId); + } + chrome.tabs.onActivated.addListener((activeInfo) => { + pushOverlayStateForAgentWindow(activeInfo.windowId); + }); + chrome.tabs.onUpdated.addListener((_tabId, changeInfo, tab) => { + if (changeInfo.status !== "complete") return; + if (typeof tab.windowId !== "number") return; + pushOverlayStateForAgentWindow(tab.windowId); + }); // Re-sync the storage.session flag on SW startup so a previous SW's // stale `true` does not keep waking us on every page load until the // first mutation (review M4/M5 round 3 m-R3-1). @@ -52,9 +141,8 @@ export default defineBackground(() => { transport, sessions, cdp, - onSessionsChanged: () => { - void sessionsLive.syncFromManager(); - }, + onSessionsChanged: onOverlaySessionStateChanged, + onBrowserControlResumed, approveBorrow: (ctx) => requestBorrowConfirmation(ctx.tabId, { ...(ctx.signal !== undefined ? { signal: ctx.signal } : {}), @@ -124,9 +212,7 @@ export default defineBackground(() => { manager: sessions, transport, cdp, - onSessionsChanged: () => { - void sessionsLive.syncFromManager(); - }, + onSessionsChanged: onOverlaySessionStateChanged, }); // MV3 service worker keepalive + reconnect supervisor (review M4/M5 @@ -197,9 +283,19 @@ export default defineBackground(() => { return false; } + if (msg.kind === OVERLAY_MSG_READY) { + sendResponse(overlayStateForWindow(sender.tab?.windowId)); + return false; + } + if (msg.kind === OVERLAY_MSG_INTERRUPT) { const req = msg as OverlayInterruptRequest; + const ctx = sessions.get(req.sessionId); + if (ctx) setControlMode(req.sessionId, "interrupting"); void handleOverlayInterrupt(transport, req.sessionId).then((reply) => { + if (reply.ok && sessions.get(req.sessionId)) { + setControlMode(req.sessionId, "paused"); + } sendResponse(reply); }); return true; // keep channel open diff --git a/apps/extension/src/entrypoints/content.ts b/apps/extension/src/entrypoints/content.ts index c6fbce3..32cd5ed 100644 --- a/apps/extension/src/entrypoints/content.ts +++ b/apps/extension/src/entrypoints/content.ts @@ -3,7 +3,6 @@ import { I18nextProvider } from "@browser-skill/i18n/react"; import React from "react"; import ReactDOM from "react-dom/client"; import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay"; -import { ControlOverlay } from "@/content/ControlOverlay"; import { HelpRequestOverlay } from "@/content/HelpRequestOverlay"; import overlayCss from "@/content/overlay.css?inline"; import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller"; @@ -27,13 +26,12 @@ import { } from "@/lib/help-bridge"; import { isOverlayAgentOverlayResetMessage, + isOverlayAgentStateMessage, OVERLAY_AUTOMATION_BYPASS, - OVERLAY_HOST_MARKER_ATTR, - OVERLAY_HOST_NAME, - OVERLAY_MSG_WHO_AM_I, + OVERLAY_MSG_READY, type OverlayAgentOverlayResetMessage, + type OverlayAgentStateMessage, type OverlayAutomationBypassMessage, - type OverlayWhoAmIResponse, } from "@/lib/overlay-bridge"; import { sendInterrupt } from "@/lib/overlay-interrupt-client"; import { @@ -42,12 +40,12 @@ import { type RecordStartAck, type RecordStopAck, } from "@/lib/record-bridge"; -import { SESSIONS_LIVE_FLAG_KEY } from "@/lib/sessions-live-flag"; import type { BorrowCancelMessage, BorrowRequestMessage, BorrowResponseMessage, } from "@/tools/borrow-confirmation"; +import logoUrl from "../../assets/logo.png"; // Run at document_end so the overlay does not block first paint. Only attach // in the top-level frame so iframes do not double-render overlays. @@ -65,34 +63,264 @@ export default defineContentScript({ let activeRecordRequestId: string | null = null; let reactRoot: ReactDOM.Root | null = null; let overlayHost: HTMLElement | null = null; + let overlayContainer: HTMLElement | null = null; + let controlRoot: HTMLElement | null = null; + let activeAgentState: OverlayAgentStateMessage | null = null; let hostLossReported = false; let remountInProgress = false; const ui = await createShadowRootUi(ctx, { - name: OVERLAY_HOST_NAME, + name: "browser-skill-overlay", position: "inline", anchor: "html", css: overlayCss, onMount(container, _shadow, shadowHost) { shadowHost.setAttribute("aria-hidden", "true"); - shadowHost.setAttribute(OVERLAY_HOST_MARKER_ATTR, ""); + shadowHost.setAttribute("data-bsk-overlay", ""); overlayHost = shadowHost; + overlayContainer = container; hostLossReported = false; const app = document.createElement("div"); app.className = "bsk-overlay-root"; container.append(app); reactRoot = ReactDOM.createRoot(app); - renderOverlay(); + renderAll(); + void requestOverlayState(); return reactRoot; }, onRemove(root) { overlayHost = null; + overlayContainer = null; + controlRoot = null; root?.unmount(); reactRoot = null; }, }); - function renderOverlay() { + function removeControlOverlay(): void { + controlRoot?.remove(); + controlRoot = null; + } + + function setControlSurfaceActive(active: boolean): void { + const host = overlayHost; + const container = overlayContainer; + if (!host || !container) return; + + if (active) { + Object.assign(host.style, { + position: "fixed", + inset: "0", + zIndex: "2147483647", + pointerEvents: "auto", + }); + Object.assign(container.style, { + position: "fixed", + inset: "0", + pointerEvents: "none", + }); + return; + } + + host.style.removeProperty("position"); + host.style.removeProperty("inset"); + host.style.removeProperty("z-index"); + host.style.removeProperty("pointer-events"); + container.style.removeProperty("position"); + container.style.removeProperty("inset"); + container.style.removeProperty("pointer-events"); + } + + function ensureControlOverlayRoot(): HTMLElement | null { + if (!overlayContainer) return null; + if (!controlRoot) { + controlRoot = document.createElement("div"); + controlRoot.className = "bsk-control-overlay"; + controlRoot.style.opacity = "0"; + controlRoot.style.transition = "opacity 300ms ease-out"; + overlayContainer.append(controlRoot); + window.requestAnimationFrame(() => { + if (controlRoot) controlRoot.style.opacity = "1"; + }); + } + return controlRoot; + } + + function createStopCircleIcon(): SVGSVGElement { + const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg"); + svg.setAttribute("width", "18"); + svg.setAttribute("height", "18"); + svg.setAttribute("viewBox", "0 0 24 24"); + svg.setAttribute("fill", "currentColor"); + svg.setAttribute("aria-hidden", "true"); + svg.style.color = "#fff"; + svg.style.flexShrink = "0"; + const path = document.createElementNS("http://www.w3.org/2000/svg", "path"); + path.setAttribute( + "d", + "M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10Zm0-2a8 8 0 1 0 0-16 8 8 0 0 0 0 16ZM9 9h6v6H9V9Z", + ); + svg.append(path); + return svg; + } + + function renderControlOverlay(): void { + const overlayState = overlays.snapshot(); + if (!shouldShowAgentControlOverlay(overlayState)) { + removeControlOverlay(); + setControlSurfaceActive(false); + return; + } + + const root = ensureControlOverlayRoot(); + if (!root) return; + + const pointerEvents = overlayState.automationBypassCount > 0 ? "none" : "auto"; + setControlSurfaceActive(true); + root.innerHTML = ""; + Object.assign(root.style, { + position: "fixed", + inset: "0", + pointerEvents: "none", + }); + + const style = document.createElement("style"); + style.textContent = ` + @keyframes bsk-breathe { + 0%, 100% { box-shadow: inset 0 0 20px 4px rgba(249,115,22,0.25); } + 50% { box-shadow: inset 0 0 40px 8px rgba(249,115,22,0.5); } + } + `; + + const border = document.createElement("div"); + border.setAttribute("data-slot", "control-overlay"); + Object.assign(border.style, { + position: "fixed", + inset: "0", + zIndex: "2147483646", + pointerEvents: "none", + animation: "bsk-breathe 3s ease-in-out infinite", + }); + + const blocker = document.createElement("div"); + blocker.setAttribute("data-slot", "control-overlay-blocker"); + Object.assign(blocker.style, { + position: "fixed", + inset: "0", + zIndex: "2147483646", + pointerEvents, + background: "transparent", + }); + blocker.addEventListener("pointerdown", (event) => { + if (overlayState.automationBypassCount > 0) return; + event.preventDefault(); + event.stopPropagation(); + }); + blocker.addEventListener("click", (event) => { + if (overlayState.automationBypassCount > 0) return; + event.preventDefault(); + event.stopPropagation(); + }); + blocker.addEventListener( + "wheel", + (event) => { + if (overlayState.automationBypassCount > 0) return; + event.preventDefault(); + event.stopPropagation(); + }, + { passive: false }, + ); + blocker.addEventListener( + "touchmove", + (event) => { + if (overlayState.automationBypassCount > 0) return; + event.preventDefault(); + event.stopPropagation(); + }, + { passive: false }, + ); + + const pill = document.createElement("div"); + pill.setAttribute("data-slot", "control-overlay-pill"); + Object.assign(pill.style, { + position: "fixed", + bottom: "32px", + left: "50%", + transform: "translateX(-50%)", + zIndex: "2147483647", + pointerEvents: "auto", + display: "flex", + alignItems: "center", + gap: "12px", + borderRadius: "999px", + backgroundColor: "#fff", + padding: "10px 10px 10px 20px", + boxShadow: "0 8px 32px rgba(124,45,18,0.16), 0 2px 8px rgba(0,0,0,0.1)", + fontFamily: + '-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif', + }); + + const logo = document.createElement("img"); + logo.src = logoUrl; + logo.alt = "browser-skill"; + Object.assign(logo.style, { + width: "24px", + height: "24px", + borderRadius: "4px", + flexShrink: "0", + }); + + const label = document.createElement("span"); + label.textContent = i18n.t("controlOverlay.status", { ns: "extension" }); + Object.assign(label.style, { + fontSize: "16px", + fontWeight: "500", + color: "#333", + whiteSpace: "nowrap", + userSelect: "none", + }); + + const button = document.createElement("button"); + button.type = "button"; + button.setAttribute("data-slot", "control-overlay-stop-all"); + button.disabled = overlayState.interrupting; + Object.assign(button.style, { + pointerEvents: "auto", + display: "flex", + alignItems: "center", + gap: "6px", + border: "none", + borderRadius: "999px", + padding: "8px 20px 8px 16px", + fontSize: "15px", + fontWeight: "600", + color: "#fff", + backgroundColor: overlayState.interrupting ? "#9ca3af" : "#f97316", + cursor: overlayState.interrupting ? "default" : "pointer", + opacity: overlayState.interrupting ? "0.7" : "1", + transition: "background-color 150ms ease-out, opacity 150ms ease-out", + whiteSpace: "nowrap", + lineHeight: "1", + }); + button.append( + createStopCircleIcon(), + document.createTextNode( + overlayState.interrupting + ? i18n.t("controlOverlay.interrupting", { ns: "extension" }) + : i18n.t("controlOverlay.interrupt", { ns: "extension" }), + ), + ); + button.addEventListener("click", (event) => { + event.preventDefault(); + event.stopPropagation(); + handleInterrupt(); + }); + + pill.append(logo, label, button); + root.append(style, border, blocker, pill); + } + + function renderReactOverlays(): void { const overlayState = overlays.snapshot(); reactRoot?.render( React.createElement( @@ -104,12 +332,6 @@ export default defineContentScript({ React.createElement(BorrowConfirmationOverlay, { requests: overlayState.borrowRequests, }), - React.createElement(ControlOverlay, { - visible: shouldShowAgentControlOverlay(overlayState), - interrupting: overlayState.interrupting, - automationBypass: overlayState.automationBypassCount > 0, - onInterrupt: handleInterrupt, - }), React.createElement(HelpRequestOverlay, { request: overlayState.activeHelp }), React.createElement(RecordOverlay, { request: overlayState.activeRecord }), ), @@ -117,6 +339,23 @@ export default defineContentScript({ ); } + function renderAll(): void { + renderReactOverlays(); + renderControlOverlay(); + } + + function clearCurrentAgentSession(): void { + const sessionId = overlays.snapshot().activeSessionId; + if (!sessionId) return; + resetAgentOverlayState(sessionId); + } + + function applyOverlayState(state: OverlayAgentStateMessage): void { + activeAgentState = state; + overlays.applyAgentControlMode(state.sessionId, state.mode); + renderAll(); + } + function resetAgentOverlayState(sessionId: string) { const previousHelp = overlays.resetAgentOverlays(sessionId); if (previousHelp) { @@ -125,7 +364,7 @@ export default defineContentScript({ recordCapture?.dispose(); recordCapture = null; activeRecordRequestId = null; - renderOverlay(); + renderAll(); } function handleInterrupt() { @@ -137,12 +376,8 @@ export default defineContentScript({ return; } overlays.setInterrupting(true); - renderOverlay(); + renderAll(); void sendInterrupt((msg) => chrome.runtime.sendMessage(msg), sessionId).then((reply) => { - // Always retract the mask after the round trip resolves - // (success, failure, or timeout). Cancellation is fire-and- - // forget on the daemon side; the user must not be stuck - // behind a transient issue. The Agent Window stays open. resetAgentOverlayState(sessionId); if (!reply.ok) { console.warn("[bsk overlay] interrupt did not get a clean ack from daemon"); @@ -157,6 +392,7 @@ export default defineContentScript({ | HelpRequestMessage | HelpCancelMessage | OverlayAgentOverlayResetMessage + | OverlayAgentStateMessage | OverlayAutomationBypassMessage, _sender: chrome.runtime.MessageSender, sendResponse: (response: BorrowResponseMessage | HelpAckMessage) => void, @@ -183,11 +419,11 @@ export default defineContentScript({ }); }, }); - renderOverlay(); + renderAll(); }, onStop: () => { overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined); - renderOverlay(); + renderAll(); }, }, sendResponse as unknown as @@ -205,7 +441,12 @@ export default defineContentScript({ ) { const bypassMsg = message as OverlayAutomationBypassMessage; overlays.setAutomationBypass(bypassMsg.enabled); - renderOverlay(); + renderAll(); + return false; + } + + if (isOverlayAgentStateMessage(message)) { + applyOverlayState(message); return false; } @@ -216,7 +457,7 @@ export default defineContentScript({ if (message.type === "borrow-cancel") { overlays.removeBorrowRequest(message.requestId); - renderOverlay(); + renderAll(); return false; } @@ -224,7 +465,7 @@ export default defineContentScript({ const state = overlays.snapshot(); if (state.activeHelp && state.activeHelp.id === message.requestId) { overlays.clearAgentHelpRequest(message.requestId); - renderOverlay(); + renderAll(); } return false; } @@ -244,7 +485,7 @@ export default defineContentScript({ if (previousHelp && previousHelp.id !== helpMsg.requestId) { void sendHelpFinish(previousHelp.id, "cancelled"); } - renderOverlay(); + renderAll(); sendResponse({ type: HELP_ACK, ok: true }); return false; } @@ -256,7 +497,7 @@ export default defineContentScript({ responded = true; sendResponse({ type: "borrow-response", allowed }); overlays.removeBorrowRequest(message.requestId); - renderOverlay(); + renderAll(); }; overlays.addBorrowRequest({ @@ -267,7 +508,7 @@ export default defineContentScript({ onAllow: () => respond(true), onDeny: () => respond(false), }); - renderOverlay(); + renderAll(); return true; } @@ -286,7 +527,7 @@ export default defineContentScript({ ...(note ? { note } : {}), }; overlays.clearAgentHelpRequest(requestId); - renderOverlay(); + renderAll(); await chrome.runtime.sendMessage(msg).catch((err) => { console.debug("[bsk overlay] help finish failed", err); }); @@ -305,7 +546,7 @@ export default defineContentScript({ }); } - async function queryActiveHelpWithRetry(): Promise { + async function queryActiveHelpWithRetry(): Promise { for (let attempt = 0; attempt < 6; attempt += 1) { try { const helpQuery = (await chrome.runtime.sendMessage({ @@ -313,38 +554,21 @@ export default defineContentScript({ })) as HelpQueryResponse | undefined; if (helpQuery?.active && helpQuery.request) { mountHelpRequest(helpQuery.request); - renderOverlay(); - return true; + renderAll(); + return; } } catch (err) { console.debug("[bsk overlay] help query failed", err); } await new Promise((resolve) => window.setTimeout(resolve, 150)); } - return false; } - async function syncAgentOverlay(): Promise { - if (!(await anySessionLive())) return; + async function queryActiveRecord(): Promise { try { - const helpActive = await queryActiveHelpWithRetry(); - const reply = (await chrome.runtime.sendMessage({ - kind: OVERLAY_MSG_WHO_AM_I, - })) as OverlayWhoAmIResponse | undefined; - if (!reply?.sessionId) return; - - // Query / re-arm recording *before* activating the control overlay so - // record start (navigate → content load) does not flash - // 「Agent 正在控制」before RecordOverlay mounts. - let recordQuery: { active?: boolean; requestId?: string } | undefined; - try { - recordQuery = (await chrome.runtime.sendMessage({ - type: RECORD_QUERY, - })) as { active?: boolean; requestId?: string } | undefined; - } catch (err) { - console.debug("[bsk overlay] record query failed", err); - } - + const recordQuery = (await chrome.runtime.sendMessage({ + type: RECORD_QUERY, + })) as { active?: boolean; requestId?: string } | undefined; if ( recordQuery?.active && typeof recordQuery.requestId === "string" && @@ -360,26 +584,38 @@ export default defineContentScript({ }); }, }); + renderAll(); } - - if (!helpActive && overlays.snapshot().activeHelp === null) { - void queryActiveHelpWithRetry(); - } - - overlays.activateAgentSession(reply.sessionId); - renderOverlay(); } catch (err) { - console.debug("[bsk overlay] syncAgentOverlay failed", err); + console.debug("[bsk overlay] record query failed", err); + } + } + + async function refreshAuxiliaryOverlayState(): Promise { + await Promise.all([queryActiveHelpWithRetry(), queryActiveRecord()]); + } + + async function requestOverlayState(): Promise { + try { + const state = (await chrome.runtime.sendMessage({ + kind: OVERLAY_MSG_READY, + })) as OverlayAgentStateMessage | undefined; + if (state && isOverlayAgentStateMessage(state)) { + applyOverlayState(state); + } + void refreshAuxiliaryOverlayState(); + } catch (err) { + console.debug("[bsk overlay] overlay.ready failed", err); } } const onPageShow = (event: PageTransitionEvent) => { - if (event.persisted) void syncAgentOverlay(); + if (event.persisted) void requestOverlayState(); }; ui.mount(); chrome.runtime.onMessage.addListener(onMessage); - void syncAgentOverlay(); + void requestOverlayState(); window.addEventListener("pageshow", onPageShow); @@ -391,7 +627,8 @@ export default defineContentScript({ remountInProgress = true; try { ui.mount(); - void syncAgentOverlay(); + if (activeAgentState) applyOverlayState(activeAgentState); + void requestOverlayState(); } finally { remountInProgress = false; } @@ -414,16 +651,3 @@ export default defineContentScript({ }); }, }); - -async function anySessionLive(): Promise { - if (!chrome.storage?.session?.get) return true; - try { - const result = (await chrome.storage.session.get({ - [SESSIONS_LIVE_FLAG_KEY]: false, - })) as Record | undefined; - return Boolean(result?.[SESSIONS_LIVE_FLAG_KEY]); - } catch (err) { - console.debug("[bsk overlay] sessions-live flag read failed", err); - return true; - } -} diff --git a/apps/extension/src/lib/help-bridge.ts b/apps/extension/src/lib/help-bridge.ts index b73b45c..41b9747 100644 --- a/apps/extension/src/lib/help-bridge.ts +++ b/apps/extension/src/lib/help-bridge.ts @@ -5,7 +5,7 @@ * * Mirrors the borrow-confirmation message style (see * `@/tools/borrow-confirmation`). The background asks a specific tab to - * enter "help mode" (render HelpRequestOverlay + hide ControlOverlay); + * enter "help mode" (render HelpRequestOverlay + hide the control mask); * the content script acks display, then later reports Done / Cancel. */ diff --git a/apps/extension/src/lib/overlay-bridge.ts b/apps/extension/src/lib/overlay-bridge.ts index f1e5ab0..b7f0638 100644 --- a/apps/extension/src/lib/overlay-bridge.ts +++ b/apps/extension/src/lib/overlay-bridge.ts @@ -3,9 +3,8 @@ * between the content-script control overlay and the background SW. * * Content script → background: - * - `{ kind: "overlay.who_am_i", tabId, windowId }` → background - * replies `{ sessionId: string | null }` so the overlay knows - * whether to render itself. + * - `{ kind: "overlay.ready" }` → background replies with the authoritative + * overlay state for the sender's window. * - `{ kind: "overlay.interrupt", sessionId }` → background asks the * daemon (via a `session.user_interrupt` WS event) to cancel * every inflight + queued tool call for that session with @@ -14,6 +13,7 @@ */ export const OVERLAY_MSG_WHO_AM_I = "overlay.who_am_i"; +export const OVERLAY_MSG_READY = "overlay.ready"; export const OVERLAY_MSG_INTERRUPT = "overlay.interrupt"; /** @@ -57,6 +57,22 @@ export interface OverlayWhoAmIResponse { sessionId: string | null; } +export interface OverlayReadyRequest { + kind: typeof OVERLAY_MSG_READY; +} + +export type OverlayMode = "control" | "interrupting" | "paused" | "hidden"; + +/** Background → content: complete, authoritative control-overlay state. */ +export const OVERLAY_AGENT_STATE = "bh-agent-overlay-state"; + +export interface OverlayAgentStateMessage { + type: typeof OVERLAY_AGENT_STATE; + sessionId: string | null; + mode: OverlayMode; + generation: number; +} + export interface OverlayInterruptRequest { kind: typeof OVERLAY_MSG_INTERRUPT; sessionId: string; @@ -90,4 +106,23 @@ export function isOverlayAgentOverlayResetMessage( return candidate.type === OVERLAY_AGENT_OVERLAY_RESET && typeof candidate.sessionId === "string"; } -export type OverlayMessage = OverlayWhoAmIRequest | OverlayInterruptRequest; +export function isOverlayAgentStateMessage(message: unknown): message is OverlayAgentStateMessage { + if (!message || typeof message !== "object") return false; + const candidate = message as { + type?: unknown; + sessionId?: unknown; + mode?: unknown; + generation?: unknown; + }; + return ( + candidate.type === OVERLAY_AGENT_STATE && + (typeof candidate.sessionId === "string" || candidate.sessionId === null) && + (candidate.mode === "control" || + candidate.mode === "interrupting" || + candidate.mode === "paused" || + candidate.mode === "hidden") && + typeof candidate.generation === "number" + ); +} + +export type OverlayMessage = OverlayWhoAmIRequest | OverlayReadyRequest | OverlayInterruptRequest; diff --git a/apps/extension/src/tools/__tests__/dispatcher.test.ts b/apps/extension/src/tools/__tests__/dispatcher.test.ts index a0a3ad5..b2f00c5 100644 --- a/apps/extension/src/tools/__tests__/dispatcher.test.ts +++ b/apps/extension/src/tools/__tests__/dispatcher.test.ts @@ -274,6 +274,35 @@ describe("ToolDispatcher", () => { expect(onSessionsChanged).toHaveBeenCalledTimes(2); }); + it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => { + const { transport, sent, deliver } = fakeTransport(); + const sessions = new SessionManager({ + agentWindow: { + create: vi.fn(async () => 1), + remove: vi.fn(async () => {}), + ensureActiveTab: vi.fn(async () => {}), + }, + }); + const onBrowserControlResumed = vi.fn(); + const dispatcher = new ToolDispatcher({ transport, sessions, onBrowserControlResumed }); + dispatcher.start(); + + deliver(makeRequest("tool.session_start", { session_id: "aa11" })); + await flushMicrotasks(); + sent.length = 0; + + deliver({ ...makeRequest("tool.snapshot", { session_id: "aa11" }), id: "r-passive" }); + await flushMicrotasks(); + expect(onBrowserControlResumed).not.toHaveBeenCalled(); + + deliver({ + ...makeRequest("tool.tab_create", { session_id: "aa11", url: "https://example.test/" }), + id: "r-control", + }); + await flushMicrotasks(); + expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11"); + }); + it("disconnects the transport when send() fails so keepalive can rebuild", async () => { const { transport, deliver } = fakeTransport(); const sessions = new SessionManager({ diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index 233e9f8..bbb5900 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -7,6 +7,7 @@ import { buildVomScene, type CdpAxNode, handleGetHtml, + handleObserve, handleScreenshot, handleSnapshot, parsePngDimensions, @@ -1412,7 +1413,7 @@ describe("buildVomScene", () => { expect(scene.surfaces).toEqual([ { triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] }, ]); - expect(renderVom(scene).text).toContain('@e1 button "Products" [→ Shoes | Bags]'); + expect(renderVom(scene).text).toContain('@e1 button "Products" [hover: Shoes | Bags]'); }); it("enriches names and active scope signals from AX properties", () => { @@ -1741,6 +1742,225 @@ describe("handleSnapshot", () => { expect(ctx.refStore.resolve("e2")).toBeNull(); }); + it("keeps snapshots static without conditional surface probing", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + await sm.start("aa11"); + const root: CdpAxNode = { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + name: { type: "computedString", value: "Example" }, + backendDOMNodeId: 100, + childIds: ["2"], + }; + const button: CdpAxNode = { + nodeId: "2", + parentId: "1", + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Products" }, + backendDOMNodeId: 200, + }; + const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"]; + const i = (s: string) => strings.indexOf(s); + const send = vi.fn(async (_tabId: number, method: string) => { + if (method === "Accessibility.enable") return {}; + if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] }; + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } }; + } + if (method === "DOMSnapshot.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") { + return { + strings, + documents: [ + { + nodes: { + parentIndex: [-1, 0], + nodeName: [i("body"), i("button")], + backendNodeId: [100, 200], + attributes: [[], []], + }, + layout: { + nodeIndex: [0, 1], + styles: [ + [i("static"), i("auto"), i("auto")], + [i("static"), i("auto"), i("auto")], + ], + bounds: [ + [0, 0, 1000, 800], + [20, 20, 120, 40], + ], + paintOrders: [0, 1], + }, + }, + ], + }; + } + if (method === "Runtime.evaluate") return { result: { value: [] } }; + throw new Error(`unexpected CDP method ${method}`); + }); + const res = await handleSnapshot( + sm, + { session_id: "aa11" }, + { + cdp: { + send: send as unknown as ( + tabId: number, + method: string, + params?: object, + ) => Promise, + trackSessionTab: vi.fn(), + }, + tabsApi: { + get: vi.fn( + async (tabId: number) => + ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, + ), + query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), + }, + }, + ); + + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(send).not.toHaveBeenCalledWith( + 4, + "Runtime.evaluate", + expect.objectContaining({ returnByValue: true }), + ); + }); + + it("enables conditional surface probing for semantic observe", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + await sm.start("aa11"); + const root: CdpAxNode = { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + name: { type: "computedString", value: "Example" }, + backendDOMNodeId: 100, + childIds: ["2"], + }; + const button: CdpAxNode = { + nodeId: "2", + parentId: "1", + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Products" }, + backendDOMNodeId: 200, + }; + const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"]; + const i = (s: string) => strings.indexOf(s); + const send = vi.fn(async (_tabId: number, method: string) => { + if (method === "Accessibility.enable") return {}; + if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] }; + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } }; + } + if (method === "DOMSnapshot.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") { + return { + strings, + documents: [ + { + nodes: { + parentIndex: [-1, 0], + nodeName: [i("body"), i("button")], + backendNodeId: [100, 200], + attributes: [[], []], + }, + layout: { + nodeIndex: [0, 1], + styles: [ + [i("static"), i("auto"), i("auto")], + [i("static"), i("auto"), i("auto")], + ], + bounds: [ + [0, 0, 1000, 800], + [20, 20, 120, 40], + ], + paintOrders: [0, 1], + }, + }, + ], + }; + } + if (method === "Runtime.evaluate") return { result: { value: [] } }; + throw new Error(`unexpected CDP method ${method}`); + }); + const res = await handleObserve( + sm, + { session_id: "aa11" }, + { + cdp: { + send: send as unknown as ( + tabId: number, + method: string, + params?: object, + ) => Promise, + trackSessionTab: vi.fn(), + }, + tabsApi: { + get: vi.fn( + async (tabId: number) => + ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, + ), + query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), + }, + }, + ); + + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(send).toHaveBeenCalledWith( + 4, + "Runtime.evaluate", + expect.objectContaining({ returnByValue: true }), + ); + }); + + it("allows passive snapshots of explicit user-window tabs", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + await sm.start("aa11"); + const root: CdpAxNode = { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + name: { type: "computedString", value: "User tab" }, + backendDOMNodeId: 100, + childIds: ["2"], + }; + const button: CdpAxNode = { + nodeId: "2", + parentId: "1", + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Read" }, + backendDOMNodeId: 200, + }; + const deps = makeDeps([root, button]); + deps.tabsApi.get = vi.fn( + async (tabId: number) => ({ id: tabId, windowId: 200, active: true }) as chrome.tabs.Tab, + ); + + const res = await handleSnapshot(sm, { session_id: "aa11", tab_id: 9 }, deps); + + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(res.tab_id).toBe(9); + expect(res.text).toContain('@e1 button "Read"'); + expect(deps.send).toHaveBeenCalledWith(9, "Accessibility.enable", {}); + }); + + it("rejects semantic observe on explicit user-window tabs before CDP probing", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + await sm.start("aa11"); + const deps = makeDeps([]); + deps.tabsApi.get = vi.fn( + async (tabId: number) => ({ id: tabId, windowId: 200, active: true }) as chrome.tabs.Tab, + ); + + const res = await handleObserve(sm, { session_id: "aa11", tab_id: 9 }, deps); + + expect(res).toMatchObject({ + code: "permission_denied", + data: { reason: "agent_window_scope" }, + }); + expect(deps.send).not.toHaveBeenCalled(); + }); + it("resets the RefStore on every fresh snapshot", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); diff --git a/apps/extension/src/tools/__tests__/shared.test.ts b/apps/extension/src/tools/__tests__/shared.test.ts index c8fa77b..ab44ad6 100644 --- a/apps/extension/src/tools/__tests__/shared.test.ts +++ b/apps/extension/src/tools/__tests__/shared.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it, vi } from "vitest"; import { SessionManager } from "@/session-manager/manager"; -import { lookupSession, parseBufferedReadBounds } from "../shared"; +import { + cdpBlockedUrlReason, + enforceCdpAccessibleTarget, + lookupSession, + parseBufferedReadBounds, + resolveCdpAccessibleTargetTab, +} from "../shared"; function fakeAgentWindow() { return { @@ -67,3 +73,101 @@ describe("parseBufferedReadBounds", () => { }); }); }); + +describe("CDP target URL guard", () => { + it("allows ordinary pages and about:blank", () => { + expect(cdpBlockedUrlReason("https://example.test/path")).toBeNull(); + expect(cdpBlockedUrlReason("http://example.test/path")).toBeNull(); + expect(cdpBlockedUrlReason("about:blank")).toBeNull(); + }); + + it("blocks browser and extension internal pages", () => { + expect(cdpBlockedUrlReason("chrome-extension://abc/options.html")).toBe("chrome-extension:"); + expect(cdpBlockedUrlReason("chrome://extensions")).toBe("chrome:"); + expect(cdpBlockedUrlReason("about:newtab")).toBe("about:"); + }); + + it("returns a structured permission error before page CDP access", () => { + expect( + enforceCdpAccessibleTarget( + { + tabId: 7, + windowId: 100, + active: true, + url: "chrome-extension://other-extension/page.html", + }, + "snapshot", + ), + ).toMatchObject({ + code: "permission_denied", + data: { reason: "restricted_tab_url" }, + message: expect.stringContaining("snapshot cannot access tab 7"), + }); + }); + + it("falls back from a restricted default active tab to an accessible agent tab", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow() }); + const ctx = await sm.start("aa11"); + const tabsApi = { + get: vi.fn(), + query: vi.fn(async (query: chrome.tabs.QueryInfo) => { + if (query.active) { + return [ + { + id: 7, + windowId: ctx.agentWindowId, + active: true, + url: "chrome-extension://other-extension/page.html", + } as chrome.tabs.Tab, + ]; + } + return [ + { + id: 7, + windowId: ctx.agentWindowId, + active: true, + url: "chrome-extension://other-extension/page.html", + } as chrome.tabs.Tab, + { + id: 8, + windowId: ctx.agentWindowId, + active: false, + url: "https://example.test/", + } as chrome.tabs.Tab, + ]; + }), + }; + + await expect( + resolveCdpAccessibleTargetTab(sm, ctx, undefined, tabsApi, "snapshot"), + ).resolves.toMatchObject({ + tabId: 8, + url: "https://example.test/", + }); + }); + + it("does not replace an explicitly requested restricted tab", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow() }); + const ctx = await sm.start("aa11"); + const tabsApi = { + get: vi.fn( + async () => + ({ + id: 7, + windowId: ctx.agentWindowId, + active: true, + url: "chrome-extension://other-extension/page.html", + }) as chrome.tabs.Tab, + ), + query: vi.fn(), + }; + + await expect( + resolveCdpAccessibleTargetTab(sm, ctx, 7, tabsApi, "snapshot"), + ).resolves.toMatchObject({ + code: "permission_denied", + data: { reason: "restricted_tab_url" }, + }); + expect(tabsApi.query).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index 17f5456..8f079b3 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -11,6 +11,7 @@ import type { NavigateForwardParams, NavigateParams, NetworkParams, + ObserveParams, PressParams, ProtocolFrame, RecordAwaitParams, @@ -42,6 +43,7 @@ import { type CdpRunner, chromeTabsCaptureApi, handleGetHtml, + handleObserve, handleScreenshot, handleSnapshot, } from "./observation"; @@ -86,6 +88,8 @@ export interface DispatcherDeps { * `chrome.storage.session` "sessions live" flag (review M4/M5 I3). */ onSessionsChanged?: () => void; + /** Invoked before a tool that dispatches page input or mutates browser state is forwarded. */ + onBrowserControlResumed?: (sessionId: string) => void; /** User approval for `tool.tab_borrow` (overlay in content script). */ approveBorrow?: BorrowConfirmationApprover; /** i18n notification copy for `tool.request_help` (resolved per-call). */ @@ -114,6 +118,7 @@ export class ToolDispatcher { private readonly sessions: SessionManager; private readonly cdp?: DispatcherCdpRunner; private readonly onSessionsChanged?: () => void; + private readonly onBrowserControlResumed?: (sessionId: string) => void; private readonly approveBorrow?: BorrowConfirmationApprover; private readonly helpNotificationCopy?: () => { title: string; body: string }; private subscription: { dispose(): void } | null = null; @@ -130,6 +135,7 @@ export class ToolDispatcher { this.sessions = deps.sessions; this.cdp = deps.cdp; this.onSessionsChanged = deps.onSessionsChanged; + this.onBrowserControlResumed = deps.onBrowserControlResumed; this.approveBorrow = deps.approveBorrow; this.helpNotificationCopy = deps.helpNotificationCopy; } @@ -193,6 +199,8 @@ export class ToolDispatcher { let body: ResponseFrame; let startedSession: string | null = null; try { + const sessionId = sessionIdForBrowserControlMethod(req); + if (sessionId) this.onBrowserControlResumed?.(sessionId); const result = await Promise.race([this.invoke(req, ac.signal), abortPromise(ac.signal)]); if (isRpcError(result)) { body = { id: req.id, error: result }; @@ -304,6 +312,12 @@ export class ToolDispatcher { req.params as SnapshotParams, this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined, ); + case "tool.observe": + return handleObserve( + this.sessions, + req.params as ObserveParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined, + ); case "tool.get_html": return handleGetHtml( this.sessions, @@ -457,6 +471,33 @@ function isRpcError(v: unknown): v is RpcError { ); } +function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { + switch (req.method) { + case "tool.tab_create": + case "tool.tab_close": + case "tool.tab_select": + case "tool.tab_borrow": + case "tool.tab_return": + case "tool.navigate": + case "tool.navigate_back": + case "tool.navigate_forward": + case "tool.reload": + case "tool.click": + case "tool.fill": + case "tool.press": + case "tool.select": + case "tool.evaluate": + case "tool.observe": + case "tool.request_help": + case "tool.record_start": { + const sessionId = (req.params as { session_id?: unknown } | undefined)?.session_id; + return typeof sessionId === "string" && sessionId.length > 0 ? sessionId : null; + } + default: + return null; + } +} + /** * Resolves never; rejects with `AbortLikeError` as soon as the signal * fires (or immediately if it is already aborted). Used by the diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index db1b1b0..3c39e33 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -1,5 +1,5 @@ -// Read-only observation handlers — `tool.screenshot`, `tool.snapshot`, -// and `tool.get_html` (design §7). Each handler resolves the target +// Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`, +// and semantic `tool.observe` (design §7). Each handler resolves the target // tab (defaulting to the Agent Window's active tab when omitted) and // returns a payload that mirrors the bsk-protocol Rust structs. @@ -15,6 +15,8 @@ import type { SessionManager } from "@/session-manager/manager"; import type { GetHtmlParams, GetHtmlResult, + ObserveParams, + ObserveResult, RpcError, ScreenshotParams, ScreenshotResult, @@ -26,11 +28,13 @@ import { nodeBoundingRect, scrollNodeIntoView } from "./element-geometry"; import { rpcError } from "./errors"; import { type ChromeTabsApi, + enforceToolTargetScope, isRpcError, lookupSession, - resolveTargetTab, + resolveCdpAccessibleTargetTab, type CdpRunner as SharedCdpRunner, normaliseRef as sharedNormaliseRef, + type ToolEffect, } from "./shared"; import { resolveSnapshotRef } from "./snapshot-ref"; import { @@ -167,7 +171,13 @@ export async function handleScreenshot( const ctxOrErr = lookupSession(manager, params, "screenshot"); if (isRpcError(ctxOrErr)) return ctxOrErr; const ctx = ctxOrErr; - const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + const target = await resolveCdpAccessibleTargetTab( + manager, + ctx, + params.tab_id, + deps.tabsApi, + "screenshot", + ); if (isRpcError(target)) return target; const dialogCursor = deps.cdp ? markDialogCursor(deps.cdp, target.tabId) : 0; const withShotDialogs = (result: T) => @@ -181,6 +191,7 @@ export async function handleScreenshot( const node = resolveSnapshotRef(ctx, ref, target.tabId); if (isRpcError(node)) return node; deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url); const captured = await captureElementScreenshot(deps.cdp, target.tabId, node.backendNodeId); if (isRpcError(captured)) return captured; return withShotDialogs({ @@ -664,6 +675,7 @@ function inputStateFor( value: string | undefined, ): VomNode["inputState"] { if (!capturedNode || !FORM_CONTROL_TAGS.has(normalizeTag(capturedNode.tag))) return undefined; + if (capturedNode.formState) return capturedNode.formState; const formValue = capturedNode.formValue; if (formValue !== undefined) { if (formValue === "") return "empty"; @@ -1099,7 +1111,13 @@ export async function handleGetHtml( const ctxOrErr = lookupSession(manager, params, "get_html"); if (isRpcError(ctxOrErr)) return ctxOrErr; const ctx = ctxOrErr; - const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + const target = await resolveCdpAccessibleTargetTab( + manager, + ctx, + params.tab_id, + deps.tabsApi, + "get_html", + ); if (isRpcError(target)) return target; const dialogCursor = markDialogCursor(deps.cdp, target.tabId); @@ -1108,6 +1126,7 @@ export async function handleGetHtml( try { deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url); let html: string; if (params.ref) { const resolved = resolveSnapshotRef(ctx, params.ref, target.tabId); @@ -1178,28 +1197,44 @@ async function fallbackCapturedViewModel( return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds }; } -async function captureForVom(cdp: CdpRunner, tabId: number): Promise { +async function captureForVom( + cdp: CdpRunner, + tabId: number, + conditionalSurfaceProbe: boolean, +): Promise { try { - return await captureViewModel(cdp, tabId); + return await captureViewModel(cdp, tabId, { conditionalSurfaceProbe }); } catch { return fallbackCapturedViewModel(cdp, tabId); } } -export async function handleSnapshot( +async function handleVomObservation( manager: SessionManager, - params: SnapshotParams, + params: SnapshotParams | ObserveParams, + toolName: "snapshot" | "observe", + effect: ToolEffect, + conditionalSurfaceProbe: boolean, deps: SnapshotDeps = getDefaultDeps(), -): Promise { - const ctxOrErr = lookupSession(manager, params, "snapshot"); +): Promise { + const ctxOrErr = lookupSession(manager, params, toolName); if (isRpcError(ctxOrErr)) return ctxOrErr; const ctx = ctxOrErr; - const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + const target = await resolveCdpAccessibleTargetTab( + manager, + ctx, + params.tab_id, + deps.tabsApi, + toolName, + ); if (isRpcError(target)) return target; + const denied = enforceToolTargetScope(ctx, target, effect, toolName); + if (denied) return denied; const dialogCursor = markDialogCursor(deps.cdp, target.tabId); try { deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url); await deps.cdp.send(target.tabId, "Accessibility.enable", {}); const result = await deps.cdp.send<{ nodes: CdpAxNode[] }>( target.tabId, @@ -1207,7 +1242,7 @@ export async function handleSnapshot( {}, ); const axNodes = result.nodes ?? []; - const captured = await captureForVom(deps.cdp, target.tabId); + const captured = await captureForVom(deps.cdp, target.tabId, conditionalSurfaceProbe); const scene = buildVomScene(axNodes, captured); const rendered = renderVom(scene, { maxDepth: params.max_depth, @@ -1232,3 +1267,19 @@ export async function handleSnapshot( }; } } + +export async function handleSnapshot( + manager: SessionManager, + params: SnapshotParams, + deps: SnapshotDeps = getDefaultDeps(), +): Promise { + return handleVomObservation(manager, params, "snapshot", "passive_read", false, deps); +} + +export async function handleObserve( + manager: SessionManager, + params: ObserveParams, + deps: SnapshotDeps = getDefaultDeps(), +): Promise { + return handleVomObservation(manager, params, "observe", "transient_input", true, deps); +} diff --git a/apps/extension/src/tools/record.ts b/apps/extension/src/tools/record.ts index 4f3dde3..e7c8b31 100644 --- a/apps/extension/src/tools/record.ts +++ b/apps/extension/src/tools/record.ts @@ -369,7 +369,7 @@ async function rearmRecording( deps: RecordDeps, ): Promise { // Do NOT toggle automation-bypass here: each retry used to increment the - // content-script counter, and a single stop decrement left the ControlOverlay + // content-script counter, and a single stop decrement left the control mask // stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay // already hides the control chrome while activeRecord is set. for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) { @@ -561,9 +561,9 @@ export async function handleRecordStart( const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); if (isRpcError(target)) return target; - // Register the recording *before* navigate so content-script syncAgentOverlay - // on the destination page can RECORD_QUERY → rearm → show RecordOverlay - // instead of flashing ControlOverlay ("Agent 正在控制"). + // Register the recording *before* navigate so the destination content script's + // overlay.ready flow can RECORD_QUERY → rearm → show RecordOverlay + // instead of flashing the control mask ("Agent 正在控制"). const requestId = makeRequestId(target.tabId); let resolveFinish!: (trace: Trace) => void; let rejectFinish!: (err: Error) => void; diff --git a/apps/extension/src/tools/shared.ts b/apps/extension/src/tools/shared.ts index 00ecde1..ef1f191 100644 --- a/apps/extension/src/tools/shared.ts +++ b/apps/extension/src/tools/shared.ts @@ -32,6 +32,13 @@ export const chromeTabsApi: ChromeTabsApi = { query: (q) => chrome.tabs.query(q), }; +export interface ResolvedTargetTab { + tabId: number; + windowId: number; + active: boolean; + url?: string; +} + export type { DialogCursor }; /** @@ -44,6 +51,7 @@ export type { DialogCursor }; */ export interface CdpRunner { send(tabId: number, method: string, params?: object): Promise; + ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise; trackSessionTab?(sessionId: string, tabId: number): void; onEvent?(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): { dispose(): void; @@ -146,7 +154,7 @@ export async function resolveTargetTab( ctx: SessionContext, tabId: number | undefined, api: ChromeTabsApi, -): Promise<{ tabId: number; windowId: number; active: boolean } | RpcError> { +): Promise { if (tabId !== undefined) { if (!Number.isSafeInteger(tabId) || tabId <= 0) { return { @@ -176,7 +184,7 @@ export async function resolveTargetTab( message: `tab ${tabId} not found in session scope`, }; } - return { tabId: tab.id, windowId: tab.windowId, active: tab.active === true }; + return { tabId: tab.id, windowId: tab.windowId, active: tab.active === true, url: tab.url }; } const tabs = await api.query({ active: true, windowId: ctx.agentWindowId }); const first = tabs.find((t) => typeof t.id === "number"); @@ -186,7 +194,12 @@ export async function resolveTargetTab( message: `no active tab in Agent Window ${ctx.agentWindowId}`, }; } - return { tabId: first.id, windowId: ctx.agentWindowId, active: first.active === true }; + return { + tabId: first.id, + windowId: ctx.agentWindowId, + active: first.active === true, + url: first.url, + }; } export function isRpcError(v: unknown): v is RpcError { @@ -202,6 +215,91 @@ export function isRpcError(v: unknown): v is RpcError { /** Re-export so M6/M7 tools keep a stable import path. */ export { normaliseRef }; +export type ToolEffect = "passive_read" | "transient_input" | "browser_mutation"; + +const CDP_BLOCKED_PROTOCOLS = new Set([ + "chrome:", + "chrome-extension:", + "devtools:", + "edge:", + "brave:", + "vivaldi:", + "opera:", +]); + +export function cdpBlockedUrlReason(url: string | undefined): string | null { + if (!url) return null; + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return null; + } + if (CDP_BLOCKED_PROTOCOLS.has(parsed.protocol)) return parsed.protocol; + if (parsed.protocol === "about:" && parsed.pathname !== "blank") return "about:"; + return null; +} + +/** + * Page CDP tools cannot inspect browser/extension internal pages. Check this + * at the tool boundary so Chrome's low-level "Cannot access..." errors do not + * leak as ambiguous page-read failures. + */ +export function enforceCdpAccessibleTarget( + target: ResolvedTargetTab, + toolName: string, +): RpcError | null { + const reason = cdpBlockedUrlReason(target.url); + if (!reason) return null; + return rpcError( + "permission_denied", + "restricted_tab_url", + `${toolName} cannot access tab ${target.tabId} because its URL is ${target.url}; navigate the Agent Window to a web page first`, + ); +} + +function resolvedTargetFromChromeTab( + tab: chrome.tabs.Tab, + fallbackWindowId: number, +): ResolvedTargetTab | null { + if (typeof tab.id !== "number") return null; + return { + tabId: tab.id, + windowId: typeof tab.windowId === "number" ? tab.windowId : fallbackWindowId, + active: tab.active === true, + url: tab.url, + }; +} + +/** + * Resolve a target for page CDP reads. Explicit tab_ids stay exact and are + * rejected when browser security blocks the URL. For default targeting, skip a + * restricted Agent Window active tab if another CDP-accessible tab exists in + * the same Agent Window. + */ +export async function resolveCdpAccessibleTargetTab( + manager: SessionManager, + ctx: SessionContext, + tabId: number | undefined, + api: ChromeTabsApi, + toolName: string, +): Promise { + const target = await resolveTargetTab(manager, ctx, tabId, api); + if (isRpcError(target)) return target; + + const restricted = enforceCdpAccessibleTarget(target, toolName); + if (!restricted) return target; + if (tabId !== undefined) return restricted; + + const tabs = await api.query({ windowId: ctx.agentWindowId }); + for (const tab of tabs) { + const candidate = resolvedTargetFromChromeTab(tab, ctx.agentWindowId); + if (!candidate) continue; + if (!enforceCdpAccessibleTarget(candidate, toolName)) return candidate; + } + return restricted; +} + /** * Sandbox guard: M7 write tools (click / fill / press / navigate*) * MUST refuse to touch a tab outside the session's Agent Window @@ -224,3 +322,17 @@ export function enforceAgentWindow( } return null; } + +/** + * Unified target-scope policy by tool effect. Passive reads may inspect user + * tabs; any tool that dispatches page input must stay inside the Agent Window. + */ +export function enforceToolTargetScope( + ctx: SessionContext, + target: { tabId: number; windowId: number }, + effect: ToolEffect, + toolName: string, +): RpcError | null { + if (effect === "passive_read") return null; + return enforceAgentWindow(ctx, target, toolName); +} diff --git a/apps/extension/src/tools/vom/__tests__/capture.test.ts b/apps/extension/src/tools/vom/__tests__/capture.test.ts index f962a4a..2fe1f2e 100644 --- a/apps/extension/src/tools/vom/__tests__/capture.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture.test.ts @@ -59,6 +59,23 @@ function makeCdp(snapshot: unknown) { if (method === "Page.getLayoutMetrics") { return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } }; } + if (method === "Runtime.evaluate") { + return { + result: { + value: { + controls: [ + { + state: "filled", + sensitive: true, + defaultValue: "", + placeholder: "secret", + }, + ], + childFrames: [], + }, + }, + }; + } throw new Error(`unexpected ${method}`); }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; @@ -80,6 +97,89 @@ describe("captureViewModel", () => { expect(input?.attrs).toEqual({ type: "password" }); }); + it("does not run conditional surface probes by default", async () => { + const cdp = makeCdp(fakeSnapshotReply()); + const result = await captureViewModel(cdp, 4); + + expect(result.surfaceProbes).toEqual([]); + expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything()); + }); + + it("batch-enriches form controls without per-node object resolution", async () => { + const cdp = makeCdp(fakeSnapshotReply()); + const result = await captureViewModel(cdp, 4); + const input = result.nodes.find((node) => node.backendNodeId === 13); + + expect(input).toMatchObject({ + formState: "filled", + formDefaultValue: "", + formPlaceholder: "secret", + }); + expect(input?.formValue).toBeUndefined(); + expect(cdp.send).toHaveBeenCalledWith( + 4, + "Runtime.evaluate", + expect.objectContaining({ returnByValue: true }), + ); + expect(cdp.send).not.toHaveBeenCalledWith(4, "DOM.resolveNode", expect.anything()); + expect(cdp.send).not.toHaveBeenCalledWith(4, "Runtime.callFunctionOn", expect.anything()); + }); + + it("runs hover surface probes when explicitly enabled", async () => { + let runtimeCalls = 0; + const cdp = { + send: vi.fn(async (_tabId: number, method: string, params?: object) => { + if (method === "DOMSnapshot.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply(); + if (method === "Page.getLayoutMetrics") { + return { + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, + }; + } + if (method === "Runtime.evaluate") { + const expression = (params as { expression?: string } | undefined)?.expression ?? ""; + if (expression.includes("input,textarea,select")) { + return { result: { value: { controls: [], childFrames: [] } } }; + } + runtimeCalls += 1; + return runtimeCalls === 1 + ? { + result: { + value: [ + { + triggerSel: ".menu", + affectedSub: " .items", + label: "Products", + x: 50, + y: 20, + }, + ], + }, + } + : { result: { value: ["Shoes", "Bags"] } }; + } + if (method === "Input.dispatchMouseEvent") return {}; + throw new Error(`unexpected ${method}`); + }) as unknown as (tabId: number, method: string, params?: object) => Promise, + }; + + const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true }); + + expect(result.surfaceProbes).toEqual([ + { triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] }, + ]); + expect(cdp.send).toHaveBeenCalledWith( + 4, + "Input.dispatchMouseEvent", + expect.objectContaining({ type: "mouseMoved", x: 50, y: 20 }), + ); + expect(cdp.send).toHaveBeenCalledWith( + 4, + "Input.dispatchMouseEvent", + expect.objectContaining({ type: "mouseMoved", x: -10, y: -10 }), + ); + }); + it("excludes the agent's own overlay shadow host and its inlined shadow subtree", async () => { // DOMSnapshot inlines an open shadow root's content as descendants of the // host. The agent's WXT overlay host carries a fixed full-viewport @@ -349,8 +449,11 @@ describe("captureViewModel", () => { }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; const { nodes } = await captureViewModel(cdp, 4); - // overlay div bounds y=0 minus scroll pageY=200 → y=-200 - expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(-200); + const div = nodes.find((n) => n.backendNodeId === 12); + // local rect preserves viewport-relative scroll math. + expect(div?.localRect?.y).toBe(-200); + // exported rect is clipped to the top-level viewport before VOM consumes it. + expect(div?.rect).toMatchObject({ y: 0, h: 600 }); }); it("normalizes device-pixel bounds by devicePixelRatio", async () => { @@ -557,8 +660,187 @@ describe("captureViewModel", () => { expect(iframeNodes.size).toBe(1); expect(iframeNodes.has(13)).toBe(true); const subNodes = iframeNodes.get(13)!; - expect(subNodes.find((n) => n.backendNodeId === 101)?.tag).toBe("input"); - expect(subNodes.find((n) => n.backendNodeId === 101)?.attrs.type).toBe("text"); + const input = subNodes.find((n) => n.backendNodeId === 101); + expect(input?.tag).toBe("input"); + expect(input?.attrs.type).toBe("text"); + expect(input?.ownerFrameBackendNodeId).toBe(13); + expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 }); + expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 }); + }); + + it("clips iframe sub-document rects to the iframe viewport in top coordinates", async () => { + const S = [ + "html", + "body", + "iframe", + "input", + "position", + "static", + "pointer-events", + "auto", + "type", + "text", + ]; + const i = (s: string) => S.indexOf(s); + const snapshot = { + strings: S, + documents: [ + { + scrollOffsetX: 0, + scrollOffsetY: 0, + nodes: { + parentIndex: [-1, 0, 1], + nodeName: [i("html"), i("body"), i("iframe")], + backendNodeId: [10, 11, 13], + attributes: [[], [], []], + contentDocumentIndex: { index: [2], value: [1] }, + }, + layout: { + nodeIndex: [1, 2], + styles: [ + [i("static"), i("auto")], + [i("static"), i("auto")], + ], + bounds: [ + [0, 0, 1000, 800], + [950, 760, 200, 100], + ], + paintOrders: [0, 1], + }, + }, + { + scrollOffsetX: 0, + scrollOffsetY: 0, + nodes: { + parentIndex: [-1, 0], + nodeName: [i("body"), i("input")], + backendNodeId: [100, 101], + attributes: [[], [i("type"), i("text")]], + }, + layout: { + nodeIndex: [1], + styles: [[i("static"), i("auto")]], + bounds: [[20, 20, 120, 60]], + paintOrders: [0], + }, + }, + ], + }; + const cdp = { + send: vi.fn(async (_t: number, method: string) => { + if (method === "DOMSnapshot.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") return snapshot; + if (method === "Page.getLayoutMetrics") { + return { + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, + }; + } + throw new Error(method); + }) as unknown as (tabId: number, method: string, params?: object) => Promise, + }; + + const { iframeNodes } = await captureViewModel(cdp, 4); + const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101); + + expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 }); + expect(input?.rect).toEqual({ x: 970, y: 780, w: 30, h: 20 }); + }); + + it("recursively normalizes nested iframe sub-documents", async () => { + const S = [ + "html", + "body", + "iframe", + "input", + "position", + "static", + "pointer-events", + "auto", + "type", + "text", + ]; + const i = (s: string) => S.indexOf(s); + const snapshot = { + strings: S, + documents: [ + { + nodes: { + parentIndex: [-1, 0, 1], + nodeName: [i("html"), i("body"), i("iframe")], + backendNodeId: [10, 11, 13], + attributes: [[], [], []], + contentDocumentIndex: { index: [2], value: [1] }, + }, + layout: { + nodeIndex: [1, 2], + styles: [ + [i("static"), i("auto")], + [i("static"), i("auto")], + ], + bounds: [ + [0, 0, 1000, 800], + [10, 20, 300, 200], + ], + paintOrders: [0, 1], + }, + }, + { + nodes: { + parentIndex: [-1, 0, 1], + nodeName: [i("html"), i("body"), i("iframe")], + backendNodeId: [20, 21, 23], + attributes: [[], [], []], + contentDocumentIndex: { index: [2], value: [2] }, + }, + layout: { + nodeIndex: [1, 2], + styles: [ + [i("static"), i("auto")], + [i("static"), i("auto")], + ], + bounds: [ + [0, 0, 300, 200], + [5, 6, 100, 80], + ], + paintOrders: [0, 1], + }, + }, + { + nodes: { + parentIndex: [-1, 0], + nodeName: [i("body"), i("input")], + backendNodeId: [30, 31], + attributes: [[], [i("type"), i("text")]], + }, + layout: { + nodeIndex: [1], + styles: [[i("static"), i("auto")]], + bounds: [[1, 2, 40, 20]], + paintOrders: [0], + }, + }, + ], + }; + const cdp = { + send: vi.fn(async (_t: number, method: string) => { + if (method === "DOMSnapshot.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") return snapshot; + if (method === "Page.getLayoutMetrics") { + return { + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, + }; + } + throw new Error(method); + }) as unknown as (tabId: number, method: string, params?: object) => Promise, + }; + + const { iframeNodes } = await captureViewModel(cdp, 4); + const nestedIframe = iframeNodes.get(13)?.find((n) => n.backendNodeId === 23); + const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31); + + expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 }); + expect(input?.ownerFrameBackendNodeId).toBe(23); + expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 }); }); it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => { diff --git a/apps/extension/src/tools/vom/capture.ts b/apps/extension/src/tools/vom/capture.ts index e4be568..d8358f5 100644 --- a/apps/extension/src/tools/vom/capture.ts +++ b/apps/extension/src/tools/vom/capture.ts @@ -16,9 +16,14 @@ const STYLE_COL = Object.fromEntries( export interface CapturedNode { backendNodeId: number; parentBackendNodeId: number | null; + /** Owning iframe backend node id; `null` for the top-level document. */ + ownerFrameBackendNodeId?: number | null; tag: string; attrs: Record; + /** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */ rect: Rect | null; + /** Frame-local viewport-relative CSS px before top-level translation/clipping. */ + localRect?: Rect | null; paintOrder: number; position: string; pointerEvents: string; @@ -31,6 +36,7 @@ export interface CapturedNode { */ cursor?: string; textContent?: string; + formState?: "empty" | "filled" | "default"; formValue?: string; formDefaultValue?: string; formPlaceholder?: string; @@ -53,6 +59,10 @@ export interface CapturedViewModel { excludedBackendNodeIds: Set; } +export interface CaptureViewModelOptions { + conditionalSurfaceProbe?: boolean; +} + /** Sparse array format Chrome uses for infrequently-set per-node fields. */ interface SparseArray { index: number[]; @@ -102,48 +112,112 @@ function isFormControlTag(tag: string): boolean { return tag === "input" || tag === "textarea" || tag === "select"; } -async function enrichFormControlState(cdp: CdpRunner, tabId: number, nodes: CapturedNode[]) { +const MAX_FORM_ENRICH_CONTROLS = 250; + +interface CapturedFormState { + value?: string; + defaultValue?: string; + placeholder?: string; + state?: "empty" | "filled" | "default"; + sensitive?: boolean; +} + +interface CapturedFrameFormState { + controls: CapturedFormState[]; + childFrames: CapturedFrameFormState[]; +} + +function formStateBatchExpression(maxControls: number): string { + return `(() => { + const maxControls = ${JSON.stringify(maxControls)}; + let remaining = maxControls; + const controlSelector = "input,textarea,select"; + const controlState = (el) => { + const tag = el.tagName.toLowerCase(); + const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag; + const sensitive = type === "password" || type === "credit-card"; + const rawValue = typeof el.value === "string" ? el.value : ""; + const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : ""; + const placeholder = typeof el.placeholder === "string" ? el.placeholder : ""; + const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled"; + return { + state, + sensitive, + defaultValue, + placeholder, + ...(sensitive ? {} : { value: rawValue }), + }; + }; + const collect = (doc) => { + const controls = []; + for (const el of Array.from(doc.querySelectorAll(controlSelector))) { + if (remaining <= 0) break; + controls.push(controlState(el)); + remaining -= 1; + } + const childFrames = []; + for (const frame of Array.from(doc.querySelectorAll("iframe"))) { + let childDoc = null; + try { childDoc = frame.contentDocument; } catch { childDoc = null; } + childFrames.push(childDoc && remaining > 0 ? collect(childDoc) : { controls: [], childFrames: [] }); + } + return { controls, childFrames }; + }; + return collect(document); + })()`; +} + +function flattenFrameFormStates( + frame: CapturedFrameFormState | undefined, + out: CapturedFormState[][] = [], +): CapturedFormState[][] { + if (!frame) return out; + out.push(Array.isArray(frame.controls) ? frame.controls : []); + for (const child of Array.isArray(frame.childFrames) ? frame.childFrames : []) { + flattenFrameFormStates(child, out); + } + return out; +} + +function applyFormStates(nodes: CapturedNode[], states: CapturedFormState[]): void { + let index = 0; for (const node of nodes) { if (!isFormControlTag(node.tag)) continue; - let objectId: string | undefined; - try { - const resolved = await cdp.send<{ object?: { objectId?: string } }>( - tabId, - "DOM.resolveNode", - { - backendNodeId: node.backendNodeId, - }, - ); - objectId = resolved.object?.objectId; - if (!objectId) continue; - const result = await cdp.send<{ - result?: { value?: { value?: string; defaultValue?: string; placeholder?: string } }; - }>(tabId, "Runtime.callFunctionOn", { - objectId, - returnByValue: true, - functionDeclaration: `function(){ - return { - value: typeof this.value === "string" ? this.value : "", - defaultValue: typeof this.defaultValue === "string" ? this.defaultValue : "", - placeholder: typeof this.placeholder === "string" ? this.placeholder : "" - }; - }`, - }); - const value = result.result?.value; - if (!value) continue; - node.formValue = value.value ?? ""; - node.formDefaultValue = value.defaultValue ?? ""; - node.formPlaceholder = value.placeholder ?? ""; - } catch { - // Best-effort enrichment. DOMSnapshot/AX data still carries the node. - } finally { - if (objectId) { - await cdp.send(tabId, "Runtime.releaseObject", { objectId }).catch(() => {}); - } + const state = states[index]; + index += 1; + if (!state) continue; + node.formState = state.state; + node.formDefaultValue = state.defaultValue ?? ""; + node.formPlaceholder = state.placeholder ?? ""; + if (!state.sensitive && state.value !== undefined) { + node.formValue = state.value; } } } +async function enrichFormControlStates( + cdp: CdpRunner, + tabId: number, + frameNodeGroups: CapturedNode[][], +): Promise { + const hasControls = frameNodeGroups.some((nodes) => + nodes.some((node) => isFormControlTag(node.tag)), + ); + if (!hasControls) return; + try { + const result = await cdp.send(tabId, "Runtime.evaluate", { + expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS), + returnByValue: true, + }); + const frameStates = flattenFrameFormStates(runtimeValue(result)); + for (let i = 0; i < frameNodeGroups.length; i += 1) { + applyFormStates(frameNodeGroups[i], frameStates[i] ?? []); + } + } catch { + // Best-effort enrichment. DOMSnapshot/AX data still carries the nodes. + } +} + interface RuntimeEvaluateReply { result?: { value?: unknown; @@ -169,6 +243,14 @@ interface ParseDocumentResult { excludedBackendNodeIds: Set; } +interface FrameContext { + ownerFrameBackendNodeId: number | null; + originInTop: { x: number; y: number }; + clipRectInTop: Rect; + scrollX: number; + scrollY: number; +} + function str(strings: string[], idx: number | undefined): string { if (idx === undefined || idx < 0) return ""; return strings[idx] ?? ""; @@ -183,6 +265,49 @@ function devicePixelRatio(metrics: LayoutMetricsReply): number { return dpr >= 1 ? dpr : 1; } +function viewportRect(viewport: Viewport): Rect { + return { x: 0, y: 0, w: Math.max(0, viewport.width), h: Math.max(0, viewport.height) }; +} + +function intersectRects(a: Rect, b: Rect): Rect | null { + const x1 = Math.max(a.x, b.x); + const y1 = Math.max(a.y, b.y); + const x2 = Math.min(a.x + a.w, b.x + b.w); + const y2 = Math.min(a.y + a.h, b.y + b.h); + if (x2 <= x1 || y2 <= y1) return null; + return { x: x1, y: y1, w: x2 - x1, h: y2 - y1 }; +} + +function rectInTop(localRect: Rect, context: FrameContext): Rect | null { + const translated = { + x: context.originInTop.x + localRect.x, + y: context.originInTop.y + localRect.y, + w: localRect.w, + h: localRect.h, + }; + return intersectRects(translated, context.clipRectInTop); +} + +function unclipRectInTop(localRect: Rect, context: FrameContext): Rect { + return { + x: context.originInTop.x + localRect.x, + y: context.originInTop.y + localRect.y, + w: localRect.w, + h: localRect.h, + }; +} + +function sparseIndexMap(sparse: SparseArray | undefined): Map { + const out = new Map(); + if (!sparse?.index || !sparse.value) return out; + for (let i = 0; i < sparse.index.length; i++) { + const nodeIndex = sparse.index[i]; + const docIndex = sparse.value[i]; + if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex); + } + return out; +} + function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set): void { if (!node) return; if (typeof node.backendNodeId === "number") { @@ -235,14 +360,24 @@ function hoverCssScanExpression(): string { const candidates = []; const seenLabels = new Set(); + const isUnsafeTrigger = (el) => { + if (!(el instanceof HTMLElement)) return true; + const tag = el.tagName.toLowerCase(); + if (["input", "textarea", "select", "option"].includes(tag)) return true; + if (el.isContentEditable) return true; + if (el.hasAttribute("disabled") || el.hasAttribute("inert")) return true; + if ((el.getAttribute("aria-disabled") || "").toLowerCase() === "true") return true; + return false; + }; for (const pair of pairs) { let elements; try { elements = Array.from(document.querySelectorAll(pair.triggerSel)); } catch { continue; } for (const el of elements) { if (!(el instanceof Element)) continue; + if (isUnsafeTrigger(el)) continue; const rect = el.getBoundingClientRect(); const style = getComputedStyle(el); - if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none") continue; + if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue; const label = (el.textContent || "").replace(/\\s+/g, " ").trim(); if (!label || seenLabels.has(label)) continue; seenLabels.add(label); @@ -304,6 +439,20 @@ async function wait(ms: number): Promise { await new Promise((resolve) => setTimeout(resolve, ms)); } +async function clearHover(cdp: CdpRunner, tabId: number): Promise { + await cdp + .send(tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: -10, + y: -10, + }) + .catch(() => + cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 }).catch(() => { + // best effort + }), + ); +} + async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise { const started = Date.now(); try { @@ -328,7 +477,6 @@ async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise(collected) ?? []) .map((item) => item.replace(/\s+/g, " ").trim()) .filter(Boolean); @@ -341,6 +489,8 @@ async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise= 4 && b[2] > 0 && b[3] > 0) { - rect = { - x: b[0] / dpr - scrollX, - y: b[1] / dpr - scrollY, + localRect = { + x: b[0] / dpr - context.scrollX, + y: b[1] / dpr - context.scrollY, w: b[2] / dpr, h: b[3] / dpr, }; + rect = rectInTop(localRect, context); } paintOrder = dl?.paintOrders?.[li] ?? 0; const styleRow = dl?.styles?.[li] ?? []; @@ -508,9 +658,11 @@ function parseDocumentNodes( nodes.push({ backendNodeId, parentBackendNodeId, + ownerFrameBackendNodeId: context.ownerFrameBackendNodeId, tag, attrs, rect, + localRect, paintOrder, position, pointerEvents, @@ -521,7 +673,81 @@ function parseDocumentNodes( return { nodes, excludedBackendNodeIds }; } -export async function captureViewModel(cdp: CdpRunner, tabId: number): Promise { +interface ParseFrameDocumentsResult { + iframeNodes: CapturedIframeNodes; + excludedBackendNodeIds: Set; +} + +function parseChildFrameDocuments( + documents: SnapshotDocument[], + strings: string[], + dpr: number, + parentDocIndex: number, + parentNodes: CapturedNode[], + parentContext: FrameContext, + visited = new Set(), +): ParseFrameDocumentsResult { + const iframeNodes: CapturedIframeNodes = new Map(); + const excludedBackendNodeIds = new Set(); + const parentDoc = documents[parentDocIndex]; + const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex); + if (cdi.size === 0) return { iframeNodes, excludedBackendNodeIds }; + + const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? []; + const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node])); + + for (const [nodeArrayIdx, childDocIndex] of cdi) { + if (visited.has(childDocIndex)) continue; + const childDoc = documents[childDocIndex]; + if (!childDoc) continue; + const iframeBackendId = parentBackendIds[nodeArrayIdx]; + if (iframeBackendId === undefined) continue; + const iframeNode = parentNodeByBackendId.get(iframeBackendId); + if (!iframeNode?.localRect) continue; + + const iframeRectInTop = unclipRectInTop(iframeNode.localRect, parentContext); + const childClip = intersectRects(iframeRectInTop, parentContext.clipRectInTop); + if (!childClip) { + iframeNodes.set(iframeBackendId, []); + continue; + } + + const childContext: FrameContext = { + ownerFrameBackendNodeId: iframeBackendId, + originInTop: { x: iframeRectInTop.x, y: iframeRectInTop.y }, + clipRectInTop: childClip, + scrollX: childDoc.scrollOffsetX ?? 0, + scrollY: childDoc.scrollOffsetY ?? 0, + }; + const nextVisited = new Set(visited); + nextVisited.add(childDocIndex); + const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext); + iframeNodes.set(iframeBackendId, parsed.nodes); + for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id); + + const nested = parseChildFrameDocuments( + documents, + strings, + dpr, + childDocIndex, + parsed.nodes, + childContext, + nextVisited, + ); + for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) { + iframeNodes.set(nestedFrameId, nestedNodes); + } + for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id); + } + + return { iframeNodes, excludedBackendNodeIds }; +} + +export async function captureViewModel( + cdp: CdpRunner, + tabId: number, + options: CaptureViewModelOptions = {}, +): Promise { const metrics = await cdp.send(tabId, "Page.getLayoutMetrics", {}); const dpr = devicePixelRatio(metrics); const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {}; @@ -552,42 +778,26 @@ export async function captureViewModel(cdp: CdpRunner, tabId: number): Promise nodes. - const iframeNodes: CapturedIframeNodes = new Map(); - if (documents.length > 1) { - const cdi = doc0.nodes?.contentDocumentIndex; - if (cdi?.index && cdi.value) { - for (let k = 0; k < cdi.index.length; k++) { - const nodeArrayIdx = cdi.index[k]; - const docIdx = cdi.value[k]; - const subDoc = documents[docIdx]; - if (!subDoc) continue; - const iframeBid = doc0.nodes?.backendNodeId?.[nodeArrayIdx]; - if (iframeBid === undefined) continue; - // Sub-documents have their own scroll offsets; bounds are relative - // to the iframe's own coordinate space (not the main viewport). - const subScrollX = subDoc.scrollOffsetX ?? 0; - const subScrollY = subDoc.scrollOffsetY ?? 0; - const subParsed = parseDocumentNodes(subDoc, strings, dpr, subScrollX, subScrollY); - iframeNodes.set(iframeBid, subParsed.nodes); - for (const id of subParsed.excludedBackendNodeIds) { - excludedBackendNodeIds.add(id); - } - } - } + const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext); + const iframeNodes = frameParsed.iframeNodes; + for (const id of frameParsed.excludedBackendNodeIds) { + excludedBackendNodeIds.add(id); } - await enrichFormControlState(cdp, tabId, nodes); - for (const subNodes of iframeNodes.values()) { - await enrichFormControlState(cdp, tabId, subNodes); - } + await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()]); - const surfaceProbes = await probeHoverSurfaces(cdp, tabId); + const surfaceProbes = options.conditionalSurfaceProbe ? await probeHoverSurfaces(cdp, tabId) : []; return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds }; } diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index 6ed8106..4bc2d61 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -30,6 +30,7 @@ export type RpcErrorReason = | "option_not_found" | "single_select_value_count" | "tab_not_active" + | "restricted_tab_url" | "borrow_conflict"; export interface RpcErrorData { @@ -313,6 +314,9 @@ export interface SnapshotResult { dialogs?: JavaScriptDialogInfo[]; } +export type ObserveParams = SnapshotParams; +export type ObserveResult = SnapshotResult; + export interface GetHtmlParams { session_id: string; tab_id?: number; diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index 3e5599a..3290d90 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -68,6 +68,7 @@ Write operations only affect tabs in the **Agent Window** (or tabs you **borrowe ``` bsk navigate --session bsk snapshot --session → aria tree with @e1, @e2, … refs +bsk observe --session → semantic VOM view; may reveal hover/focus surfaces bsk click @e3 --session → or bsk fill, bsk select, bsk press bsk snapshot --session → again after navigation / DOM change ``` @@ -78,11 +79,12 @@ Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / ## Observation priority -Start with `bsk snapshot` to understand page structure, text, controls, and element refs. Only escalate when the latest snapshot cannot answer the question: +Start with `bsk snapshot` to understand page structure, text, controls, and element refs. Use `bsk observe` when semantic VOM output or conditional hover/focus surfaces would materially help. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question: -1. `bsk snapshot` — default for page understanding and interaction planning -2. `bsk get-html` — when hidden DOM, metadata, or markup details are required -3. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the snapshot. Use `--ref @eN` (from the latest snapshot) to crop to one element; omit `--ref` for the full visible tab. +1. `bsk snapshot` — strict static page understanding and interaction planning +2. `bsk observe` — semantic VOM observation; may run bounded perception probes such as hover-surface discovery +3. `bsk get-html` — when hidden DOM, metadata, or markup details are required +4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab. Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page. @@ -141,7 +143,8 @@ Details and flags: **`bsk --help`** | Command | Summary | |---------|---------| -| `bsk snapshot` | First-choice page understanding: accessibility tree with `@eN` element refs | +| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs | +| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces | | `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) | | `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) | @@ -260,7 +263,7 @@ Always **`bsk session stop `** in a `finally`-style path so the Agent Window 2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks. 3. **No skip stop** — always `bsk session stop `; never assume idle timeout will clean up. 4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open. -5. **No observe escalation before snapshot** — use `bsk snapshot` first; only use `bsk get-html` or `bsk screenshot` when the snapshot is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot ref — never skip snapshot just to grab a visual. +5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics or conditional surfaces help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual. 6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces. --- diff --git a/crates/bsk-cli/src/cli/mod.rs b/crates/bsk-cli/src/cli/mod.rs index 9e8e48a..ff1e3a6 100644 --- a/crates/bsk-cli/src/cli/mod.rs +++ b/crates/bsk-cli/src/cli/mod.rs @@ -19,6 +19,7 @@ pub mod interaction; pub mod logs; pub mod navigate; pub mod network; +pub mod observe; pub mod record; pub mod record_state; pub mod render_error; @@ -41,6 +42,7 @@ use crate::cli::install_skill::InstallSkillArgs; use crate::cli::interaction::{ClickArgs, FillArgs, PressArgs, SelectArgs}; use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs}; use crate::cli::network::NetworkArgs; +use crate::cli::observe::ObserveArgs; use crate::cli::record::RecordCmd; use crate::cli::screenshot::ScreenshotArgs; use crate::cli::session::SessionCmd; @@ -122,6 +124,9 @@ pub enum Command { /// Produce an indented aria-snapshot with @eN refs. Snapshot(SnapshotArgs), + /// Produce a semantic VOM observation with perception probes. + Observe(ObserveArgs), + /// Read buffered console/log/exception messages. Console(ConsoleArgs), diff --git a/crates/bsk-cli/src/cli/observe.rs b/crates/bsk-cli/src/cli/observe.rs new file mode 100644 index 0000000..72efa28 --- /dev/null +++ b/crates/bsk-cli/src/cli/observe.rs @@ -0,0 +1,79 @@ +//! `bsk observe` — semantic VOM observation with bounded perception probes. + +use std::path::PathBuf; + +use anyhow::Context; +use bsk_protocol::Method; +use bsk_protocol::tools::{ObserveParams, ObserveResult}; +use clap::Args; + +use crate::cli::TOOL_IPC_TIMEOUT; +use crate::cli::dialogs::print_dialog_summaries; +use crate::cli::ensure_daemon::ensure_daemon; +use crate::cli::error::{CliError, Format}; + +#[derive(Debug, Clone, Args)] +pub struct ObserveArgs { + /// Session id (must be active). + #[arg(long)] + pub session: String, + + /// Target tab. Defaults to the Agent Window's active tab. + #[arg(long = "tab-id")] + pub tab_id: Option, + + /// Cap on VOM tree depth before truncating. + #[arg(long = "max-depth")] + pub max_depth: Option, + + /// Soft cap on rendered tokens (~4 chars/token). + #[arg(long = "max-tokens")] + pub max_tokens: Option, +} + +pub fn dispatch(args: ObserveArgs, format: Format) -> Result<(), CliError> { + let info = ensure_daemon().context("ensure daemon is running")?; + run(info.sock_path, args, format) +} + +fn run(sock: PathBuf, args: ObserveArgs, format: Format) -> Result<(), CliError> { + let params = ObserveParams { + session_id: args.session.clone(), + tab_id: args.tab_id, + max_depth: args.max_depth, + max_tokens: args.max_tokens, + }; + let reply: ObserveResult = call(sock, params)?; + match format { + Format::Json => { + let json = serde_json::to_string_pretty(&reply) + .map_err(|e| CliError::Local(anyhow::anyhow!(e)))?; + println!("{json}"); + } + Format::Human => { + if reply.text.is_empty() { + println!("(empty observation — page may still be loading)"); + } else { + println!("{}", reply.text); + } + if reply.truncated { + eprintln!( + "warning: observation truncated (refs={}, tab={}). Increase --max-depth / --max-tokens if needed.", + reply.ref_count, reply.tab_id + ); + } + print_dialog_summaries(&reply.dialogs); + } + } + Ok(()) +} + +fn call(sock: PathBuf, params: ObserveParams) -> Result { + crate::cli::business_rpc::call::( + sock, + "observe", + Method::ToolObserve, + Some(params), + TOOL_IPC_TIMEOUT, + ) +} diff --git a/crates/bsk-cli/src/daemon/ipc.rs b/crates/bsk-cli/src/daemon/ipc.rs index d273bcf..0efb977 100644 --- a/crates/bsk-cli/src/daemon/ipc.rs +++ b/crates/bsk-cli/src/daemon/ipc.rs @@ -231,6 +231,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc) -> RpcHandler | Method::ToolConsole | Method::ToolNetwork | Method::ToolSnapshot + | Method::ToolObserve | Method::ToolGetHtml | Method::ToolNavigate | Method::ToolNavigateBack @@ -291,16 +292,14 @@ async fn handle_tool_dispatch( }; // Pre-flight: if the user has clicked the agent-window mask's // stop button, every session carries a one-shot "pending - // interrupt" marker. The marker is consumed by the next - // *mutating* tool call (which is rejected with `UserAborted`). - // Read-only tools (snapshot / get_html / waits / tab_list) and - // session-lifecycle RPCs pass through transparently — gating - // them would prevent the agent from observing page state - // before asking the user, or from cleanly tearing down the - // session. Classification lives on `Method::is_mutating()` so - // adding a new tool variant requires an explicit - // classification call. - if method.is_mutating() && state.session_interrupts.try_consume(&session_id) { + // interrupt" marker. The marker is consumed by the next method + // that dispatches browser/page input (which is rejected with + // `UserAborted`). Passive reads and control-plane RPCs pass through + // transparently — gating them would prevent the agent from observing + // page state before asking the user, or from cleanly tearing down the + // session. Classification lives on `Method::effect()` so adding a new + // tool variant requires an explicit classification call. + if method.requires_interrupt_gate() && state.session_interrupts.try_consume(&session_id) { return ResponseBody::Err(RpcError { code: ErrorCode::UserAborted, message: "tool dispatch rejected: pending user interrupt. The user explicitly requested to stop. Ask the user how to proceed before issuing further actions.".into(), diff --git a/crates/bsk-cli/src/daemon/session_interrupt.rs b/crates/bsk-cli/src/daemon/session_interrupt.rs index 3497ade..ce9ab69 100644 --- a/crates/bsk-cli/src/daemon/session_interrupt.rs +++ b/crates/bsk-cli/src/daemon/session_interrupt.rs @@ -2,13 +2,13 @@ //! //! Holds a single-use marker per `SessionId` indicating that the //! user has clicked the agent-window mask's stop button. The next -//! mutating `tool.*` call for that session is rejected with -//! `ErrorCode::UserAborted`; read-only tools and session-lifecycle +//! browser-input-dispatching `tool.*` call for that session is rejected +//! with `ErrorCode::UserAborted`; passive reads and session-lifecycle //! RPCs pass through transparently and do not consume the marker. //! //! The marker is single-use and has no expiry: it sits in the -//! registry until consumed by a mutating call or until the session -//! is torn down. This lets the user's interrupt survive an LLM +//! registry until consumed by an input-dispatching call or until the +//! session is torn down. This lets the user's interrupt survive an LLM //! thinking phase of arbitrary length — the v1 time-window //! mechanism dropped interrupts whenever the LLM took longer to //! respond than the window allowed. diff --git a/crates/bsk-cli/src/daemon/state.rs b/crates/bsk-cli/src/daemon/state.rs index d6632da..7ea9cee 100644 --- a/crates/bsk-cli/src/daemon/state.rs +++ b/crates/bsk-cli/src/daemon/state.rs @@ -46,8 +46,8 @@ pub struct DaemonState { /// Per-session "pending interrupt" signal. The WS event handler /// `mark`s the session when the user clicks the agent-window /// mask's stop button; the IPC tool-dispatch handler - /// `try_consume`s on the way in so the next mutating tool call - /// is rejected with `UserAborted`. Independent of + /// `try_consume`s on the way in so the next browser-input-dispatching + /// tool call is rejected with `UserAborted`. Independent of /// `SessionRegistry` because the signal is a transient runtime /// control state. pub session_interrupts: Arc, diff --git a/crates/bsk-cli/src/main.rs b/crates/bsk-cli/src/main.rs index cdc9e73..75d912c 100644 --- a/crates/bsk-cli/src/main.rs +++ b/crates/bsk-cli/src/main.rs @@ -82,6 +82,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> { Command::Tab(cmd) => cli::tab::dispatch(cmd, format), Command::Screenshot(args) => cli::screenshot::dispatch(args, format), Command::Snapshot(args) => cli::snapshot::dispatch(args, format), + Command::Observe(args) => cli::observe::dispatch(args, format), Command::Console(args) => cli::console::dispatch(args, format), Command::Network(args) => cli::network::dispatch(args, format), Command::GetHtml(args) => cli::get_html::dispatch(args, format), diff --git a/crates/bsk-cli/tests/session_user_interrupt.rs b/crates/bsk-cli/tests/session_user_interrupt.rs index b7d6e61..9c5288c 100644 --- a/crates/bsk-cli/tests/session_user_interrupt.rs +++ b/crates/bsk-cli/tests/session_user_interrupt.rs @@ -283,11 +283,11 @@ async fn session_user_interrupt_event_cancels_inflight_with_user_aborted() { handle.shutdown().await; } -/// The pending-interrupt marker must reject a *new* mutating tool +/// The pending-interrupt marker must reject a *new* browser-input tool /// call that arrives AFTER the user clicked stop, even when no /// tool was inflight at click time. Under v2 single-use marker /// semantics, the marker survives indefinitely until consumed by -/// a mutating tool call — there is no time window. This is the +/// an input-dispatching tool call — there is no time window. This is the /// core motivating scenario: agents spend most of their time /// between tool calls (LLM thinking), and the stop button must /// catch a tool dispatched arbitrarily later. @@ -426,12 +426,12 @@ async fn read_only_tool_passes_through_without_consuming_interrupt_marker() { let ws_sink = Arc::new(tokio::sync::Mutex::new(ws_sink)); // Fake extension: replies to session_start, console, and snapshot - // (read-only tools must succeed transparently). Records click count - // — must remain 0 because the daemon should reject the click + // (passive reads must succeed transparently). Records observe count + // — must remain 0 because the daemon should reject transient input // without forwarding. let ws_sink_for_responder = Arc::clone(&ws_sink); - let observed_click_count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let observed_click_count_clone = Arc::clone(&observed_click_count); + let observed_observe_count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let observed_observe_count_clone = Arc::clone(&observed_observe_count); let responder = tokio::spawn(async move { let mut ws_stream = ws_stream; while let Some(Ok(msg)) = ws_stream.next().await { @@ -458,8 +458,8 @@ async fn read_only_tool_passes_through_without_consuming_interrupt_marker() { "truncated": false })), Method::ToolSnapshot => ResponseBody::Ok(json!({"ok": true})), - Method::ToolClick => { - observed_click_count_clone + Method::ToolObserve => { + observed_observe_count_clone .fetch_add(1, std::sync::atomic::Ordering::SeqCst); continue; } @@ -547,26 +547,25 @@ async fn read_only_tool_passes_through_without_consuming_interrupt_marker() { snapshot_outcome ); - // The mutating click that follows MUST still be rejected — the - // marker survived the snapshot. - let click_outcome = ipc + // The transient-input observe that follows MUST still be rejected — the + // marker survived the passive reads. + let observe_outcome = ipc .call::<_, serde_json::Value>( - "click-after-snapshot", - Method::ToolClick, + "observe-after-snapshot", + Method::ToolObserve, Some(json!({ "session_id": start.session_id, - "ref": "fake-ref-2", })), Duration::from_secs(3), ) .await .unwrap(); - let err = click_outcome.expect_err("tool.click must be rejected"); + let err = observe_outcome.expect_err("tool.observe must be rejected"); assert_eq!(err.code, ErrorCode::UserAborted, "got {:?}", err); assert_eq!( - observed_click_count.load(std::sync::atomic::Ordering::SeqCst), + observed_observe_count.load(std::sync::atomic::Ordering::SeqCst), 0, - "tool.click must NOT have been forwarded after snapshot transparency" + "tool.observe must NOT have been forwarded after passive-read transparency" ); responder.abort(); @@ -574,7 +573,7 @@ async fn read_only_tool_passes_through_without_consuming_interrupt_marker() { } /// The pending-interrupt marker must survive an arbitrary delay -/// between user click and next mutating tool call. This is the +/// between user click and next browser-input tool call. This is the /// regression test for v2's core motivating bug: v1's 500ms time /// window dropped interrupts whenever the LLM's thinking phase /// took longer to respond than the window allowed. diff --git a/crates/bsk-cli/tests/tools_ipc.rs b/crates/bsk-cli/tests/tools_ipc.rs index 00d821a..63714a0 100644 --- a/crates/bsk-cli/tests/tools_ipc.rs +++ b/crates/bsk-cli/tests/tools_ipc.rs @@ -11,9 +11,9 @@ use bsk::ipc_client::IpcClient; use bsk_protocol::system::{HandshakeParams, HandshakeResult}; use bsk_protocol::tools::{ ConsoleEntry, ConsoleEntryKind, ConsoleParams, ConsoleResult, GetHtmlParams, GetHtmlResult, - NetworkEntry, NetworkEntryKind, NetworkParams, NetworkResult, ScreenshotParams, - ScreenshotResult, SessionStartParams, SessionStartResult, SnapshotParams, SnapshotResult, - TabInfo, TabListParams, TabListResult, TabScope, + NetworkEntry, NetworkEntryKind, NetworkParams, NetworkResult, ObserveParams, ObserveResult, + ScreenshotParams, ScreenshotResult, SessionStartParams, SessionStartResult, SnapshotParams, + SnapshotResult, TabInfo, TabListParams, TabListResult, TabScope, }; use bsk_protocol::{ BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError, @@ -462,6 +462,46 @@ async fn snapshot_returns_text_and_ref_count() { handle.shutdown().await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn observe_returns_semantic_text_and_ref_count() { + let (handle, sock) = spawn_daemon().await; + let mut ws = connect_ext(handle.ws_addr()).await; + let _ = do_handshake(&mut ws).await; + + run_extension(ws, |req| { + assert_eq!(req.method, Method::ToolObserve); + let _: ObserveParams = serde_json::from_value(req.params.clone().unwrap()).unwrap(); + ResponseBody::Ok( + serde_json::to_value(ObserveResult { + text: "@vom 1\n @e1 button \"Products\" [hover: Shoes]\n".into(), + ref_count: 1, + tab_id: 13, + truncated: false, + dialogs: vec![], + }) + .unwrap(), + ) + }); + + let session_id = ipc_session_start(&sock).await; + let result: ObserveResult = ipc_tool_call( + &sock, + Method::ToolObserve, + ObserveParams { + session_id, + tab_id: None, + max_depth: None, + max_tokens: None, + }, + ) + .await + .expect("observe ok"); + assert_eq!(result.ref_count, 1); + assert!(result.text.contains("[hover: Shoes]")); + assert_eq!(result.tab_id, 13); + handle.shutdown().await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn get_html_round_trips_with_ref_alias() { let (handle, sock) = spawn_daemon().await; diff --git a/crates/bsk-protocol/schema/tool_observe_params.json b/crates/bsk-protocol/schema/tool_observe_params.json new file mode 100644 index 0000000..2da54a9 --- /dev/null +++ b/crates/bsk-protocol/schema/tool_observe_params.json @@ -0,0 +1,40 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ObserveParams", + "description": "Parameters for `tool.observe` — produce a semantic VOM observation for one tab plus a fresh `@e` ref-store on the extension side.\n\nUnlike `tool.snapshot`, this path may run bounded perception probes such as hover-surface discovery. It must not submit input, click, navigate, or otherwise commit page state.", + "type": "object", + "required": [ + "session_id" + ], + "properties": { + "max_depth": { + "description": "Cap the depth of the rendered VOM tree.", + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0.0 + }, + "max_tokens": { + "description": "Soft cap on rendered tokens (approximate; extension uses a best-effort heuristic based on character count).", + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0.0 + }, + "session_id": { + "type": "string" + }, + "tab_id": { + "description": "Optional target tab. Defaults to the Agent Window's currently active tab.", + "type": [ + "integer", + "null" + ], + "format": "int64" + } + } +} diff --git a/crates/bsk-protocol/schema/tool_observe_result.json b/crates/bsk-protocol/schema/tool_observe_result.json new file mode 100644 index 0000000..2b2d9fd --- /dev/null +++ b/crates/bsk-protocol/schema/tool_observe_result.json @@ -0,0 +1,108 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ObserveResult", + "type": "object", + "required": [ + "ref_count", + "tab_id", + "text" + ], + "properties": { + "dialogs": { + "type": "array", + "items": { + "$ref": "#/definitions/JavaScriptDialogInfo" + } + }, + "ref_count": { + "description": "Number of `@e` refs registered for this session by this observation. Equivalent to the size of the new ref-store map.", + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "tab_id": { + "description": "Tab the observation was computed for.", + "type": "integer", + "format": "int64" + }, + "text": { + "description": "Semantic VOM observation text. Refs are rendered as `@e` so the agent can copy them into subsequent interaction tools.", + "type": "string" + }, + "truncated": { + "description": "Whether the rendered tree was truncated because of `max_depth` / `max_tokens`. Agents may re-run with looser caps.", + "default": false, + "type": "boolean" + } + }, + "definitions": { + "JavaScriptDialogHandledAction": { + "description": "How the extension resolved the dialog so CDP could continue.", + "type": "string", + "enum": [ + "accepted", + "dismissed" + ] + }, + "JavaScriptDialogInfo": { + "description": "One observed + handled JavaScript dialog during a tool call.", + "type": "object", + "required": [ + "handled", + "message", + "sequence", + "tab_id", + "type" + ], + "properties": { + "default_prompt": { + "type": [ + "string", + "null" + ] + }, + "handled": { + "$ref": "#/definitions/JavaScriptDialogHandledAction" + }, + "has_browser_handler": { + "type": [ + "boolean", + "null" + ] + }, + "message": { + "type": "string" + }, + "sequence": { + "description": "Monotonic per-tab sequence for ordering within a session.", + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "tab_id": { + "type": "integer", + "format": "int64" + }, + "type": { + "$ref": "#/definitions/JavaScriptDialogType" + }, + "url": { + "type": [ + "string", + "null" + ] + } + } + }, + "JavaScriptDialogType": { + "description": "Native JS dialog kind reported by CDP.", + "type": "string", + "enum": [ + "alert", + "confirm", + "prompt", + "beforeunload" + ] + } + } +} diff --git a/crates/bsk-protocol/src/bin/dump-schema.rs b/crates/bsk-protocol/src/bin/dump-schema.rs index f226e44..6b34449 100644 --- a/crates/bsk-protocol/src/bin/dump-schema.rs +++ b/crates/bsk-protocol/src/bin/dump-schema.rs @@ -78,6 +78,8 @@ fn main() { dump!(SnapshotParams, "tool_snapshot_params"); dump!(SnapshotResult, "tool_snapshot_result"); + dump!(ObserveParams, "tool_observe_params"); + dump!(ObserveResult, "tool_observe_result"); dump!(GetHtmlParams, "tool_get_html_params"); dump!(GetHtmlResult, "tool_get_html_result"); dump!(ScreenshotParams, "tool_screenshot_params"); diff --git a/crates/bsk-protocol/src/method.rs b/crates/bsk-protocol/src/method.rs index d4148c7..6b712bf 100644 --- a/crates/bsk-protocol/src/method.rs +++ b/crates/bsk-protocol/src/method.rs @@ -2,6 +2,22 @@ use serde::{Deserialize, Serialize}; +/// Observable browser-side effect class for a protocol method. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MethodEffect { + /// Reads browser/session state without dispatching page input. + PassiveRead, + /// Dispatches temporary input such as hover probes. It must not submit, + /// navigate, or persist page state, but it can trigger page event handlers. + TransientInput, + /// Drives browser/page state such as clicking, filling, navigation, tabs, + /// or arbitrary page script. + BrowserMutation, + /// Control-plane/session lifecycle operation. These are deliberately not + /// gated by the pending browser-action interrupt path. + ControlPlane, +} + /// Namespaced method string (`system.handshake`, `tool.tab_list`, …). #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum Method { @@ -58,6 +74,8 @@ pub enum Method { ToolSelect, #[serde(rename = "tool.snapshot")] ToolSnapshot, + #[serde(rename = "tool.observe")] + ToolObserve, #[serde(rename = "tool.get_html")] ToolGetHtml, #[serde(rename = "tool.screenshot")] @@ -86,13 +104,13 @@ pub enum Method { } impl Method { - /// Whether this RPC may modify browser state. + /// Browser-side effect classification for this method. /// /// Used by the daemon's pending-interrupt machinery: when the /// user has clicked the agent-window mask's stop button, the - /// next *mutating* tool call for that session is rejected with - /// `ErrorCode::UserAborted`. Read-only tools and session- - /// lifecycle RPCs pass through transparently. + /// next tool call that dispatches browser/page input for that + /// session is rejected with `ErrorCode::UserAborted`. Passive reads + /// and control-plane RPCs pass through transparently. /// /// **Compile-time enforcement.** The match below is exhaustive /// (no `_ =>` fallthrough). Adding a new `Method` variant is a @@ -101,9 +119,12 @@ impl Method { /// /// **Judgment calls** (read these before adding new variants): /// - /// * `tool.evaluate` is classified as mutating because the + /// * `tool.evaluate` is classified as browser-mutating because the /// daemon cannot statically distinguish a `document.title` /// read from a `form.submit()` write. + /// * `tool.observe` is classified as transient input: its bounded + /// hover probes do not commit browser state, but they dispatch real + /// page input events and therefore must be gated like automation. /// * `tool.wait_*` are classified as read-only: they do not /// initiate any browser action; they observe state only. /// * `session.*` and `tool.session_*` are NOT gated. Blocking @@ -111,9 +132,9 @@ impl Method { /// tearing down after observing the user's interrupt. /// * `cancel` is NOT gated. It's a control-plane operation /// (stops another in-flight RPC), not a browser action. - pub fn is_mutating(&self) -> bool { + pub fn effect(&self) -> MethodEffect { match self { - // Mutating tool calls — gated by pending-interrupt. + // Browser/page mutations — gated by pending-interrupt. Method::ToolTabCreate | Method::ToolTabClose | Method::ToolTabBorrow @@ -130,10 +151,13 @@ impl Method { | Method::ToolEvaluate // May navigate via optional `url` and changes Agent Window // chrome; gate behind pending-interrupt like other writes. - | Method::ToolRecordStart => true, + | Method::ToolRecordStart => MethodEffect::BrowserMutation, - // Read-only tool calls — transparent. - // + // Transient input — no committed browser action, but still page + // input. It must be stopped by pending user interrupts. + Method::ToolObserve => MethodEffect::TransientInput, + + // Passive reads — transparent. // `record_stop` / `record_await` observe / finish a recording // without driving new automation gestures, so they stay // ungated (teardown after interrupt must still work). @@ -147,7 +171,7 @@ impl Method { | Method::ToolWaitMs | Method::ToolRequestHelp | Method::ToolRecordStop - | Method::ToolRecordAwait => false, + | Method::ToolRecordAwait => MethodEffect::PassiveRead, // Session lifecycle — not gated. Method::SessionStart @@ -155,16 +179,29 @@ impl Method { | Method::SessionStopAll | Method::SessionList | Method::ToolSessionStart - | Method::ToolSessionStop => false, + | Method::ToolSessionStop => MethodEffect::ControlPlane, // System / control — not gated. Method::SystemHandshake | Method::SystemPing | Method::SystemStatus | Method::BrowserList - | Method::Cancel => false, + | Method::Cancel => MethodEffect::ControlPlane, } } + + /// Whether this RPC drives browser/page state in the traditional sense. + pub fn is_mutating(&self) -> bool { + self.effect() == MethodEffect::BrowserMutation + } + + /// Whether a pending user interrupt should reject this method. + pub fn requires_interrupt_gate(&self) -> bool { + matches!( + self.effect(), + MethodEffect::TransientInput | MethodEffect::BrowserMutation + ) + } } #[cfg(test)] @@ -209,6 +246,7 @@ mod tests { fn is_mutating_classifies_read_only_tools_as_non_mutating() { assert!(!Method::ToolTabList.is_mutating()); assert!(!Method::ToolSnapshot.is_mutating()); + assert!(!Method::ToolObserve.is_mutating()); assert!(!Method::ToolGetHtml.is_mutating()); assert!(!Method::ToolScreenshot.is_mutating()); assert!(!Method::ToolConsole.is_mutating()); @@ -264,4 +302,20 @@ mod tests { assert!(!Method::BrowserList.is_mutating()); assert!(!Method::Cancel.is_mutating()); } + + #[test] + fn effect_classifies_observe_as_transient_input() { + assert_eq!(Method::ToolSnapshot.effect(), MethodEffect::PassiveRead); + assert_eq!(Method::ToolObserve.effect(), MethodEffect::TransientInput); + assert_eq!(Method::ToolClick.effect(), MethodEffect::BrowserMutation); + assert_eq!(Method::Cancel.effect(), MethodEffect::ControlPlane); + } + + #[test] + fn interrupt_gate_includes_transient_input() { + assert!(!Method::ToolSnapshot.requires_interrupt_gate()); + assert!(Method::ToolObserve.requires_interrupt_gate()); + assert!(Method::ToolClick.requires_interrupt_gate()); + assert!(!Method::Cancel.requires_interrupt_gate()); + } } diff --git a/crates/bsk-protocol/src/tools/observation.rs b/crates/bsk-protocol/src/tools/observation.rs index 7fb4ae6..715ddc9 100644 --- a/crates/bsk-protocol/src/tools/observation.rs +++ b/crates/bsk-protocol/src/tools/observation.rs @@ -1,4 +1,4 @@ -//! Read-only observation tools (`tool.snapshot`, `tool.get_html`, `tool.screenshot`). +//! Observation tools (`tool.snapshot`, `tool.observe`, `tool.get_html`, `tool.screenshot`). use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -50,6 +50,50 @@ pub struct SnapshotResult { pub dialogs: Vec, } +// --------------------------------------------------------------------------- +// observe +// --------------------------------------------------------------------------- + +/// Parameters for `tool.observe` — produce a semantic VOM observation +/// for one tab plus a fresh `@e` ref-store on the extension side. +/// +/// Unlike `tool.snapshot`, this path may run bounded perception probes +/// such as hover-surface discovery. It must not submit input, click, +/// navigate, or otherwise commit page state. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ObserveParams { + pub session_id: String, + /// Optional target tab. Defaults to the Agent Window's currently + /// active tab. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tab_id: Option, + /// Cap the depth of the rendered VOM tree. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_depth: Option, + /// Soft cap on rendered tokens (approximate; extension uses a + /// best-effort heuristic based on character count). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_tokens: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ObserveResult { + /// Semantic VOM observation text. Refs are rendered as `@e` so + /// the agent can copy them into subsequent interaction tools. + pub text: String, + /// Number of `@e` refs registered for this session by this + /// observation. Equivalent to the size of the new ref-store map. + pub ref_count: u32, + /// Tab the observation was computed for. + pub tab_id: i64, + /// Whether the rendered tree was truncated because of `max_depth` / + /// `max_tokens`. Agents may re-run with looser caps. + #[serde(default)] + pub truncated: bool, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub dialogs: Vec, +} + // --------------------------------------------------------------------------- // get_html // --------------------------------------------------------------------------- @@ -227,4 +271,19 @@ mod tests { let round: ScreenshotResult = serde_json::from_value(v).unwrap(); assert_eq!(round, r); } + + #[test] + fn observe_result_round_trips_with_ref_count() { + let r = ObserveResult { + text: "@vom 1\n @e1 button \"submit\"\n".into(), + ref_count: 1, + tab_id: 42, + truncated: false, + dialogs: Vec::new(), + }; + let v = serde_json::to_value(&r).unwrap(); + assert_eq!(v.get("ref_count").and_then(|v| v.as_u64()), Some(1)); + let round: ObserveResult = serde_json::from_value(v).unwrap(); + assert_eq!(round, r); + } } diff --git a/packages/vom/src/__tests__/render.test.ts b/packages/vom/src/__tests__/render.test.ts index 958011e..4eeccc8 100644 --- a/packages/vom/src/__tests__/render.test.ts +++ b/packages/vom/src/__tests__/render.test.ts @@ -386,7 +386,7 @@ describe("renderVom single-layer page", () => { ], }); - expect(out.text).toContain('@e1 button "Products" [→ Shoes | Bags | Accessories]'); + expect(out.text).toContain('@e1 button "Products" [hover: Shoes | Bags | Accessories]'); expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); }); diff --git a/packages/vom/src/render.ts b/packages/vom/src/render.ts index 71c372f..5f72cb0 100644 --- a/packages/vom/src/render.ts +++ b/packages/vom/src/render.ts @@ -633,7 +633,7 @@ function renderNodeLine( if (surface && surface.subItems.length > 0) { const items = surface.subItems.slice(0, MAX_SURFACE_ITEMS).join(" | "); const suffix = surface.subItems.length > MAX_SURFACE_ITEMS ? " | …" : ""; - line += ` [→ ${items}${suffix}]`; + line += ` [${surface.triggerAction}: ${items}${suffix}]`; } return line;