mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(vom): update vom algorithm, fix overlay problem with background state machine
This commit is contained in:
@@ -119,6 +119,88 @@ describe("ChromiumCdp", () => {
|
||||
expect(api.sendCommand).toHaveBeenCalledWith({ tabId: 9 }, "Page.enable", {});
|
||||
});
|
||||
|
||||
it("validates the attached top-frame URL for page reads", async () => {
|
||||
const { api } = fakeApi();
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
async (_target, method: string) => {
|
||||
if (method === "Page.getFrameTree") {
|
||||
return { frameTree: { frame: { url: "https://example.test/page" } } };
|
||||
}
|
||||
return {};
|
||||
},
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
|
||||
await expect(
|
||||
cdp.ensureAttachedToUrl(9, "https://example.test/page#section"),
|
||||
).resolves.toBeUndefined();
|
||||
expect(api.detach).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reattaches when the cached CDP target points at an extension page", async () => {
|
||||
const { api } = fakeApi();
|
||||
let frameReads = 0;
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
async (_target, method: string) => {
|
||||
if (method === "Page.getFrameTree") {
|
||||
frameReads += 1;
|
||||
return {
|
||||
frameTree: {
|
||||
frame: {
|
||||
url:
|
||||
frameReads === 1
|
||||
? "chrome-extension://other-extension/page.html"
|
||||
: "https://example.test/page",
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
return {};
|
||||
},
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
|
||||
await expect(cdp.ensureAttachedToUrl(9, "https://example.test/page")).resolves.toBeUndefined();
|
||||
expect(api.attach).toHaveBeenCalledTimes(2);
|
||||
expect(api.detach).toHaveBeenCalledWith({ tabId: 9 });
|
||||
});
|
||||
|
||||
it("rejects when reattach still points at the wrong page", async () => {
|
||||
const { api } = fakeApi();
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
async (_target, method: string) => {
|
||||
if (method === "Page.getFrameTree") {
|
||||
return { frameTree: { frame: { url: "https://wrong.test/" } } };
|
||||
}
|
||||
return {};
|
||||
},
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
|
||||
await expect(cdp.ensureAttachedToUrl(9, "https://example.test/")).rejects.toThrow(
|
||||
/CDP target URL mismatch/,
|
||||
);
|
||||
expect(api.attach).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("clears the attach cache after an internal-page access error", async () => {
|
||||
const { api } = fakeApi();
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
async (_target, method: string) => {
|
||||
if (method === "DOM.getDocument") {
|
||||
throw new Error("Cannot access a chrome-extension:// URL of different extension");
|
||||
}
|
||||
return {};
|
||||
},
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
|
||||
await expect(cdp.ensureAttached(9)).resolves.toBeUndefined();
|
||||
expect(cdp.isAttached(9)).toBe(true);
|
||||
await expect(cdp.send(9, "DOM.getDocument")).rejects.toThrow(/chrome-extension/);
|
||||
expect(cdp.isAttached(9)).toBe(false);
|
||||
});
|
||||
|
||||
it("enables console capture domains best-effort during attach", async () => {
|
||||
const { api } = fakeApi();
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
|
||||
@@ -93,6 +93,15 @@ const MAX_CONSOLE_STACK_FRAMES = 20;
|
||||
const MAX_NETWORK_BUFFER = 200;
|
||||
const MAX_NETWORK_FIELD_LENGTH = 4096;
|
||||
const MAX_NETWORK_REQUEST_META = 1024;
|
||||
const CDP_INTERNAL_PROTOCOLS = new Set([
|
||||
"chrome:",
|
||||
"chrome-extension:",
|
||||
"devtools:",
|
||||
"edge:",
|
||||
"brave:",
|
||||
"vivaldi:",
|
||||
"opera:",
|
||||
]);
|
||||
|
||||
interface ParsedDialogOpening {
|
||||
type: JavaScriptDialogType;
|
||||
@@ -173,6 +182,31 @@ export class ChromiumCdp {
|
||||
await attach;
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach and verify Chrome's actual CDP target still points at the page URL
|
||||
* Chrome tabs reported to the tool resolver. A stale debugger target can
|
||||
* otherwise surface another extension/internal URL while the tab record looks
|
||||
* like a normal page.
|
||||
*/
|
||||
async ensureAttachedToUrl(tabId: number, expectedUrl: string | undefined): Promise<void> {
|
||||
await this.ensureAttached(tabId);
|
||||
if (!expectedUrl) return;
|
||||
|
||||
const first = await this.readTopFrameUrl(tabId).catch((err) => {
|
||||
if (isInternalTargetAccessError(err)) return null;
|
||||
throw err;
|
||||
});
|
||||
if (first && cdpUrlsCompatible(first, expectedUrl)) return;
|
||||
|
||||
await this.forceReattach(tabId);
|
||||
const second = await this.readTopFrameUrl(tabId);
|
||||
if (cdpUrlsCompatible(second, expectedUrl)) return;
|
||||
|
||||
throw new Error(
|
||||
`CDP target URL mismatch for tab ${tabId}: expected ${expectedUrl}, attached to ${second}`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a CDP command and decode the result as `T`. Throws on any
|
||||
* `chrome.runtime.lastError`.
|
||||
@@ -185,6 +219,9 @@ export class ChromiumCdp {
|
||||
const result = await this.api.sendCommand({ tabId }, method, params ?? {});
|
||||
return result as T;
|
||||
} catch (err) {
|
||||
if (isInternalTargetAccessError(err)) {
|
||||
this.clearAttachedState(tabId);
|
||||
}
|
||||
throw normalizeError(err);
|
||||
}
|
||||
}
|
||||
@@ -359,6 +396,31 @@ export class ChromiumCdp {
|
||||
}
|
||||
}
|
||||
|
||||
private async forceReattach(tabId: number): Promise<void> {
|
||||
this.clearAttachedState(tabId);
|
||||
try {
|
||||
await this.api.detach({ tabId });
|
||||
} catch {
|
||||
// Chrome may already have detached; attach below is the source of truth.
|
||||
}
|
||||
await this.ensureAttached(tabId);
|
||||
}
|
||||
|
||||
private clearAttachedState(tabId: number): void {
|
||||
this.attachInFlight.delete(tabId);
|
||||
this.attachedTabs.delete(tabId);
|
||||
this.clearDialogState(tabId);
|
||||
this.clearConsoleState(tabId);
|
||||
this.clearNetworkState(tabId);
|
||||
}
|
||||
|
||||
private async readTopFrameUrl(tabId: number): Promise<string> {
|
||||
const frameTree = await this.api.sendCommand({ tabId }, "Page.getFrameTree", {});
|
||||
const url = topFrameUrl(frameTree);
|
||||
if (!url) throw new Error(`Page.getFrameTree returned no top-frame URL for tab ${tabId}`);
|
||||
return url;
|
||||
}
|
||||
|
||||
private bindDialogHandler(): void {
|
||||
if (this.dialogSubscription) return;
|
||||
const listener = (source: chrome.debugger.Debuggee, method: string, params: unknown) => {
|
||||
@@ -906,3 +968,49 @@ function normalizeError(err: unknown): Error {
|
||||
}
|
||||
return new Error("unknown chrome.debugger error");
|
||||
}
|
||||
|
||||
function isInternalTargetAccessError(err: unknown): boolean {
|
||||
const message = normalizeError(err).message;
|
||||
return (
|
||||
/Cannot access a chrome-extension:\/\/ URL/i.test(message) ||
|
||||
/Cannot access a chrome:\/\/ URL/i.test(message) ||
|
||||
/Cannot access contents of url/i.test(message)
|
||||
);
|
||||
}
|
||||
|
||||
function topFrameUrl(frameTree: unknown): string | null {
|
||||
const raw = (frameTree ?? {}) as Record<string, unknown>;
|
||||
const tree = (raw.frameTree ?? {}) as Record<string, unknown>;
|
||||
const frame = (tree.frame ?? {}) as Record<string, unknown>;
|
||||
return typeof frame.url === "string" && frame.url.length > 0 ? frame.url : null;
|
||||
}
|
||||
|
||||
function cdpUrlsCompatible(actualUrl: string, expectedUrl: string): boolean {
|
||||
const actual = normalizeComparableUrl(actualUrl);
|
||||
const expected = normalizeComparableUrl(expectedUrl);
|
||||
if (!actual || !expected) return false;
|
||||
if (isInternalCdpUrl(actual)) return false;
|
||||
return actual === expected;
|
||||
}
|
||||
|
||||
function normalizeComparableUrl(value: string): string | null {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(value);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
parsed.hash = "";
|
||||
return parsed.toString();
|
||||
}
|
||||
|
||||
function isInternalCdpUrl(value: string): boolean {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(value);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
if (CDP_INTERNAL_PROTOCOLS.has(parsed.protocol)) return true;
|
||||
return parsed.protocol === "about:" && parsed.pathname !== "blank";
|
||||
}
|
||||
|
||||
@@ -1,153 +0,0 @@
|
||||
import { useTranslation } from "@browser-skill/i18n/react";
|
||||
import { RiStopCircleLine } from "@remixicon/react";
|
||||
import { useEffect, useState } from "react";
|
||||
import logoUrl from "../../assets/logo.png";
|
||||
|
||||
export interface ControlOverlayProps {
|
||||
visible: boolean;
|
||||
interrupting: boolean;
|
||||
automationBypass: boolean;
|
||||
onInterrupt: () => void;
|
||||
}
|
||||
|
||||
export function ControlOverlay({
|
||||
visible,
|
||||
interrupting,
|
||||
automationBypass,
|
||||
onInterrupt,
|
||||
}: ControlOverlayProps) {
|
||||
const { t } = useTranslation("extension");
|
||||
const [show, setShow] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (visible) {
|
||||
const raf = requestAnimationFrame(() => setShow(true));
|
||||
return () => cancelAnimationFrame(raf);
|
||||
}
|
||||
setShow(false);
|
||||
}, [visible]);
|
||||
|
||||
if (!visible) return null;
|
||||
|
||||
const pointerEvents = automationBypass ? "none" : "auto";
|
||||
|
||||
return (
|
||||
<>
|
||||
<style>{`
|
||||
@keyframes bsk-breathe {
|
||||
0%, 100% {
|
||||
box-shadow: inset 0 0 20px 4px rgba(249,115,22,0.25);
|
||||
}
|
||||
50% {
|
||||
box-shadow: inset 0 0 40px 8px rgba(249,115,22,0.5);
|
||||
}
|
||||
}
|
||||
`}</style>
|
||||
|
||||
<div
|
||||
data-slot="control-overlay"
|
||||
style={{
|
||||
position: "fixed",
|
||||
inset: 0,
|
||||
zIndex: 2147483646,
|
||||
pointerEvents: "none",
|
||||
animation: "bsk-breathe 3s ease-in-out infinite",
|
||||
opacity: show ? 1 : 0,
|
||||
transition: "opacity 300ms ease-out",
|
||||
}}
|
||||
/>
|
||||
|
||||
<div
|
||||
data-slot="control-overlay-blocker"
|
||||
onPointerDown={(event) => {
|
||||
if (automationBypass) return;
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
}}
|
||||
onClick={(event) => {
|
||||
if (automationBypass) return;
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
}}
|
||||
style={{
|
||||
position: "fixed",
|
||||
inset: 0,
|
||||
zIndex: 2147483646,
|
||||
pointerEvents,
|
||||
background: "transparent",
|
||||
opacity: show ? 1 : 0,
|
||||
transition: "opacity 300ms ease-out",
|
||||
}}
|
||||
/>
|
||||
|
||||
<div
|
||||
data-slot="control-overlay-pill"
|
||||
style={{
|
||||
position: "fixed",
|
||||
bottom: 32,
|
||||
left: "50%",
|
||||
transform: "translateX(-50%)",
|
||||
zIndex: 2147483647,
|
||||
// Always receive clicks so Interrupt works even while the page
|
||||
// blocker is in automation-bypass mode (CDP pass-through).
|
||||
pointerEvents: "auto",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 12,
|
||||
backgroundColor: "#fff",
|
||||
borderRadius: 9999,
|
||||
padding: "10px 10px 10px 20px",
|
||||
boxShadow: "0 8px 32px rgba(124,45,18,0.16), 0 2px 8px rgba(0,0,0,0.1)",
|
||||
opacity: show ? 1 : 0,
|
||||
transition: "opacity 300ms ease-out",
|
||||
fontFamily:
|
||||
'-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif',
|
||||
}}
|
||||
>
|
||||
<img
|
||||
src={logoUrl}
|
||||
alt="browser-skill"
|
||||
style={{ width: 24, height: 24, borderRadius: 4, flexShrink: 0 }}
|
||||
/>
|
||||
<span
|
||||
style={{
|
||||
fontSize: 16,
|
||||
fontWeight: 500,
|
||||
color: "#333",
|
||||
whiteSpace: "nowrap",
|
||||
userSelect: "none",
|
||||
}}
|
||||
>
|
||||
{t("controlOverlay.status")}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
data-slot="control-overlay-stop-all"
|
||||
disabled={interrupting}
|
||||
onClick={onInterrupt}
|
||||
style={{
|
||||
pointerEvents: "auto",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 6,
|
||||
border: "none",
|
||||
borderRadius: 9999,
|
||||
padding: "8px 20px 8px 16px",
|
||||
fontSize: 15,
|
||||
fontWeight: 600,
|
||||
color: "#fff",
|
||||
backgroundColor: interrupting ? "#9ca3af" : "#f97316",
|
||||
cursor: interrupting ? "default" : "pointer",
|
||||
opacity: interrupting ? 0.7 : 1,
|
||||
transition: "background-color 150ms ease-out, opacity 150ms ease-out",
|
||||
whiteSpace: "nowrap",
|
||||
lineHeight: 1,
|
||||
}}
|
||||
>
|
||||
<RiStopCircleLine size={18} color="#fff" />
|
||||
{interrupting ? t("controlOverlay.interrupting") : t("controlOverlay.interrupt")}
|
||||
</button>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
import { cleanup, render } from "@testing-library/react";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { ControlOverlay } from "../ControlOverlay";
|
||||
|
||||
describe("ControlOverlay", () => {
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
});
|
||||
|
||||
it("keeps page blocker none under automationBypass but Interrupt stays clickable", () => {
|
||||
const { container } = render(
|
||||
<ControlOverlay
|
||||
visible={true}
|
||||
interrupting={false}
|
||||
automationBypass={true}
|
||||
onInterrupt={() => {}}
|
||||
/>,
|
||||
);
|
||||
|
||||
const blocker = container.querySelector("[data-slot='control-overlay-blocker']");
|
||||
expect(blocker).toBeTruthy();
|
||||
expect((blocker as HTMLElement).style.pointerEvents).toBe("none");
|
||||
|
||||
const pill = container.querySelector("[data-slot='control-overlay-pill']");
|
||||
expect(pill).toBeTruthy();
|
||||
expect((pill as HTMLElement).style.pointerEvents).toBe("auto");
|
||||
|
||||
const stopBtn = container.querySelector("[data-slot='control-overlay-stop-all']");
|
||||
expect(stopBtn).toBeTruthy();
|
||||
expect((stopBtn as HTMLElement).style.pointerEvents).toBe("auto");
|
||||
});
|
||||
|
||||
it("uses pointer-events auto on blocker when automationBypass is false", () => {
|
||||
const { container } = render(
|
||||
<ControlOverlay
|
||||
visible={true}
|
||||
interrupting={false}
|
||||
automationBypass={false}
|
||||
onInterrupt={() => {}}
|
||||
/>,
|
||||
);
|
||||
|
||||
const blocker = container.querySelector("[data-slot='control-overlay-blocker']");
|
||||
expect(blocker).toBeTruthy();
|
||||
expect((blocker as HTMLElement).style.pointerEvents).toBe("auto");
|
||||
});
|
||||
});
|
||||
@@ -116,4 +116,18 @@ describe("OverlayController", () => {
|
||||
expect(controller.snapshot().suppressControlAfterRecord).toBe(false);
|
||||
expect(controller.snapshot().controlVisible).toBe(false);
|
||||
});
|
||||
|
||||
it("hides control while paused and shows it again only after authoritative control mode", () => {
|
||||
const controller = new OverlayController();
|
||||
|
||||
controller.applyAgentControlMode("sess-1", "paused");
|
||||
expect(controller.snapshot().activeSessionId).toBe("sess-1");
|
||||
expect(controller.snapshot().controlVisible).toBe(false);
|
||||
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false);
|
||||
|
||||
controller.applyAgentControlMode("sess-1", "control");
|
||||
expect(controller.snapshot().activeSessionId).toBe("sess-1");
|
||||
expect(controller.snapshot().controlVisible).toBe(true);
|
||||
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { OverlayMode } from "@/lib/overlay-bridge";
|
||||
import type { BorrowRequestData } from "./BorrowConfirmationOverlay";
|
||||
import type { HelpRequestData } from "./HelpRequestOverlay";
|
||||
import type { RecordRequestData } from "./RecordOverlay";
|
||||
@@ -9,6 +10,7 @@ export interface OverlayState {
|
||||
controlVisible: boolean;
|
||||
interrupting: boolean;
|
||||
activeSessionId: string | null;
|
||||
controlMode: OverlayMode;
|
||||
automationBypassCount: number;
|
||||
/**
|
||||
* After record Finish/Stop clears `activeRecord`, the session is still
|
||||
@@ -31,6 +33,7 @@ export class OverlayController {
|
||||
activeRecord: null,
|
||||
interrupting: false,
|
||||
activeSessionId: null,
|
||||
controlMode: "hidden",
|
||||
automationBypassCount: 0,
|
||||
suppressControlAfterRecord: false,
|
||||
};
|
||||
@@ -44,7 +47,7 @@ export class OverlayController {
|
||||
}
|
||||
|
||||
isControlVisible(): boolean {
|
||||
return this.state.activeSessionId !== null;
|
||||
return this.state.activeSessionId !== null && this.state.controlMode === "control";
|
||||
}
|
||||
|
||||
addBorrowRequest(request: BorrowRequestData): void {
|
||||
@@ -61,6 +64,25 @@ export class OverlayController {
|
||||
activateAgentSession(sessionId: string): void {
|
||||
this.state.activeSessionId = sessionId;
|
||||
this.state.interrupting = false;
|
||||
this.state.controlMode = "control";
|
||||
}
|
||||
|
||||
applyAgentControlMode(sessionId: string | null, mode: OverlayMode): void {
|
||||
if (mode === "control" && sessionId) {
|
||||
this.state.activeSessionId = sessionId;
|
||||
this.state.interrupting = false;
|
||||
this.state.controlMode = "control";
|
||||
return;
|
||||
}
|
||||
if ((mode === "interrupting" || mode === "paused") && sessionId) {
|
||||
this.state.activeSessionId = sessionId;
|
||||
this.state.interrupting = mode === "interrupting";
|
||||
this.state.controlMode = mode;
|
||||
return;
|
||||
}
|
||||
this.state.activeSessionId = null;
|
||||
this.state.interrupting = false;
|
||||
this.state.controlMode = "hidden";
|
||||
}
|
||||
|
||||
setInterrupting(interrupting: boolean): void {
|
||||
@@ -118,6 +140,7 @@ export class OverlayController {
|
||||
activeRecord: null,
|
||||
interrupting: false,
|
||||
activeSessionId: null,
|
||||
controlMode: "hidden",
|
||||
automationBypassCount: 0,
|
||||
suppressControlAfterRecord: false,
|
||||
};
|
||||
|
||||
@@ -9,12 +9,16 @@ import {
|
||||
} from "@/lib/instance-id";
|
||||
import { startKeepalive } from "@/lib/keepalive";
|
||||
import {
|
||||
OVERLAY_AGENT_STATE,
|
||||
OVERLAY_AUTOMATION_BYPASS,
|
||||
OVERLAY_MSG_INTERRUPT,
|
||||
OVERLAY_MSG_READY,
|
||||
OVERLAY_MSG_WHO_AM_I,
|
||||
type OverlayAgentStateMessage,
|
||||
type OverlayInterruptRequest,
|
||||
type OverlayInterruptResponse,
|
||||
type OverlayMessage,
|
||||
type OverlayMode,
|
||||
} from "@/lib/overlay-bridge";
|
||||
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
|
||||
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
|
||||
@@ -44,6 +48,91 @@ export default defineBackground(() => {
|
||||
const sessions = new SessionManager();
|
||||
const cdp = new ChromiumCdp();
|
||||
const sessionsLive = attachSessionsLiveFlag({ manager: sessions });
|
||||
let overlayGeneration = 0;
|
||||
const controlModes = new Map<string, OverlayMode>();
|
||||
|
||||
function setControlMode(sessionId: string, mode: OverlayMode): void {
|
||||
if (controlModes.get(sessionId) === mode) return;
|
||||
controlModes.set(sessionId, mode);
|
||||
overlayGeneration += 1;
|
||||
const ctx = sessions.get(sessionId);
|
||||
if (ctx) void pushOverlayStateForWindow(ctx.agentWindowId);
|
||||
}
|
||||
|
||||
function overlayStateForWindow(windowId?: number): OverlayAgentStateMessage {
|
||||
const ctx = typeof windowId === "number" ? sessions.findByWindowId(windowId) : null;
|
||||
if (!ctx) {
|
||||
return {
|
||||
type: OVERLAY_AGENT_STATE,
|
||||
sessionId: null,
|
||||
mode: "hidden",
|
||||
generation: overlayGeneration,
|
||||
};
|
||||
}
|
||||
return {
|
||||
type: OVERLAY_AGENT_STATE,
|
||||
sessionId: ctx.sessionId,
|
||||
mode: controlModes.get(ctx.sessionId) ?? "control",
|
||||
generation: overlayGeneration,
|
||||
};
|
||||
}
|
||||
|
||||
async function pushOverlayStateToTab(
|
||||
tabId: number,
|
||||
state: OverlayAgentStateMessage,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await chrome.tabs.sendMessage(tabId, state);
|
||||
} catch {
|
||||
// Restricted pages and not-yet-loaded content scripts cannot receive messages.
|
||||
}
|
||||
}
|
||||
|
||||
async function pushOverlayStateForWindow(windowId: number): Promise<void> {
|
||||
const state = overlayStateForWindow(windowId);
|
||||
const tabs = await chrome.tabs.query({ windowId });
|
||||
await Promise.all(
|
||||
tabs.map((tab) =>
|
||||
typeof tab.id === "number" ? pushOverlayStateToTab(tab.id, state) : Promise.resolve(),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function pushAllAgentOverlayStates(): void {
|
||||
const windowIds = new Set(sessions.list().map((ctx) => ctx.agentWindowId));
|
||||
for (const windowId of windowIds) {
|
||||
void pushOverlayStateForWindow(windowId);
|
||||
}
|
||||
}
|
||||
|
||||
function onOverlaySessionStateChanged(): void {
|
||||
void sessionsLive.syncFromManager();
|
||||
const liveSessionIds = new Set(sessions.list().map((ctx) => ctx.sessionId));
|
||||
for (const sessionId of controlModes.keys()) {
|
||||
if (!liveSessionIds.has(sessionId)) controlModes.delete(sessionId);
|
||||
}
|
||||
overlayGeneration += 1;
|
||||
pushAllAgentOverlayStates();
|
||||
}
|
||||
|
||||
function onBrowserControlResumed(sessionId: string): void {
|
||||
const ctx = sessions.get(sessionId);
|
||||
if (!ctx) return;
|
||||
setControlMode(sessionId, "control");
|
||||
}
|
||||
|
||||
function pushOverlayStateForAgentWindow(windowId: number): void {
|
||||
if (!sessions.findByWindowId(windowId)) return;
|
||||
void pushOverlayStateForWindow(windowId);
|
||||
}
|
||||
chrome.tabs.onActivated.addListener((activeInfo) => {
|
||||
pushOverlayStateForAgentWindow(activeInfo.windowId);
|
||||
});
|
||||
chrome.tabs.onUpdated.addListener((_tabId, changeInfo, tab) => {
|
||||
if (changeInfo.status !== "complete") return;
|
||||
if (typeof tab.windowId !== "number") return;
|
||||
pushOverlayStateForAgentWindow(tab.windowId);
|
||||
});
|
||||
// Re-sync the storage.session flag on SW startup so a previous SW's
|
||||
// stale `true` does not keep waking us on every page load until the
|
||||
// first mutation (review M4/M5 round 3 m-R3-1).
|
||||
@@ -52,9 +141,8 @@ export default defineBackground(() => {
|
||||
transport,
|
||||
sessions,
|
||||
cdp,
|
||||
onSessionsChanged: () => {
|
||||
void sessionsLive.syncFromManager();
|
||||
},
|
||||
onSessionsChanged: onOverlaySessionStateChanged,
|
||||
onBrowserControlResumed,
|
||||
approveBorrow: (ctx) =>
|
||||
requestBorrowConfirmation(ctx.tabId, {
|
||||
...(ctx.signal !== undefined ? { signal: ctx.signal } : {}),
|
||||
@@ -124,9 +212,7 @@ export default defineBackground(() => {
|
||||
manager: sessions,
|
||||
transport,
|
||||
cdp,
|
||||
onSessionsChanged: () => {
|
||||
void sessionsLive.syncFromManager();
|
||||
},
|
||||
onSessionsChanged: onOverlaySessionStateChanged,
|
||||
});
|
||||
|
||||
// MV3 service worker keepalive + reconnect supervisor (review M4/M5
|
||||
@@ -197,9 +283,19 @@ export default defineBackground(() => {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (msg.kind === OVERLAY_MSG_READY) {
|
||||
sendResponse(overlayStateForWindow(sender.tab?.windowId));
|
||||
return false;
|
||||
}
|
||||
|
||||
if (msg.kind === OVERLAY_MSG_INTERRUPT) {
|
||||
const req = msg as OverlayInterruptRequest;
|
||||
const ctx = sessions.get(req.sessionId);
|
||||
if (ctx) setControlMode(req.sessionId, "interrupting");
|
||||
void handleOverlayInterrupt(transport, req.sessionId).then((reply) => {
|
||||
if (reply.ok && sessions.get(req.sessionId)) {
|
||||
setControlMode(req.sessionId, "paused");
|
||||
}
|
||||
sendResponse(reply);
|
||||
});
|
||||
return true; // keep channel open
|
||||
|
||||
@@ -3,7 +3,6 @@ import { I18nextProvider } from "@browser-skill/i18n/react";
|
||||
import React from "react";
|
||||
import ReactDOM from "react-dom/client";
|
||||
import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay";
|
||||
import { ControlOverlay } from "@/content/ControlOverlay";
|
||||
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
|
||||
import overlayCss from "@/content/overlay.css?inline";
|
||||
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
|
||||
@@ -27,13 +26,12 @@ import {
|
||||
} from "@/lib/help-bridge";
|
||||
import {
|
||||
isOverlayAgentOverlayResetMessage,
|
||||
isOverlayAgentStateMessage,
|
||||
OVERLAY_AUTOMATION_BYPASS,
|
||||
OVERLAY_HOST_MARKER_ATTR,
|
||||
OVERLAY_HOST_NAME,
|
||||
OVERLAY_MSG_WHO_AM_I,
|
||||
OVERLAY_MSG_READY,
|
||||
type OverlayAgentOverlayResetMessage,
|
||||
type OverlayAgentStateMessage,
|
||||
type OverlayAutomationBypassMessage,
|
||||
type OverlayWhoAmIResponse,
|
||||
} from "@/lib/overlay-bridge";
|
||||
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
|
||||
import {
|
||||
@@ -42,12 +40,12 @@ import {
|
||||
type RecordStartAck,
|
||||
type RecordStopAck,
|
||||
} from "@/lib/record-bridge";
|
||||
import { SESSIONS_LIVE_FLAG_KEY } from "@/lib/sessions-live-flag";
|
||||
import type {
|
||||
BorrowCancelMessage,
|
||||
BorrowRequestMessage,
|
||||
BorrowResponseMessage,
|
||||
} from "@/tools/borrow-confirmation";
|
||||
import logoUrl from "../../assets/logo.png";
|
||||
|
||||
// Run at document_end so the overlay does not block first paint. Only attach
|
||||
// in the top-level frame so iframes do not double-render overlays.
|
||||
@@ -65,34 +63,264 @@ export default defineContentScript({
|
||||
let activeRecordRequestId: string | null = null;
|
||||
let reactRoot: ReactDOM.Root | null = null;
|
||||
let overlayHost: HTMLElement | null = null;
|
||||
let overlayContainer: HTMLElement | null = null;
|
||||
let controlRoot: HTMLElement | null = null;
|
||||
let activeAgentState: OverlayAgentStateMessage | null = null;
|
||||
let hostLossReported = false;
|
||||
let remountInProgress = false;
|
||||
|
||||
const ui = await createShadowRootUi(ctx, {
|
||||
name: OVERLAY_HOST_NAME,
|
||||
name: "browser-skill-overlay",
|
||||
position: "inline",
|
||||
anchor: "html",
|
||||
css: overlayCss,
|
||||
onMount(container, _shadow, shadowHost) {
|
||||
shadowHost.setAttribute("aria-hidden", "true");
|
||||
shadowHost.setAttribute(OVERLAY_HOST_MARKER_ATTR, "");
|
||||
shadowHost.setAttribute("data-bsk-overlay", "");
|
||||
overlayHost = shadowHost;
|
||||
overlayContainer = container;
|
||||
hostLossReported = false;
|
||||
const app = document.createElement("div");
|
||||
app.className = "bsk-overlay-root";
|
||||
container.append(app);
|
||||
reactRoot = ReactDOM.createRoot(app);
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
void requestOverlayState();
|
||||
return reactRoot;
|
||||
},
|
||||
onRemove(root) {
|
||||
overlayHost = null;
|
||||
overlayContainer = null;
|
||||
controlRoot = null;
|
||||
root?.unmount();
|
||||
reactRoot = null;
|
||||
},
|
||||
});
|
||||
|
||||
function renderOverlay() {
|
||||
function removeControlOverlay(): void {
|
||||
controlRoot?.remove();
|
||||
controlRoot = null;
|
||||
}
|
||||
|
||||
function setControlSurfaceActive(active: boolean): void {
|
||||
const host = overlayHost;
|
||||
const container = overlayContainer;
|
||||
if (!host || !container) return;
|
||||
|
||||
if (active) {
|
||||
Object.assign(host.style, {
|
||||
position: "fixed",
|
||||
inset: "0",
|
||||
zIndex: "2147483647",
|
||||
pointerEvents: "auto",
|
||||
});
|
||||
Object.assign(container.style, {
|
||||
position: "fixed",
|
||||
inset: "0",
|
||||
pointerEvents: "none",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
host.style.removeProperty("position");
|
||||
host.style.removeProperty("inset");
|
||||
host.style.removeProperty("z-index");
|
||||
host.style.removeProperty("pointer-events");
|
||||
container.style.removeProperty("position");
|
||||
container.style.removeProperty("inset");
|
||||
container.style.removeProperty("pointer-events");
|
||||
}
|
||||
|
||||
function ensureControlOverlayRoot(): HTMLElement | null {
|
||||
if (!overlayContainer) return null;
|
||||
if (!controlRoot) {
|
||||
controlRoot = document.createElement("div");
|
||||
controlRoot.className = "bsk-control-overlay";
|
||||
controlRoot.style.opacity = "0";
|
||||
controlRoot.style.transition = "opacity 300ms ease-out";
|
||||
overlayContainer.append(controlRoot);
|
||||
window.requestAnimationFrame(() => {
|
||||
if (controlRoot) controlRoot.style.opacity = "1";
|
||||
});
|
||||
}
|
||||
return controlRoot;
|
||||
}
|
||||
|
||||
function createStopCircleIcon(): SVGSVGElement {
|
||||
const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg");
|
||||
svg.setAttribute("width", "18");
|
||||
svg.setAttribute("height", "18");
|
||||
svg.setAttribute("viewBox", "0 0 24 24");
|
||||
svg.setAttribute("fill", "currentColor");
|
||||
svg.setAttribute("aria-hidden", "true");
|
||||
svg.style.color = "#fff";
|
||||
svg.style.flexShrink = "0";
|
||||
const path = document.createElementNS("http://www.w3.org/2000/svg", "path");
|
||||
path.setAttribute(
|
||||
"d",
|
||||
"M12 22C6.477 22 2 17.523 2 12S6.477 2 12 2s10 4.477 10 10-4.477 10-10 10Zm0-2a8 8 0 1 0 0-16 8 8 0 0 0 0 16ZM9 9h6v6H9V9Z",
|
||||
);
|
||||
svg.append(path);
|
||||
return svg;
|
||||
}
|
||||
|
||||
function renderControlOverlay(): void {
|
||||
const overlayState = overlays.snapshot();
|
||||
if (!shouldShowAgentControlOverlay(overlayState)) {
|
||||
removeControlOverlay();
|
||||
setControlSurfaceActive(false);
|
||||
return;
|
||||
}
|
||||
|
||||
const root = ensureControlOverlayRoot();
|
||||
if (!root) return;
|
||||
|
||||
const pointerEvents = overlayState.automationBypassCount > 0 ? "none" : "auto";
|
||||
setControlSurfaceActive(true);
|
||||
root.innerHTML = "";
|
||||
Object.assign(root.style, {
|
||||
position: "fixed",
|
||||
inset: "0",
|
||||
pointerEvents: "none",
|
||||
});
|
||||
|
||||
const style = document.createElement("style");
|
||||
style.textContent = `
|
||||
@keyframes bsk-breathe {
|
||||
0%, 100% { box-shadow: inset 0 0 20px 4px rgba(249,115,22,0.25); }
|
||||
50% { box-shadow: inset 0 0 40px 8px rgba(249,115,22,0.5); }
|
||||
}
|
||||
`;
|
||||
|
||||
const border = document.createElement("div");
|
||||
border.setAttribute("data-slot", "control-overlay");
|
||||
Object.assign(border.style, {
|
||||
position: "fixed",
|
||||
inset: "0",
|
||||
zIndex: "2147483646",
|
||||
pointerEvents: "none",
|
||||
animation: "bsk-breathe 3s ease-in-out infinite",
|
||||
});
|
||||
|
||||
const blocker = document.createElement("div");
|
||||
blocker.setAttribute("data-slot", "control-overlay-blocker");
|
||||
Object.assign(blocker.style, {
|
||||
position: "fixed",
|
||||
inset: "0",
|
||||
zIndex: "2147483646",
|
||||
pointerEvents,
|
||||
background: "transparent",
|
||||
});
|
||||
blocker.addEventListener("pointerdown", (event) => {
|
||||
if (overlayState.automationBypassCount > 0) return;
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
});
|
||||
blocker.addEventListener("click", (event) => {
|
||||
if (overlayState.automationBypassCount > 0) return;
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
});
|
||||
blocker.addEventListener(
|
||||
"wheel",
|
||||
(event) => {
|
||||
if (overlayState.automationBypassCount > 0) return;
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
},
|
||||
{ passive: false },
|
||||
);
|
||||
blocker.addEventListener(
|
||||
"touchmove",
|
||||
(event) => {
|
||||
if (overlayState.automationBypassCount > 0) return;
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
},
|
||||
{ passive: false },
|
||||
);
|
||||
|
||||
const pill = document.createElement("div");
|
||||
pill.setAttribute("data-slot", "control-overlay-pill");
|
||||
Object.assign(pill.style, {
|
||||
position: "fixed",
|
||||
bottom: "32px",
|
||||
left: "50%",
|
||||
transform: "translateX(-50%)",
|
||||
zIndex: "2147483647",
|
||||
pointerEvents: "auto",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: "12px",
|
||||
borderRadius: "999px",
|
||||
backgroundColor: "#fff",
|
||||
padding: "10px 10px 10px 20px",
|
||||
boxShadow: "0 8px 32px rgba(124,45,18,0.16), 0 2px 8px rgba(0,0,0,0.1)",
|
||||
fontFamily:
|
||||
'-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif',
|
||||
});
|
||||
|
||||
const logo = document.createElement("img");
|
||||
logo.src = logoUrl;
|
||||
logo.alt = "browser-skill";
|
||||
Object.assign(logo.style, {
|
||||
width: "24px",
|
||||
height: "24px",
|
||||
borderRadius: "4px",
|
||||
flexShrink: "0",
|
||||
});
|
||||
|
||||
const label = document.createElement("span");
|
||||
label.textContent = i18n.t("controlOverlay.status", { ns: "extension" });
|
||||
Object.assign(label.style, {
|
||||
fontSize: "16px",
|
||||
fontWeight: "500",
|
||||
color: "#333",
|
||||
whiteSpace: "nowrap",
|
||||
userSelect: "none",
|
||||
});
|
||||
|
||||
const button = document.createElement("button");
|
||||
button.type = "button";
|
||||
button.setAttribute("data-slot", "control-overlay-stop-all");
|
||||
button.disabled = overlayState.interrupting;
|
||||
Object.assign(button.style, {
|
||||
pointerEvents: "auto",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: "6px",
|
||||
border: "none",
|
||||
borderRadius: "999px",
|
||||
padding: "8px 20px 8px 16px",
|
||||
fontSize: "15px",
|
||||
fontWeight: "600",
|
||||
color: "#fff",
|
||||
backgroundColor: overlayState.interrupting ? "#9ca3af" : "#f97316",
|
||||
cursor: overlayState.interrupting ? "default" : "pointer",
|
||||
opacity: overlayState.interrupting ? "0.7" : "1",
|
||||
transition: "background-color 150ms ease-out, opacity 150ms ease-out",
|
||||
whiteSpace: "nowrap",
|
||||
lineHeight: "1",
|
||||
});
|
||||
button.append(
|
||||
createStopCircleIcon(),
|
||||
document.createTextNode(
|
||||
overlayState.interrupting
|
||||
? i18n.t("controlOverlay.interrupting", { ns: "extension" })
|
||||
: i18n.t("controlOverlay.interrupt", { ns: "extension" }),
|
||||
),
|
||||
);
|
||||
button.addEventListener("click", (event) => {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
handleInterrupt();
|
||||
});
|
||||
|
||||
pill.append(logo, label, button);
|
||||
root.append(style, border, blocker, pill);
|
||||
}
|
||||
|
||||
function renderReactOverlays(): void {
|
||||
const overlayState = overlays.snapshot();
|
||||
reactRoot?.render(
|
||||
React.createElement(
|
||||
@@ -104,12 +332,6 @@ export default defineContentScript({
|
||||
React.createElement(BorrowConfirmationOverlay, {
|
||||
requests: overlayState.borrowRequests,
|
||||
}),
|
||||
React.createElement(ControlOverlay, {
|
||||
visible: shouldShowAgentControlOverlay(overlayState),
|
||||
interrupting: overlayState.interrupting,
|
||||
automationBypass: overlayState.automationBypassCount > 0,
|
||||
onInterrupt: handleInterrupt,
|
||||
}),
|
||||
React.createElement(HelpRequestOverlay, { request: overlayState.activeHelp }),
|
||||
React.createElement(RecordOverlay, { request: overlayState.activeRecord }),
|
||||
),
|
||||
@@ -117,6 +339,23 @@ export default defineContentScript({
|
||||
);
|
||||
}
|
||||
|
||||
function renderAll(): void {
|
||||
renderReactOverlays();
|
||||
renderControlOverlay();
|
||||
}
|
||||
|
||||
function clearCurrentAgentSession(): void {
|
||||
const sessionId = overlays.snapshot().activeSessionId;
|
||||
if (!sessionId) return;
|
||||
resetAgentOverlayState(sessionId);
|
||||
}
|
||||
|
||||
function applyOverlayState(state: OverlayAgentStateMessage): void {
|
||||
activeAgentState = state;
|
||||
overlays.applyAgentControlMode(state.sessionId, state.mode);
|
||||
renderAll();
|
||||
}
|
||||
|
||||
function resetAgentOverlayState(sessionId: string) {
|
||||
const previousHelp = overlays.resetAgentOverlays(sessionId);
|
||||
if (previousHelp) {
|
||||
@@ -125,7 +364,7 @@ export default defineContentScript({
|
||||
recordCapture?.dispose();
|
||||
recordCapture = null;
|
||||
activeRecordRequestId = null;
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
}
|
||||
|
||||
function handleInterrupt() {
|
||||
@@ -137,12 +376,8 @@ export default defineContentScript({
|
||||
return;
|
||||
}
|
||||
overlays.setInterrupting(true);
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
void sendInterrupt((msg) => chrome.runtime.sendMessage(msg), sessionId).then((reply) => {
|
||||
// Always retract the mask after the round trip resolves
|
||||
// (success, failure, or timeout). Cancellation is fire-and-
|
||||
// forget on the daemon side; the user must not be stuck
|
||||
// behind a transient issue. The Agent Window stays open.
|
||||
resetAgentOverlayState(sessionId);
|
||||
if (!reply.ok) {
|
||||
console.warn("[bsk overlay] interrupt did not get a clean ack from daemon");
|
||||
@@ -157,6 +392,7 @@ export default defineContentScript({
|
||||
| HelpRequestMessage
|
||||
| HelpCancelMessage
|
||||
| OverlayAgentOverlayResetMessage
|
||||
| OverlayAgentStateMessage
|
||||
| OverlayAutomationBypassMessage,
|
||||
_sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: BorrowResponseMessage | HelpAckMessage) => void,
|
||||
@@ -183,11 +419,11 @@ export default defineContentScript({
|
||||
});
|
||||
},
|
||||
});
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
},
|
||||
onStop: () => {
|
||||
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
},
|
||||
},
|
||||
sendResponse as unknown as
|
||||
@@ -205,7 +441,12 @@ export default defineContentScript({
|
||||
) {
|
||||
const bypassMsg = message as OverlayAutomationBypassMessage;
|
||||
overlays.setAutomationBypass(bypassMsg.enabled);
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isOverlayAgentStateMessage(message)) {
|
||||
applyOverlayState(message);
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -216,7 +457,7 @@ export default defineContentScript({
|
||||
|
||||
if (message.type === "borrow-cancel") {
|
||||
overlays.removeBorrowRequest(message.requestId);
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -224,7 +465,7 @@ export default defineContentScript({
|
||||
const state = overlays.snapshot();
|
||||
if (state.activeHelp && state.activeHelp.id === message.requestId) {
|
||||
overlays.clearAgentHelpRequest(message.requestId);
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -244,7 +485,7 @@ export default defineContentScript({
|
||||
if (previousHelp && previousHelp.id !== helpMsg.requestId) {
|
||||
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||
}
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
sendResponse({ type: HELP_ACK, ok: true });
|
||||
return false;
|
||||
}
|
||||
@@ -256,7 +497,7 @@ export default defineContentScript({
|
||||
responded = true;
|
||||
sendResponse({ type: "borrow-response", allowed });
|
||||
overlays.removeBorrowRequest(message.requestId);
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
};
|
||||
|
||||
overlays.addBorrowRequest({
|
||||
@@ -267,7 +508,7 @@ export default defineContentScript({
|
||||
onAllow: () => respond(true),
|
||||
onDeny: () => respond(false),
|
||||
});
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -286,7 +527,7 @@ export default defineContentScript({
|
||||
...(note ? { note } : {}),
|
||||
};
|
||||
overlays.clearAgentHelpRequest(requestId);
|
||||
renderOverlay();
|
||||
renderAll();
|
||||
await chrome.runtime.sendMessage(msg).catch((err) => {
|
||||
console.debug("[bsk overlay] help finish failed", err);
|
||||
});
|
||||
@@ -305,7 +546,7 @@ export default defineContentScript({
|
||||
});
|
||||
}
|
||||
|
||||
async function queryActiveHelpWithRetry(): Promise<boolean> {
|
||||
async function queryActiveHelpWithRetry(): Promise<void> {
|
||||
for (let attempt = 0; attempt < 6; attempt += 1) {
|
||||
try {
|
||||
const helpQuery = (await chrome.runtime.sendMessage({
|
||||
@@ -313,38 +554,21 @@ export default defineContentScript({
|
||||
})) as HelpQueryResponse | undefined;
|
||||
if (helpQuery?.active && helpQuery.request) {
|
||||
mountHelpRequest(helpQuery.request);
|
||||
renderOverlay();
|
||||
return true;
|
||||
renderAll();
|
||||
return;
|
||||
}
|
||||
} catch (err) {
|
||||
console.debug("[bsk overlay] help query failed", err);
|
||||
}
|
||||
await new Promise((resolve) => window.setTimeout(resolve, 150));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
async function syncAgentOverlay(): Promise<void> {
|
||||
if (!(await anySessionLive())) return;
|
||||
async function queryActiveRecord(): Promise<void> {
|
||||
try {
|
||||
const helpActive = await queryActiveHelpWithRetry();
|
||||
const reply = (await chrome.runtime.sendMessage({
|
||||
kind: OVERLAY_MSG_WHO_AM_I,
|
||||
})) as OverlayWhoAmIResponse | undefined;
|
||||
if (!reply?.sessionId) return;
|
||||
|
||||
// Query / re-arm recording *before* activating the control overlay so
|
||||
// record start (navigate → content load) does not flash
|
||||
// 「Agent 正在控制」before RecordOverlay mounts.
|
||||
let recordQuery: { active?: boolean; requestId?: string } | undefined;
|
||||
try {
|
||||
recordQuery = (await chrome.runtime.sendMessage({
|
||||
type: RECORD_QUERY,
|
||||
})) as { active?: boolean; requestId?: string } | undefined;
|
||||
} catch (err) {
|
||||
console.debug("[bsk overlay] record query failed", err);
|
||||
}
|
||||
|
||||
const recordQuery = (await chrome.runtime.sendMessage({
|
||||
type: RECORD_QUERY,
|
||||
})) as { active?: boolean; requestId?: string } | undefined;
|
||||
if (
|
||||
recordQuery?.active &&
|
||||
typeof recordQuery.requestId === "string" &&
|
||||
@@ -360,26 +584,38 @@ export default defineContentScript({
|
||||
});
|
||||
},
|
||||
});
|
||||
renderAll();
|
||||
}
|
||||
|
||||
if (!helpActive && overlays.snapshot().activeHelp === null) {
|
||||
void queryActiveHelpWithRetry();
|
||||
}
|
||||
|
||||
overlays.activateAgentSession(reply.sessionId);
|
||||
renderOverlay();
|
||||
} catch (err) {
|
||||
console.debug("[bsk overlay] syncAgentOverlay failed", err);
|
||||
console.debug("[bsk overlay] record query failed", err);
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshAuxiliaryOverlayState(): Promise<void> {
|
||||
await Promise.all([queryActiveHelpWithRetry(), queryActiveRecord()]);
|
||||
}
|
||||
|
||||
async function requestOverlayState(): Promise<void> {
|
||||
try {
|
||||
const state = (await chrome.runtime.sendMessage({
|
||||
kind: OVERLAY_MSG_READY,
|
||||
})) as OverlayAgentStateMessage | undefined;
|
||||
if (state && isOverlayAgentStateMessage(state)) {
|
||||
applyOverlayState(state);
|
||||
}
|
||||
void refreshAuxiliaryOverlayState();
|
||||
} catch (err) {
|
||||
console.debug("[bsk overlay] overlay.ready failed", err);
|
||||
}
|
||||
}
|
||||
|
||||
const onPageShow = (event: PageTransitionEvent) => {
|
||||
if (event.persisted) void syncAgentOverlay();
|
||||
if (event.persisted) void requestOverlayState();
|
||||
};
|
||||
|
||||
ui.mount();
|
||||
chrome.runtime.onMessage.addListener(onMessage);
|
||||
void syncAgentOverlay();
|
||||
void requestOverlayState();
|
||||
|
||||
window.addEventListener("pageshow", onPageShow);
|
||||
|
||||
@@ -391,7 +627,8 @@ export default defineContentScript({
|
||||
remountInProgress = true;
|
||||
try {
|
||||
ui.mount();
|
||||
void syncAgentOverlay();
|
||||
if (activeAgentState) applyOverlayState(activeAgentState);
|
||||
void requestOverlayState();
|
||||
} finally {
|
||||
remountInProgress = false;
|
||||
}
|
||||
@@ -414,16 +651,3 @@ export default defineContentScript({
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
async function anySessionLive(): Promise<boolean> {
|
||||
if (!chrome.storage?.session?.get) return true;
|
||||
try {
|
||||
const result = (await chrome.storage.session.get({
|
||||
[SESSIONS_LIVE_FLAG_KEY]: false,
|
||||
})) as Record<string, unknown> | undefined;
|
||||
return Boolean(result?.[SESSIONS_LIVE_FLAG_KEY]);
|
||||
} catch (err) {
|
||||
console.debug("[bsk overlay] sessions-live flag read failed", err);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
*
|
||||
* Mirrors the borrow-confirmation message style (see
|
||||
* `@/tools/borrow-confirmation`). The background asks a specific tab to
|
||||
* enter "help mode" (render HelpRequestOverlay + hide ControlOverlay);
|
||||
* enter "help mode" (render HelpRequestOverlay + hide the control mask);
|
||||
* the content script acks display, then later reports Done / Cancel.
|
||||
*/
|
||||
|
||||
|
||||
@@ -3,9 +3,8 @@
|
||||
* between the content-script control overlay and the background SW.
|
||||
*
|
||||
* Content script → background:
|
||||
* - `{ kind: "overlay.who_am_i", tabId, windowId }` → background
|
||||
* replies `{ sessionId: string | null }` so the overlay knows
|
||||
* whether to render itself.
|
||||
* - `{ kind: "overlay.ready" }` → background replies with the authoritative
|
||||
* overlay state for the sender's window.
|
||||
* - `{ kind: "overlay.interrupt", sessionId }` → background asks the
|
||||
* daemon (via a `session.user_interrupt` WS event) to cancel
|
||||
* every inflight + queued tool call for that session with
|
||||
@@ -14,6 +13,7 @@
|
||||
*/
|
||||
|
||||
export const OVERLAY_MSG_WHO_AM_I = "overlay.who_am_i";
|
||||
export const OVERLAY_MSG_READY = "overlay.ready";
|
||||
export const OVERLAY_MSG_INTERRUPT = "overlay.interrupt";
|
||||
|
||||
/**
|
||||
@@ -57,6 +57,22 @@ export interface OverlayWhoAmIResponse {
|
||||
sessionId: string | null;
|
||||
}
|
||||
|
||||
export interface OverlayReadyRequest {
|
||||
kind: typeof OVERLAY_MSG_READY;
|
||||
}
|
||||
|
||||
export type OverlayMode = "control" | "interrupting" | "paused" | "hidden";
|
||||
|
||||
/** Background → content: complete, authoritative control-overlay state. */
|
||||
export const OVERLAY_AGENT_STATE = "bh-agent-overlay-state";
|
||||
|
||||
export interface OverlayAgentStateMessage {
|
||||
type: typeof OVERLAY_AGENT_STATE;
|
||||
sessionId: string | null;
|
||||
mode: OverlayMode;
|
||||
generation: number;
|
||||
}
|
||||
|
||||
export interface OverlayInterruptRequest {
|
||||
kind: typeof OVERLAY_MSG_INTERRUPT;
|
||||
sessionId: string;
|
||||
@@ -90,4 +106,23 @@ export function isOverlayAgentOverlayResetMessage(
|
||||
return candidate.type === OVERLAY_AGENT_OVERLAY_RESET && typeof candidate.sessionId === "string";
|
||||
}
|
||||
|
||||
export type OverlayMessage = OverlayWhoAmIRequest | OverlayInterruptRequest;
|
||||
export function isOverlayAgentStateMessage(message: unknown): message is OverlayAgentStateMessage {
|
||||
if (!message || typeof message !== "object") return false;
|
||||
const candidate = message as {
|
||||
type?: unknown;
|
||||
sessionId?: unknown;
|
||||
mode?: unknown;
|
||||
generation?: unknown;
|
||||
};
|
||||
return (
|
||||
candidate.type === OVERLAY_AGENT_STATE &&
|
||||
(typeof candidate.sessionId === "string" || candidate.sessionId === null) &&
|
||||
(candidate.mode === "control" ||
|
||||
candidate.mode === "interrupting" ||
|
||||
candidate.mode === "paused" ||
|
||||
candidate.mode === "hidden") &&
|
||||
typeof candidate.generation === "number"
|
||||
);
|
||||
}
|
||||
|
||||
export type OverlayMessage = OverlayWhoAmIRequest | OverlayReadyRequest | OverlayInterruptRequest;
|
||||
|
||||
@@ -274,6 +274,35 @@ describe("ToolDispatcher", () => {
|
||||
expect(onSessionsChanged).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => {
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 1),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const onBrowserControlResumed = vi.fn();
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, onBrowserControlResumed });
|
||||
dispatcher.start();
|
||||
|
||||
deliver(makeRequest("tool.session_start", { session_id: "aa11" }));
|
||||
await flushMicrotasks();
|
||||
sent.length = 0;
|
||||
|
||||
deliver({ ...makeRequest("tool.snapshot", { session_id: "aa11" }), id: "r-passive" });
|
||||
await flushMicrotasks();
|
||||
expect(onBrowserControlResumed).not.toHaveBeenCalled();
|
||||
|
||||
deliver({
|
||||
...makeRequest("tool.tab_create", { session_id: "aa11", url: "https://example.test/" }),
|
||||
id: "r-control",
|
||||
});
|
||||
await flushMicrotasks();
|
||||
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
|
||||
});
|
||||
|
||||
it("disconnects the transport when send() fails so keepalive can rebuild", async () => {
|
||||
const { transport, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
buildVomScene,
|
||||
type CdpAxNode,
|
||||
handleGetHtml,
|
||||
handleObserve,
|
||||
handleScreenshot,
|
||||
handleSnapshot,
|
||||
parsePngDimensions,
|
||||
@@ -1412,7 +1413,7 @@ describe("buildVomScene", () => {
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
]);
|
||||
expect(renderVom(scene).text).toContain('@e1 button "Products" [→ Shoes | Bags]');
|
||||
expect(renderVom(scene).text).toContain('@e1 button "Products" [hover: Shoes | Bags]');
|
||||
});
|
||||
|
||||
it("enriches names and active scope signals from AX properties", () => {
|
||||
@@ -1741,6 +1742,225 @@ describe("handleSnapshot", () => {
|
||||
expect(ctx.refStore.resolve("e2")).toBeNull();
|
||||
});
|
||||
|
||||
it("keeps snapshots static without conditional surface probing", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const root: CdpAxNode = {
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
name: { type: "computedString", value: "Example" },
|
||||
backendDOMNodeId: 100,
|
||||
childIds: ["2"],
|
||||
};
|
||||
const button: CdpAxNode = {
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Products" },
|
||||
backendDOMNodeId: 200,
|
||||
};
|
||||
const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"];
|
||||
const i = (s: string) => strings.indexOf(s);
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("button")],
|
||||
backendNodeId: [100, 200],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[20, 20, 120, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (method === "Runtime.evaluate") return { result: { value: [] } };
|
||||
throw new Error(`unexpected CDP method ${method}`);
|
||||
});
|
||||
const res = await handleSnapshot(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) =>
|
||||
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(send).not.toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({ returnByValue: true }),
|
||||
);
|
||||
});
|
||||
|
||||
it("enables conditional surface probing for semantic observe", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const root: CdpAxNode = {
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
name: { type: "computedString", value: "Example" },
|
||||
backendDOMNodeId: 100,
|
||||
childIds: ["2"],
|
||||
};
|
||||
const button: CdpAxNode = {
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Products" },
|
||||
backendDOMNodeId: 200,
|
||||
};
|
||||
const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"];
|
||||
const i = (s: string) => strings.indexOf(s);
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("button")],
|
||||
backendNodeId: [100, 200],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[20, 20, 120, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (method === "Runtime.evaluate") return { result: { value: [] } };
|
||||
throw new Error(`unexpected CDP method ${method}`);
|
||||
});
|
||||
const res = await handleObserve(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) =>
|
||||
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({ returnByValue: true }),
|
||||
);
|
||||
});
|
||||
|
||||
it("allows passive snapshots of explicit user-window tabs", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const root: CdpAxNode = {
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
name: { type: "computedString", value: "User tab" },
|
||||
backendDOMNodeId: 100,
|
||||
childIds: ["2"],
|
||||
};
|
||||
const button: CdpAxNode = {
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Read" },
|
||||
backendDOMNodeId: 200,
|
||||
};
|
||||
const deps = makeDeps([root, button]);
|
||||
deps.tabsApi.get = vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 200, active: true }) as chrome.tabs.Tab,
|
||||
);
|
||||
|
||||
const res = await handleSnapshot(sm, { session_id: "aa11", tab_id: 9 }, deps);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.tab_id).toBe(9);
|
||||
expect(res.text).toContain('@e1 button "Read"');
|
||||
expect(deps.send).toHaveBeenCalledWith(9, "Accessibility.enable", {});
|
||||
});
|
||||
|
||||
it("rejects semantic observe on explicit user-window tabs before CDP probing", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const deps = makeDeps([]);
|
||||
deps.tabsApi.get = vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 200, active: true }) as chrome.tabs.Tab,
|
||||
);
|
||||
|
||||
const res = await handleObserve(sm, { session_id: "aa11", tab_id: 9 }, deps);
|
||||
|
||||
expect(res).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "agent_window_scope" },
|
||||
});
|
||||
expect(deps.send).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("resets the RefStore on every fresh snapshot", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { lookupSession, parseBufferedReadBounds } from "../shared";
|
||||
import {
|
||||
cdpBlockedUrlReason,
|
||||
enforceCdpAccessibleTarget,
|
||||
lookupSession,
|
||||
parseBufferedReadBounds,
|
||||
resolveCdpAccessibleTargetTab,
|
||||
} from "../shared";
|
||||
|
||||
function fakeAgentWindow() {
|
||||
return {
|
||||
@@ -67,3 +73,101 @@ describe("parseBufferedReadBounds", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("CDP target URL guard", () => {
|
||||
it("allows ordinary pages and about:blank", () => {
|
||||
expect(cdpBlockedUrlReason("https://example.test/path")).toBeNull();
|
||||
expect(cdpBlockedUrlReason("http://example.test/path")).toBeNull();
|
||||
expect(cdpBlockedUrlReason("about:blank")).toBeNull();
|
||||
});
|
||||
|
||||
it("blocks browser and extension internal pages", () => {
|
||||
expect(cdpBlockedUrlReason("chrome-extension://abc/options.html")).toBe("chrome-extension:");
|
||||
expect(cdpBlockedUrlReason("chrome://extensions")).toBe("chrome:");
|
||||
expect(cdpBlockedUrlReason("about:newtab")).toBe("about:");
|
||||
});
|
||||
|
||||
it("returns a structured permission error before page CDP access", () => {
|
||||
expect(
|
||||
enforceCdpAccessibleTarget(
|
||||
{
|
||||
tabId: 7,
|
||||
windowId: 100,
|
||||
active: true,
|
||||
url: "chrome-extension://other-extension/page.html",
|
||||
},
|
||||
"snapshot",
|
||||
),
|
||||
).toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "restricted_tab_url" },
|
||||
message: expect.stringContaining("snapshot cannot access tab 7"),
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back from a restricted default active tab to an accessible agent tab", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
const tabsApi = {
|
||||
get: vi.fn(),
|
||||
query: vi.fn(async (query: chrome.tabs.QueryInfo) => {
|
||||
if (query.active) {
|
||||
return [
|
||||
{
|
||||
id: 7,
|
||||
windowId: ctx.agentWindowId,
|
||||
active: true,
|
||||
url: "chrome-extension://other-extension/page.html",
|
||||
} as chrome.tabs.Tab,
|
||||
];
|
||||
}
|
||||
return [
|
||||
{
|
||||
id: 7,
|
||||
windowId: ctx.agentWindowId,
|
||||
active: true,
|
||||
url: "chrome-extension://other-extension/page.html",
|
||||
} as chrome.tabs.Tab,
|
||||
{
|
||||
id: 8,
|
||||
windowId: ctx.agentWindowId,
|
||||
active: false,
|
||||
url: "https://example.test/",
|
||||
} as chrome.tabs.Tab,
|
||||
];
|
||||
}),
|
||||
};
|
||||
|
||||
await expect(
|
||||
resolveCdpAccessibleTargetTab(sm, ctx, undefined, tabsApi, "snapshot"),
|
||||
).resolves.toMatchObject({
|
||||
tabId: 8,
|
||||
url: "https://example.test/",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not replace an explicitly requested restricted tab", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
|
||||
const ctx = await sm.start("aa11");
|
||||
const tabsApi = {
|
||||
get: vi.fn(
|
||||
async () =>
|
||||
({
|
||||
id: 7,
|
||||
windowId: ctx.agentWindowId,
|
||||
active: true,
|
||||
url: "chrome-extension://other-extension/page.html",
|
||||
}) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(),
|
||||
};
|
||||
|
||||
await expect(
|
||||
resolveCdpAccessibleTargetTab(sm, ctx, 7, tabsApi, "snapshot"),
|
||||
).resolves.toMatchObject({
|
||||
code: "permission_denied",
|
||||
data: { reason: "restricted_tab_url" },
|
||||
});
|
||||
expect(tabsApi.query).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -11,6 +11,7 @@ import type {
|
||||
NavigateForwardParams,
|
||||
NavigateParams,
|
||||
NetworkParams,
|
||||
ObserveParams,
|
||||
PressParams,
|
||||
ProtocolFrame,
|
||||
RecordAwaitParams,
|
||||
@@ -42,6 +43,7 @@ import {
|
||||
type CdpRunner,
|
||||
chromeTabsCaptureApi,
|
||||
handleGetHtml,
|
||||
handleObserve,
|
||||
handleScreenshot,
|
||||
handleSnapshot,
|
||||
} from "./observation";
|
||||
@@ -86,6 +88,8 @@ export interface DispatcherDeps {
|
||||
* `chrome.storage.session` "sessions live" flag (review M4/M5 I3).
|
||||
*/
|
||||
onSessionsChanged?: () => void;
|
||||
/** Invoked before a tool that dispatches page input or mutates browser state is forwarded. */
|
||||
onBrowserControlResumed?: (sessionId: string) => void;
|
||||
/** User approval for `tool.tab_borrow` (overlay in content script). */
|
||||
approveBorrow?: BorrowConfirmationApprover;
|
||||
/** i18n notification copy for `tool.request_help` (resolved per-call). */
|
||||
@@ -114,6 +118,7 @@ export class ToolDispatcher {
|
||||
private readonly sessions: SessionManager;
|
||||
private readonly cdp?: DispatcherCdpRunner;
|
||||
private readonly onSessionsChanged?: () => void;
|
||||
private readonly onBrowserControlResumed?: (sessionId: string) => void;
|
||||
private readonly approveBorrow?: BorrowConfirmationApprover;
|
||||
private readonly helpNotificationCopy?: () => { title: string; body: string };
|
||||
private subscription: { dispose(): void } | null = null;
|
||||
@@ -130,6 +135,7 @@ export class ToolDispatcher {
|
||||
this.sessions = deps.sessions;
|
||||
this.cdp = deps.cdp;
|
||||
this.onSessionsChanged = deps.onSessionsChanged;
|
||||
this.onBrowserControlResumed = deps.onBrowserControlResumed;
|
||||
this.approveBorrow = deps.approveBorrow;
|
||||
this.helpNotificationCopy = deps.helpNotificationCopy;
|
||||
}
|
||||
@@ -193,6 +199,8 @@ export class ToolDispatcher {
|
||||
let body: ResponseFrame;
|
||||
let startedSession: string | null = null;
|
||||
try {
|
||||
const sessionId = sessionIdForBrowserControlMethod(req);
|
||||
if (sessionId) this.onBrowserControlResumed?.(sessionId);
|
||||
const result = await Promise.race([this.invoke(req, ac.signal), abortPromise(ac.signal)]);
|
||||
if (isRpcError(result)) {
|
||||
body = { id: req.id, error: result };
|
||||
@@ -304,6 +312,12 @@ export class ToolDispatcher {
|
||||
req.params as SnapshotParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||
);
|
||||
case "tool.observe":
|
||||
return handleObserve(
|
||||
this.sessions,
|
||||
req.params as ObserveParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||
);
|
||||
case "tool.get_html":
|
||||
return handleGetHtml(
|
||||
this.sessions,
|
||||
@@ -457,6 +471,33 @@ function isRpcError(v: unknown): v is RpcError {
|
||||
);
|
||||
}
|
||||
|
||||
function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
||||
switch (req.method) {
|
||||
case "tool.tab_create":
|
||||
case "tool.tab_close":
|
||||
case "tool.tab_select":
|
||||
case "tool.tab_borrow":
|
||||
case "tool.tab_return":
|
||||
case "tool.navigate":
|
||||
case "tool.navigate_back":
|
||||
case "tool.navigate_forward":
|
||||
case "tool.reload":
|
||||
case "tool.click":
|
||||
case "tool.fill":
|
||||
case "tool.press":
|
||||
case "tool.select":
|
||||
case "tool.evaluate":
|
||||
case "tool.observe":
|
||||
case "tool.request_help":
|
||||
case "tool.record_start": {
|
||||
const sessionId = (req.params as { session_id?: unknown } | undefined)?.session_id;
|
||||
return typeof sessionId === "string" && sessionId.length > 0 ? sessionId : null;
|
||||
}
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves never; rejects with `AbortLikeError` as soon as the signal
|
||||
* fires (or immediately if it is already aborted). Used by the
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Read-only observation handlers — `tool.screenshot`, `tool.snapshot`,
|
||||
// and `tool.get_html` (design §7). Each handler resolves the target
|
||||
// Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`,
|
||||
// and semantic `tool.observe` (design §7). Each handler resolves the target
|
||||
// tab (defaulting to the Agent Window's active tab when omitted) and
|
||||
// returns a payload that mirrors the bsk-protocol Rust structs.
|
||||
|
||||
@@ -15,6 +15,8 @@ import type { SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
GetHtmlParams,
|
||||
GetHtmlResult,
|
||||
ObserveParams,
|
||||
ObserveResult,
|
||||
RpcError,
|
||||
ScreenshotParams,
|
||||
ScreenshotResult,
|
||||
@@ -26,11 +28,13 @@ import { nodeBoundingRect, scrollNodeIntoView } from "./element-geometry";
|
||||
import { rpcError } from "./errors";
|
||||
import {
|
||||
type ChromeTabsApi,
|
||||
enforceToolTargetScope,
|
||||
isRpcError,
|
||||
lookupSession,
|
||||
resolveTargetTab,
|
||||
resolveCdpAccessibleTargetTab,
|
||||
type CdpRunner as SharedCdpRunner,
|
||||
normaliseRef as sharedNormaliseRef,
|
||||
type ToolEffect,
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import {
|
||||
@@ -167,7 +171,13 @@ export async function handleScreenshot(
|
||||
const ctxOrErr = lookupSession(manager, params, "screenshot");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
const target = await resolveCdpAccessibleTargetTab(
|
||||
manager,
|
||||
ctx,
|
||||
params.tab_id,
|
||||
deps.tabsApi,
|
||||
"screenshot",
|
||||
);
|
||||
if (isRpcError(target)) return target;
|
||||
const dialogCursor = deps.cdp ? markDialogCursor(deps.cdp, target.tabId) : 0;
|
||||
const withShotDialogs = <T extends object>(result: T) =>
|
||||
@@ -181,6 +191,7 @@ export async function handleScreenshot(
|
||||
const node = resolveSnapshotRef(ctx, ref, target.tabId);
|
||||
if (isRpcError(node)) return node;
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
const captured = await captureElementScreenshot(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(captured)) return captured;
|
||||
return withShotDialogs({
|
||||
@@ -664,6 +675,7 @@ function inputStateFor(
|
||||
value: string | undefined,
|
||||
): VomNode["inputState"] {
|
||||
if (!capturedNode || !FORM_CONTROL_TAGS.has(normalizeTag(capturedNode.tag))) return undefined;
|
||||
if (capturedNode.formState) return capturedNode.formState;
|
||||
const formValue = capturedNode.formValue;
|
||||
if (formValue !== undefined) {
|
||||
if (formValue === "") return "empty";
|
||||
@@ -1099,7 +1111,13 @@ export async function handleGetHtml(
|
||||
const ctxOrErr = lookupSession(manager, params, "get_html");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
const target = await resolveCdpAccessibleTargetTab(
|
||||
manager,
|
||||
ctx,
|
||||
params.tab_id,
|
||||
deps.tabsApi,
|
||||
"get_html",
|
||||
);
|
||||
if (isRpcError(target)) return target;
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
@@ -1108,6 +1126,7 @@ export async function handleGetHtml(
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
let html: string;
|
||||
if (params.ref) {
|
||||
const resolved = resolveSnapshotRef(ctx, params.ref, target.tabId);
|
||||
@@ -1178,28 +1197,44 @@ async function fallbackCapturedViewModel(
|
||||
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
async function captureForVom(cdp: CdpRunner, tabId: number): Promise<CapturedViewModel> {
|
||||
async function captureForVom(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
conditionalSurfaceProbe: boolean,
|
||||
): Promise<CapturedViewModel> {
|
||||
try {
|
||||
return await captureViewModel(cdp, tabId);
|
||||
return await captureViewModel(cdp, tabId, { conditionalSurfaceProbe });
|
||||
} catch {
|
||||
return fallbackCapturedViewModel(cdp, tabId);
|
||||
}
|
||||
}
|
||||
|
||||
export async function handleSnapshot(
|
||||
async function handleVomObservation(
|
||||
manager: SessionManager,
|
||||
params: SnapshotParams,
|
||||
params: SnapshotParams | ObserveParams,
|
||||
toolName: "snapshot" | "observe",
|
||||
effect: ToolEffect,
|
||||
conditionalSurfaceProbe: boolean,
|
||||
deps: SnapshotDeps = getDefaultDeps(),
|
||||
): Promise<SnapshotResult | RpcError> {
|
||||
const ctxOrErr = lookupSession(manager, params, "snapshot");
|
||||
): Promise<SnapshotResult | ObserveResult | RpcError> {
|
||||
const ctxOrErr = lookupSession(manager, params, toolName);
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
const target = await resolveCdpAccessibleTargetTab(
|
||||
manager,
|
||||
ctx,
|
||||
params.tab_id,
|
||||
deps.tabsApi,
|
||||
toolName,
|
||||
);
|
||||
if (isRpcError(target)) return target;
|
||||
const denied = enforceToolTargetScope(ctx, target, effect, toolName);
|
||||
if (denied) return denied;
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
await deps.cdp.send<unknown>(target.tabId, "Accessibility.enable", {});
|
||||
const result = await deps.cdp.send<{ nodes: CdpAxNode[] }>(
|
||||
target.tabId,
|
||||
@@ -1207,7 +1242,7 @@ export async function handleSnapshot(
|
||||
{},
|
||||
);
|
||||
const axNodes = result.nodes ?? [];
|
||||
const captured = await captureForVom(deps.cdp, target.tabId);
|
||||
const captured = await captureForVom(deps.cdp, target.tabId, conditionalSurfaceProbe);
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
const rendered = renderVom(scene, {
|
||||
maxDepth: params.max_depth,
|
||||
@@ -1232,3 +1267,19 @@ export async function handleSnapshot(
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function handleSnapshot(
|
||||
manager: SessionManager,
|
||||
params: SnapshotParams,
|
||||
deps: SnapshotDeps = getDefaultDeps(),
|
||||
): Promise<SnapshotResult | RpcError> {
|
||||
return handleVomObservation(manager, params, "snapshot", "passive_read", false, deps);
|
||||
}
|
||||
|
||||
export async function handleObserve(
|
||||
manager: SessionManager,
|
||||
params: ObserveParams,
|
||||
deps: SnapshotDeps = getDefaultDeps(),
|
||||
): Promise<ObserveResult | RpcError> {
|
||||
return handleVomObservation(manager, params, "observe", "transient_input", true, deps);
|
||||
}
|
||||
|
||||
@@ -369,7 +369,7 @@ async function rearmRecording(
|
||||
deps: RecordDeps,
|
||||
): Promise<boolean> {
|
||||
// Do NOT toggle automation-bypass here: each retry used to increment the
|
||||
// content-script counter, and a single stop decrement left the ControlOverlay
|
||||
// content-script counter, and a single stop decrement left the control mask
|
||||
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
|
||||
// already hides the control chrome while activeRecord is set.
|
||||
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
|
||||
@@ -561,9 +561,9 @@ export async function handleRecordStart(
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
|
||||
// Register the recording *before* navigate so content-script syncAgentOverlay
|
||||
// on the destination page can RECORD_QUERY → rearm → show RecordOverlay
|
||||
// instead of flashing ControlOverlay ("Agent 正在控制").
|
||||
// Register the recording *before* navigate so the destination content script's
|
||||
// overlay.ready flow can RECORD_QUERY → rearm → show RecordOverlay
|
||||
// instead of flashing the control mask ("Agent 正在控制").
|
||||
const requestId = makeRequestId(target.tabId);
|
||||
let resolveFinish!: (trace: Trace) => void;
|
||||
let rejectFinish!: (err: Error) => void;
|
||||
|
||||
@@ -32,6 +32,13 @@ export const chromeTabsApi: ChromeTabsApi = {
|
||||
query: (q) => chrome.tabs.query(q),
|
||||
};
|
||||
|
||||
export interface ResolvedTargetTab {
|
||||
tabId: number;
|
||||
windowId: number;
|
||||
active: boolean;
|
||||
url?: string;
|
||||
}
|
||||
|
||||
export type { DialogCursor };
|
||||
|
||||
/**
|
||||
@@ -44,6 +51,7 @@ export type { DialogCursor };
|
||||
*/
|
||||
export interface CdpRunner {
|
||||
send<T = unknown>(tabId: number, method: string, params?: object): Promise<T>;
|
||||
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
|
||||
trackSessionTab?(sessionId: string, tabId: number): void;
|
||||
onEvent?(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): {
|
||||
dispose(): void;
|
||||
@@ -146,7 +154,7 @@ export async function resolveTargetTab(
|
||||
ctx: SessionContext,
|
||||
tabId: number | undefined,
|
||||
api: ChromeTabsApi,
|
||||
): Promise<{ tabId: number; windowId: number; active: boolean } | RpcError> {
|
||||
): Promise<ResolvedTargetTab | RpcError> {
|
||||
if (tabId !== undefined) {
|
||||
if (!Number.isSafeInteger(tabId) || tabId <= 0) {
|
||||
return {
|
||||
@@ -176,7 +184,7 @@ export async function resolveTargetTab(
|
||||
message: `tab ${tabId} not found in session scope`,
|
||||
};
|
||||
}
|
||||
return { tabId: tab.id, windowId: tab.windowId, active: tab.active === true };
|
||||
return { tabId: tab.id, windowId: tab.windowId, active: tab.active === true, url: tab.url };
|
||||
}
|
||||
const tabs = await api.query({ active: true, windowId: ctx.agentWindowId });
|
||||
const first = tabs.find((t) => typeof t.id === "number");
|
||||
@@ -186,7 +194,12 @@ export async function resolveTargetTab(
|
||||
message: `no active tab in Agent Window ${ctx.agentWindowId}`,
|
||||
};
|
||||
}
|
||||
return { tabId: first.id, windowId: ctx.agentWindowId, active: first.active === true };
|
||||
return {
|
||||
tabId: first.id,
|
||||
windowId: ctx.agentWindowId,
|
||||
active: first.active === true,
|
||||
url: first.url,
|
||||
};
|
||||
}
|
||||
|
||||
export function isRpcError(v: unknown): v is RpcError {
|
||||
@@ -202,6 +215,91 @@ export function isRpcError(v: unknown): v is RpcError {
|
||||
/** Re-export so M6/M7 tools keep a stable import path. */
|
||||
export { normaliseRef };
|
||||
|
||||
export type ToolEffect = "passive_read" | "transient_input" | "browser_mutation";
|
||||
|
||||
const CDP_BLOCKED_PROTOCOLS = new Set([
|
||||
"chrome:",
|
||||
"chrome-extension:",
|
||||
"devtools:",
|
||||
"edge:",
|
||||
"brave:",
|
||||
"vivaldi:",
|
||||
"opera:",
|
||||
]);
|
||||
|
||||
export function cdpBlockedUrlReason(url: string | undefined): string | null {
|
||||
if (!url) return null;
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (CDP_BLOCKED_PROTOCOLS.has(parsed.protocol)) return parsed.protocol;
|
||||
if (parsed.protocol === "about:" && parsed.pathname !== "blank") return "about:";
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Page CDP tools cannot inspect browser/extension internal pages. Check this
|
||||
* at the tool boundary so Chrome's low-level "Cannot access..." errors do not
|
||||
* leak as ambiguous page-read failures.
|
||||
*/
|
||||
export function enforceCdpAccessibleTarget(
|
||||
target: ResolvedTargetTab,
|
||||
toolName: string,
|
||||
): RpcError | null {
|
||||
const reason = cdpBlockedUrlReason(target.url);
|
||||
if (!reason) return null;
|
||||
return rpcError(
|
||||
"permission_denied",
|
||||
"restricted_tab_url",
|
||||
`${toolName} cannot access tab ${target.tabId} because its URL is ${target.url}; navigate the Agent Window to a web page first`,
|
||||
);
|
||||
}
|
||||
|
||||
function resolvedTargetFromChromeTab(
|
||||
tab: chrome.tabs.Tab,
|
||||
fallbackWindowId: number,
|
||||
): ResolvedTargetTab | null {
|
||||
if (typeof tab.id !== "number") return null;
|
||||
return {
|
||||
tabId: tab.id,
|
||||
windowId: typeof tab.windowId === "number" ? tab.windowId : fallbackWindowId,
|
||||
active: tab.active === true,
|
||||
url: tab.url,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a target for page CDP reads. Explicit tab_ids stay exact and are
|
||||
* rejected when browser security blocks the URL. For default targeting, skip a
|
||||
* restricted Agent Window active tab if another CDP-accessible tab exists in
|
||||
* the same Agent Window.
|
||||
*/
|
||||
export async function resolveCdpAccessibleTargetTab(
|
||||
manager: SessionManager,
|
||||
ctx: SessionContext,
|
||||
tabId: number | undefined,
|
||||
api: ChromeTabsApi,
|
||||
toolName: string,
|
||||
): Promise<ResolvedTargetTab | RpcError> {
|
||||
const target = await resolveTargetTab(manager, ctx, tabId, api);
|
||||
if (isRpcError(target)) return target;
|
||||
|
||||
const restricted = enforceCdpAccessibleTarget(target, toolName);
|
||||
if (!restricted) return target;
|
||||
if (tabId !== undefined) return restricted;
|
||||
|
||||
const tabs = await api.query({ windowId: ctx.agentWindowId });
|
||||
for (const tab of tabs) {
|
||||
const candidate = resolvedTargetFromChromeTab(tab, ctx.agentWindowId);
|
||||
if (!candidate) continue;
|
||||
if (!enforceCdpAccessibleTarget(candidate, toolName)) return candidate;
|
||||
}
|
||||
return restricted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sandbox guard: M7 write tools (click / fill / press / navigate*)
|
||||
* MUST refuse to touch a tab outside the session's Agent Window
|
||||
@@ -224,3 +322,17 @@ export function enforceAgentWindow(
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Unified target-scope policy by tool effect. Passive reads may inspect user
|
||||
* tabs; any tool that dispatches page input must stay inside the Agent Window.
|
||||
*/
|
||||
export function enforceToolTargetScope(
|
||||
ctx: SessionContext,
|
||||
target: { tabId: number; windowId: number },
|
||||
effect: ToolEffect,
|
||||
toolName: string,
|
||||
): RpcError | null {
|
||||
if (effect === "passive_read") return null;
|
||||
return enforceAgentWindow(ctx, target, toolName);
|
||||
}
|
||||
|
||||
@@ -59,6 +59,23 @@ function makeCdp(snapshot: unknown) {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "Runtime.evaluate") {
|
||||
return {
|
||||
result: {
|
||||
value: {
|
||||
controls: [
|
||||
{
|
||||
state: "filled",
|
||||
sensitive: true,
|
||||
defaultValue: "",
|
||||
placeholder: "secret",
|
||||
},
|
||||
],
|
||||
childFrames: [],
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
@@ -80,6 +97,89 @@ describe("captureViewModel", () => {
|
||||
expect(input?.attrs).toEqual({ type: "password" });
|
||||
});
|
||||
|
||||
it("does not run conditional surface probes by default", async () => {
|
||||
const cdp = makeCdp(fakeSnapshotReply());
|
||||
const result = await captureViewModel(cdp, 4);
|
||||
|
||||
expect(result.surfaceProbes).toEqual([]);
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything());
|
||||
});
|
||||
|
||||
it("batch-enriches form controls without per-node object resolution", async () => {
|
||||
const cdp = makeCdp(fakeSnapshotReply());
|
||||
const result = await captureViewModel(cdp, 4);
|
||||
const input = result.nodes.find((node) => node.backendNodeId === 13);
|
||||
|
||||
expect(input).toMatchObject({
|
||||
formState: "filled",
|
||||
formDefaultValue: "",
|
||||
formPlaceholder: "secret",
|
||||
});
|
||||
expect(input?.formValue).toBeUndefined();
|
||||
expect(cdp.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({ returnByValue: true }),
|
||||
);
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "DOM.resolveNode", expect.anything());
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "Runtime.callFunctionOn", expect.anything());
|
||||
});
|
||||
|
||||
it("runs hover surface probes when explicitly enabled", async () => {
|
||||
let runtimeCalls = 0;
|
||||
const cdp = {
|
||||
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply();
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
if (method === "Runtime.evaluate") {
|
||||
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
|
||||
if (expression.includes("input,textarea,select")) {
|
||||
return { result: { value: { controls: [], childFrames: [] } } };
|
||||
}
|
||||
runtimeCalls += 1;
|
||||
return runtimeCalls === 1
|
||||
? {
|
||||
result: {
|
||||
value: [
|
||||
{
|
||||
triggerSel: ".menu",
|
||||
affectedSub: " .items",
|
||||
label: "Products",
|
||||
x: 50,
|
||||
y: 20,
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
: { result: { value: ["Shoes", "Bags"] } };
|
||||
}
|
||||
if (method === "Input.dispatchMouseEvent") return {};
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||
|
||||
expect(result.surfaceProbes).toEqual([
|
||||
{ triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
]);
|
||||
expect(cdp.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Input.dispatchMouseEvent",
|
||||
expect.objectContaining({ type: "mouseMoved", x: 50, y: 20 }),
|
||||
);
|
||||
expect(cdp.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Input.dispatchMouseEvent",
|
||||
expect.objectContaining({ type: "mouseMoved", x: -10, y: -10 }),
|
||||
);
|
||||
});
|
||||
|
||||
it("excludes the agent's own overlay shadow host and its inlined shadow subtree", async () => {
|
||||
// DOMSnapshot inlines an open shadow root's content as descendants of the
|
||||
// host. The agent's WXT overlay host carries a fixed full-viewport
|
||||
@@ -349,8 +449,11 @@ describe("captureViewModel", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
// overlay div bounds y=0 minus scroll pageY=200 → y=-200
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(-200);
|
||||
const div = nodes.find((n) => n.backendNodeId === 12);
|
||||
// local rect preserves viewport-relative scroll math.
|
||||
expect(div?.localRect?.y).toBe(-200);
|
||||
// exported rect is clipped to the top-level viewport before VOM consumes it.
|
||||
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
|
||||
});
|
||||
|
||||
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
|
||||
@@ -557,8 +660,187 @@ describe("captureViewModel", () => {
|
||||
expect(iframeNodes.size).toBe(1);
|
||||
expect(iframeNodes.has(13)).toBe(true);
|
||||
const subNodes = iframeNodes.get(13)!;
|
||||
expect(subNodes.find((n) => n.backendNodeId === 101)?.tag).toBe("input");
|
||||
expect(subNodes.find((n) => n.backendNodeId === 101)?.attrs.type).toBe("text");
|
||||
const input = subNodes.find((n) => n.backendNodeId === 101);
|
||||
expect(input?.tag).toBe("input");
|
||||
expect(input?.attrs.type).toBe("text");
|
||||
expect(input?.ownerFrameBackendNodeId).toBe(13);
|
||||
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
|
||||
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
|
||||
});
|
||||
|
||||
it("clips iframe sub-document rects to the iframe viewport in top coordinates", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"iframe",
|
||||
"input",
|
||||
"position",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"type",
|
||||
"text",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("iframe")],
|
||||
backendNodeId: [10, 11, 13],
|
||||
attributes: [[], [], []],
|
||||
contentDocumentIndex: { index: [2], value: [1] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[950, 760, 200, 100],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
{
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("input")],
|
||||
backendNodeId: [100, 101],
|
||||
attributes: [[], [i("type"), i("text")]],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1],
|
||||
styles: [[i("static"), i("auto")]],
|
||||
bounds: [[20, 20, 120, 60]],
|
||||
paintOrders: [0],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const { iframeNodes } = await captureViewModel(cdp, 4);
|
||||
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
|
||||
|
||||
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
|
||||
expect(input?.rect).toEqual({ x: 970, y: 780, w: 30, h: 20 });
|
||||
});
|
||||
|
||||
it("recursively normalizes nested iframe sub-documents", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"iframe",
|
||||
"input",
|
||||
"position",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"type",
|
||||
"text",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("iframe")],
|
||||
backendNodeId: [10, 11, 13],
|
||||
attributes: [[], [], []],
|
||||
contentDocumentIndex: { index: [2], value: [1] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[10, 20, 300, 200],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("iframe")],
|
||||
backendNodeId: [20, 21, 23],
|
||||
attributes: [[], [], []],
|
||||
contentDocumentIndex: { index: [2], value: [2] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 300, 200],
|
||||
[5, 6, 100, 80],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("input")],
|
||||
backendNodeId: [30, 31],
|
||||
attributes: [[], [i("type"), i("text")]],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1],
|
||||
styles: [[i("static"), i("auto")]],
|
||||
bounds: [[1, 2, 40, 20]],
|
||||
paintOrders: [0],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const { iframeNodes } = await captureViewModel(cdp, 4);
|
||||
const nestedIframe = iframeNodes.get(13)?.find((n) => n.backendNodeId === 23);
|
||||
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31);
|
||||
|
||||
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
|
||||
expect(input?.ownerFrameBackendNodeId).toBe(23);
|
||||
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
|
||||
});
|
||||
|
||||
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => {
|
||||
|
||||
@@ -16,9 +16,14 @@ const STYLE_COL = Object.fromEntries(
|
||||
export interface CapturedNode {
|
||||
backendNodeId: number;
|
||||
parentBackendNodeId: number | null;
|
||||
/** Owning iframe backend node id; `null` for the top-level document. */
|
||||
ownerFrameBackendNodeId?: number | null;
|
||||
tag: string;
|
||||
attrs: Record<string, string>;
|
||||
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
|
||||
rect: Rect | null;
|
||||
/** Frame-local viewport-relative CSS px before top-level translation/clipping. */
|
||||
localRect?: Rect | null;
|
||||
paintOrder: number;
|
||||
position: string;
|
||||
pointerEvents: string;
|
||||
@@ -31,6 +36,7 @@ export interface CapturedNode {
|
||||
*/
|
||||
cursor?: string;
|
||||
textContent?: string;
|
||||
formState?: "empty" | "filled" | "default";
|
||||
formValue?: string;
|
||||
formDefaultValue?: string;
|
||||
formPlaceholder?: string;
|
||||
@@ -53,6 +59,10 @@ export interface CapturedViewModel {
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
export interface CaptureViewModelOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
}
|
||||
|
||||
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
|
||||
interface SparseArray {
|
||||
index: number[];
|
||||
@@ -102,48 +112,112 @@ function isFormControlTag(tag: string): boolean {
|
||||
return tag === "input" || tag === "textarea" || tag === "select";
|
||||
}
|
||||
|
||||
async function enrichFormControlState(cdp: CdpRunner, tabId: number, nodes: CapturedNode[]) {
|
||||
const MAX_FORM_ENRICH_CONTROLS = 250;
|
||||
|
||||
interface CapturedFormState {
|
||||
value?: string;
|
||||
defaultValue?: string;
|
||||
placeholder?: string;
|
||||
state?: "empty" | "filled" | "default";
|
||||
sensitive?: boolean;
|
||||
}
|
||||
|
||||
interface CapturedFrameFormState {
|
||||
controls: CapturedFormState[];
|
||||
childFrames: CapturedFrameFormState[];
|
||||
}
|
||||
|
||||
function formStateBatchExpression(maxControls: number): string {
|
||||
return `(() => {
|
||||
const maxControls = ${JSON.stringify(maxControls)};
|
||||
let remaining = maxControls;
|
||||
const controlSelector = "input,textarea,select";
|
||||
const controlState = (el) => {
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
|
||||
const sensitive = type === "password" || type === "credit-card";
|
||||
const rawValue = typeof el.value === "string" ? el.value : "";
|
||||
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
|
||||
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
|
||||
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
|
||||
return {
|
||||
state,
|
||||
sensitive,
|
||||
defaultValue,
|
||||
placeholder,
|
||||
...(sensitive ? {} : { value: rawValue }),
|
||||
};
|
||||
};
|
||||
const collect = (doc) => {
|
||||
const controls = [];
|
||||
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
|
||||
if (remaining <= 0) break;
|
||||
controls.push(controlState(el));
|
||||
remaining -= 1;
|
||||
}
|
||||
const childFrames = [];
|
||||
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
|
||||
let childDoc = null;
|
||||
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
|
||||
childFrames.push(childDoc && remaining > 0 ? collect(childDoc) : { controls: [], childFrames: [] });
|
||||
}
|
||||
return { controls, childFrames };
|
||||
};
|
||||
return collect(document);
|
||||
})()`;
|
||||
}
|
||||
|
||||
function flattenFrameFormStates(
|
||||
frame: CapturedFrameFormState | undefined,
|
||||
out: CapturedFormState[][] = [],
|
||||
): CapturedFormState[][] {
|
||||
if (!frame) return out;
|
||||
out.push(Array.isArray(frame.controls) ? frame.controls : []);
|
||||
for (const child of Array.isArray(frame.childFrames) ? frame.childFrames : []) {
|
||||
flattenFrameFormStates(child, out);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function applyFormStates(nodes: CapturedNode[], states: CapturedFormState[]): void {
|
||||
let index = 0;
|
||||
for (const node of nodes) {
|
||||
if (!isFormControlTag(node.tag)) continue;
|
||||
let objectId: string | undefined;
|
||||
try {
|
||||
const resolved = await cdp.send<{ object?: { objectId?: string } }>(
|
||||
tabId,
|
||||
"DOM.resolveNode",
|
||||
{
|
||||
backendNodeId: node.backendNodeId,
|
||||
},
|
||||
);
|
||||
objectId = resolved.object?.objectId;
|
||||
if (!objectId) continue;
|
||||
const result = await cdp.send<{
|
||||
result?: { value?: { value?: string; defaultValue?: string; placeholder?: string } };
|
||||
}>(tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
returnByValue: true,
|
||||
functionDeclaration: `function(){
|
||||
return {
|
||||
value: typeof this.value === "string" ? this.value : "",
|
||||
defaultValue: typeof this.defaultValue === "string" ? this.defaultValue : "",
|
||||
placeholder: typeof this.placeholder === "string" ? this.placeholder : ""
|
||||
};
|
||||
}`,
|
||||
});
|
||||
const value = result.result?.value;
|
||||
if (!value) continue;
|
||||
node.formValue = value.value ?? "";
|
||||
node.formDefaultValue = value.defaultValue ?? "";
|
||||
node.formPlaceholder = value.placeholder ?? "";
|
||||
} catch {
|
||||
// Best-effort enrichment. DOMSnapshot/AX data still carries the node.
|
||||
} finally {
|
||||
if (objectId) {
|
||||
await cdp.send(tabId, "Runtime.releaseObject", { objectId }).catch(() => {});
|
||||
}
|
||||
const state = states[index];
|
||||
index += 1;
|
||||
if (!state) continue;
|
||||
node.formState = state.state;
|
||||
node.formDefaultValue = state.defaultValue ?? "";
|
||||
node.formPlaceholder = state.placeholder ?? "";
|
||||
if (!state.sensitive && state.value !== undefined) {
|
||||
node.formValue = state.value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function enrichFormControlStates(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
frameNodeGroups: CapturedNode[][],
|
||||
): Promise<void> {
|
||||
const hasControls = frameNodeGroups.some((nodes) =>
|
||||
nodes.some((node) => isFormControlTag(node.tag)),
|
||||
);
|
||||
if (!hasControls) return;
|
||||
try {
|
||||
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
|
||||
returnByValue: true,
|
||||
});
|
||||
const frameStates = flattenFrameFormStates(runtimeValue<CapturedFrameFormState>(result));
|
||||
for (let i = 0; i < frameNodeGroups.length; i += 1) {
|
||||
applyFormStates(frameNodeGroups[i], frameStates[i] ?? []);
|
||||
}
|
||||
} catch {
|
||||
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
|
||||
}
|
||||
}
|
||||
|
||||
interface RuntimeEvaluateReply {
|
||||
result?: {
|
||||
value?: unknown;
|
||||
@@ -169,6 +243,14 @@ interface ParseDocumentResult {
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
interface FrameContext {
|
||||
ownerFrameBackendNodeId: number | null;
|
||||
originInTop: { x: number; y: number };
|
||||
clipRectInTop: Rect;
|
||||
scrollX: number;
|
||||
scrollY: number;
|
||||
}
|
||||
|
||||
function str(strings: string[], idx: number | undefined): string {
|
||||
if (idx === undefined || idx < 0) return "";
|
||||
return strings[idx] ?? "";
|
||||
@@ -183,6 +265,49 @@ function devicePixelRatio(metrics: LayoutMetricsReply): number {
|
||||
return dpr >= 1 ? dpr : 1;
|
||||
}
|
||||
|
||||
function viewportRect(viewport: Viewport): Rect {
|
||||
return { x: 0, y: 0, w: Math.max(0, viewport.width), h: Math.max(0, viewport.height) };
|
||||
}
|
||||
|
||||
function intersectRects(a: Rect, b: Rect): Rect | null {
|
||||
const x1 = Math.max(a.x, b.x);
|
||||
const y1 = Math.max(a.y, b.y);
|
||||
const x2 = Math.min(a.x + a.w, b.x + b.w);
|
||||
const y2 = Math.min(a.y + a.h, b.y + b.h);
|
||||
if (x2 <= x1 || y2 <= y1) return null;
|
||||
return { x: x1, y: y1, w: x2 - x1, h: y2 - y1 };
|
||||
}
|
||||
|
||||
function rectInTop(localRect: Rect, context: FrameContext): Rect | null {
|
||||
const translated = {
|
||||
x: context.originInTop.x + localRect.x,
|
||||
y: context.originInTop.y + localRect.y,
|
||||
w: localRect.w,
|
||||
h: localRect.h,
|
||||
};
|
||||
return intersectRects(translated, context.clipRectInTop);
|
||||
}
|
||||
|
||||
function unclipRectInTop(localRect: Rect, context: FrameContext): Rect {
|
||||
return {
|
||||
x: context.originInTop.x + localRect.x,
|
||||
y: context.originInTop.y + localRect.y,
|
||||
w: localRect.w,
|
||||
h: localRect.h,
|
||||
};
|
||||
}
|
||||
|
||||
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
|
||||
const out = new Map<number, number>();
|
||||
if (!sparse?.index || !sparse.value) return out;
|
||||
for (let i = 0; i < sparse.index.length; i++) {
|
||||
const nodeIndex = sparse.index[i];
|
||||
const docIndex = sparse.value[i];
|
||||
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<number>): void {
|
||||
if (!node) return;
|
||||
if (typeof node.backendNodeId === "number") {
|
||||
@@ -235,14 +360,24 @@ function hoverCssScanExpression(): string {
|
||||
|
||||
const candidates = [];
|
||||
const seenLabels = new Set();
|
||||
const isUnsafeTrigger = (el) => {
|
||||
if (!(el instanceof HTMLElement)) return true;
|
||||
const tag = el.tagName.toLowerCase();
|
||||
if (["input", "textarea", "select", "option"].includes(tag)) return true;
|
||||
if (el.isContentEditable) return true;
|
||||
if (el.hasAttribute("disabled") || el.hasAttribute("inert")) return true;
|
||||
if ((el.getAttribute("aria-disabled") || "").toLowerCase() === "true") return true;
|
||||
return false;
|
||||
};
|
||||
for (const pair of pairs) {
|
||||
let elements;
|
||||
try { elements = Array.from(document.querySelectorAll(pair.triggerSel)); } catch { continue; }
|
||||
for (const el of elements) {
|
||||
if (!(el instanceof Element)) continue;
|
||||
if (isUnsafeTrigger(el)) continue;
|
||||
const rect = el.getBoundingClientRect();
|
||||
const style = getComputedStyle(el);
|
||||
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none") continue;
|
||||
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue;
|
||||
const label = (el.textContent || "").replace(/\\s+/g, " ").trim();
|
||||
if (!label || seenLabels.has(label)) continue;
|
||||
seenLabels.add(label);
|
||||
@@ -304,6 +439,20 @@ async function wait(ms: number): Promise<void> {
|
||||
await new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
|
||||
await cdp
|
||||
.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: -10,
|
||||
y: -10,
|
||||
})
|
||||
.catch(() =>
|
||||
cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 }).catch(() => {
|
||||
// best effort
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<CapturedSurfaceProbe[]> {
|
||||
const started = Date.now();
|
||||
try {
|
||||
@@ -328,7 +477,6 @@ async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<Captur
|
||||
expression: hoverCollectExpression(candidate),
|
||||
returnByValue: true,
|
||||
});
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 });
|
||||
const subItems = (runtimeValue<string[]>(collected) ?? [])
|
||||
.map((item) => item.replace(/\s+/g, " ").trim())
|
||||
.filter(Boolean);
|
||||
@@ -341,6 +489,8 @@ async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<Captur
|
||||
});
|
||||
} catch {
|
||||
continue;
|
||||
} finally {
|
||||
await clearHover(cdp, tabId);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
@@ -391,15 +541,13 @@ export async function collectOverlayExcludedBackendIds(
|
||||
* @param doc - the raw DOMSnapshot document object
|
||||
* @param strings - the shared string table for the whole snapshot
|
||||
* @param dpr - device-pixel-ratio from Page.getLayoutMetrics
|
||||
* @param scrollX - horizontal scroll offset to subtract (CSS px)
|
||||
* @param scrollY - vertical scroll offset to subtract (CSS px)
|
||||
* @param context - frame coordinate context; output rects are top viewport-relative
|
||||
*/
|
||||
function parseDocumentNodes(
|
||||
doc: SnapshotDocument,
|
||||
strings: string[],
|
||||
dpr: number,
|
||||
scrollX: number,
|
||||
scrollY: number,
|
||||
context: FrameContext,
|
||||
): ParseDocumentResult {
|
||||
const dn = doc.nodes;
|
||||
const dl = doc.layout;
|
||||
@@ -477,6 +625,7 @@ function parseDocumentNodes(
|
||||
}
|
||||
|
||||
let rect: Rect | null = null;
|
||||
let localRect: Rect | null = null;
|
||||
let paintOrder = 0;
|
||||
let position = "static";
|
||||
let pointerEvents = "auto";
|
||||
@@ -485,12 +634,13 @@ function parseDocumentNodes(
|
||||
if (li !== undefined) {
|
||||
const b = dl?.bounds?.[li];
|
||||
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
|
||||
rect = {
|
||||
x: b[0] / dpr - scrollX,
|
||||
y: b[1] / dpr - scrollY,
|
||||
localRect = {
|
||||
x: b[0] / dpr - context.scrollX,
|
||||
y: b[1] / dpr - context.scrollY,
|
||||
w: b[2] / dpr,
|
||||
h: b[3] / dpr,
|
||||
};
|
||||
rect = rectInTop(localRect, context);
|
||||
}
|
||||
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
||||
const styleRow = dl?.styles?.[li] ?? [];
|
||||
@@ -508,9 +658,11 @@ function parseDocumentNodes(
|
||||
nodes.push({
|
||||
backendNodeId,
|
||||
parentBackendNodeId,
|
||||
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
|
||||
tag,
|
||||
attrs,
|
||||
rect,
|
||||
localRect,
|
||||
paintOrder,
|
||||
position,
|
||||
pointerEvents,
|
||||
@@ -521,7 +673,81 @@ function parseDocumentNodes(
|
||||
return { nodes, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
export async function captureViewModel(cdp: CdpRunner, tabId: number): Promise<CapturedViewModel> {
|
||||
interface ParseFrameDocumentsResult {
|
||||
iframeNodes: CapturedIframeNodes;
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
function parseChildFrameDocuments(
|
||||
documents: SnapshotDocument[],
|
||||
strings: string[],
|
||||
dpr: number,
|
||||
parentDocIndex: number,
|
||||
parentNodes: CapturedNode[],
|
||||
parentContext: FrameContext,
|
||||
visited = new Set<number>(),
|
||||
): ParseFrameDocumentsResult {
|
||||
const iframeNodes: CapturedIframeNodes = new Map();
|
||||
const excludedBackendNodeIds = new Set<number>();
|
||||
const parentDoc = documents[parentDocIndex];
|
||||
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
|
||||
if (cdi.size === 0) return { iframeNodes, excludedBackendNodeIds };
|
||||
|
||||
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
|
||||
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
|
||||
|
||||
for (const [nodeArrayIdx, childDocIndex] of cdi) {
|
||||
if (visited.has(childDocIndex)) continue;
|
||||
const childDoc = documents[childDocIndex];
|
||||
if (!childDoc) continue;
|
||||
const iframeBackendId = parentBackendIds[nodeArrayIdx];
|
||||
if (iframeBackendId === undefined) continue;
|
||||
const iframeNode = parentNodeByBackendId.get(iframeBackendId);
|
||||
if (!iframeNode?.localRect) continue;
|
||||
|
||||
const iframeRectInTop = unclipRectInTop(iframeNode.localRect, parentContext);
|
||||
const childClip = intersectRects(iframeRectInTop, parentContext.clipRectInTop);
|
||||
if (!childClip) {
|
||||
iframeNodes.set(iframeBackendId, []);
|
||||
continue;
|
||||
}
|
||||
|
||||
const childContext: FrameContext = {
|
||||
ownerFrameBackendNodeId: iframeBackendId,
|
||||
originInTop: { x: iframeRectInTop.x, y: iframeRectInTop.y },
|
||||
clipRectInTop: childClip,
|
||||
scrollX: childDoc.scrollOffsetX ?? 0,
|
||||
scrollY: childDoc.scrollOffsetY ?? 0,
|
||||
};
|
||||
const nextVisited = new Set(visited);
|
||||
nextVisited.add(childDocIndex);
|
||||
const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext);
|
||||
iframeNodes.set(iframeBackendId, parsed.nodes);
|
||||
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
|
||||
const nested = parseChildFrameDocuments(
|
||||
documents,
|
||||
strings,
|
||||
dpr,
|
||||
childDocIndex,
|
||||
parsed.nodes,
|
||||
childContext,
|
||||
nextVisited,
|
||||
);
|
||||
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
|
||||
iframeNodes.set(nestedFrameId, nestedNodes);
|
||||
}
|
||||
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
}
|
||||
|
||||
return { iframeNodes, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
export async function captureViewModel(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
options: CaptureViewModelOptions = {},
|
||||
): Promise<CapturedViewModel> {
|
||||
const metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
const dpr = devicePixelRatio(metrics);
|
||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
@@ -552,42 +778,26 @@ export async function captureViewModel(cdp: CdpRunner, tabId: number): Promise<C
|
||||
};
|
||||
}
|
||||
|
||||
const mainParsed = parseDocumentNodes(doc0, strings, dpr, scrollX, scrollY);
|
||||
const topContext: FrameContext = {
|
||||
ownerFrameBackendNodeId: null,
|
||||
originInTop: { x: 0, y: 0 },
|
||||
clipRectInTop: viewportRect(viewport),
|
||||
scrollX,
|
||||
scrollY,
|
||||
};
|
||||
const mainParsed = parseDocumentNodes(doc0, strings, dpr, topContext);
|
||||
const nodes = mainParsed.nodes;
|
||||
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
|
||||
|
||||
// Build a map from node-array-index → sub-document-index using the sparse
|
||||
// contentDocumentIndex field. Chrome emits this only for <iframe> nodes.
|
||||
const iframeNodes: CapturedIframeNodes = new Map();
|
||||
if (documents.length > 1) {
|
||||
const cdi = doc0.nodes?.contentDocumentIndex;
|
||||
if (cdi?.index && cdi.value) {
|
||||
for (let k = 0; k < cdi.index.length; k++) {
|
||||
const nodeArrayIdx = cdi.index[k];
|
||||
const docIdx = cdi.value[k];
|
||||
const subDoc = documents[docIdx];
|
||||
if (!subDoc) continue;
|
||||
const iframeBid = doc0.nodes?.backendNodeId?.[nodeArrayIdx];
|
||||
if (iframeBid === undefined) continue;
|
||||
// Sub-documents have their own scroll offsets; bounds are relative
|
||||
// to the iframe's own coordinate space (not the main viewport).
|
||||
const subScrollX = subDoc.scrollOffsetX ?? 0;
|
||||
const subScrollY = subDoc.scrollOffsetY ?? 0;
|
||||
const subParsed = parseDocumentNodes(subDoc, strings, dpr, subScrollX, subScrollY);
|
||||
iframeNodes.set(iframeBid, subParsed.nodes);
|
||||
for (const id of subParsed.excludedBackendNodeIds) {
|
||||
excludedBackendNodeIds.add(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext);
|
||||
const iframeNodes = frameParsed.iframeNodes;
|
||||
for (const id of frameParsed.excludedBackendNodeIds) {
|
||||
excludedBackendNodeIds.add(id);
|
||||
}
|
||||
|
||||
await enrichFormControlState(cdp, tabId, nodes);
|
||||
for (const subNodes of iframeNodes.values()) {
|
||||
await enrichFormControlState(cdp, tabId, subNodes);
|
||||
}
|
||||
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()]);
|
||||
|
||||
const surfaceProbes = await probeHoverSurfaces(cdp, tabId);
|
||||
const surfaceProbes = options.conditionalSurfaceProbe ? await probeHoverSurfaces(cdp, tabId) : [];
|
||||
|
||||
return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ export type RpcErrorReason =
|
||||
| "option_not_found"
|
||||
| "single_select_value_count"
|
||||
| "tab_not_active"
|
||||
| "restricted_tab_url"
|
||||
| "borrow_conflict";
|
||||
|
||||
export interface RpcErrorData {
|
||||
@@ -313,6 +314,9 @@ export interface SnapshotResult {
|
||||
dialogs?: JavaScriptDialogInfo[];
|
||||
}
|
||||
|
||||
export type ObserveParams = SnapshotParams;
|
||||
export type ObserveResult = SnapshotResult;
|
||||
|
||||
export interface GetHtmlParams {
|
||||
session_id: string;
|
||||
tab_id?: number;
|
||||
|
||||
@@ -68,6 +68,7 @@ Write operations only affect tabs in the **Agent Window** (or tabs you **borrowe
|
||||
```
|
||||
bsk navigate <url> --session <id>
|
||||
bsk snapshot --session <id> → aria tree with @e1, @e2, … refs
|
||||
bsk observe --session <id> → semantic VOM view; may reveal hover/focus surfaces
|
||||
bsk click @e3 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk snapshot --session <id> → again after navigation / DOM change
|
||||
```
|
||||
@@ -78,11 +79,12 @@ Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` /
|
||||
|
||||
## Observation priority
|
||||
|
||||
Start with `bsk snapshot` to understand page structure, text, controls, and element refs. Only escalate when the latest snapshot cannot answer the question:
|
||||
Start with `bsk snapshot` to understand page structure, text, controls, and element refs. Use `bsk observe` when semantic VOM output or conditional hover/focus surfaces would materially help. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
|
||||
1. `bsk snapshot` — default for page understanding and interaction planning
|
||||
2. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
||||
3. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the snapshot. Use `--ref @eN` (from the latest snapshot) to crop to one element; omit `--ref` for the full visible tab.
|
||||
1. `bsk snapshot` — strict static page understanding and interaction planning
|
||||
2. `bsk observe` — semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
||||
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
||||
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
||||
|
||||
Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page.
|
||||
|
||||
@@ -141,7 +143,8 @@ Details and flags: **`bsk <cmd> --help`**
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk snapshot` | First-choice page understanding: accessibility tree with `@eN` element refs |
|
||||
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
|
||||
| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
|
||||
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
|
||||
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
|
||||
|
||||
@@ -260,7 +263,7 @@ Always **`bsk session stop <id>`** in a `finally`-style path so the Agent Window
|
||||
2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
|
||||
3. **No skip stop** — always `bsk session stop <id>`; never assume idle timeout will clean up.
|
||||
4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
|
||||
5. **No observe escalation before snapshot** — use `bsk snapshot` first; only use `bsk get-html` or `bsk screenshot` when the snapshot is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot ref — never skip snapshot just to grab a visual.
|
||||
5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics or conditional surfaces help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual.
|
||||
6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
|
||||
|
||||
---
|
||||
|
||||
@@ -19,6 +19,7 @@ pub mod interaction;
|
||||
pub mod logs;
|
||||
pub mod navigate;
|
||||
pub mod network;
|
||||
pub mod observe;
|
||||
pub mod record;
|
||||
pub mod record_state;
|
||||
pub mod render_error;
|
||||
@@ -41,6 +42,7 @@ use crate::cli::install_skill::InstallSkillArgs;
|
||||
use crate::cli::interaction::{ClickArgs, FillArgs, PressArgs, SelectArgs};
|
||||
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
|
||||
use crate::cli::network::NetworkArgs;
|
||||
use crate::cli::observe::ObserveArgs;
|
||||
use crate::cli::record::RecordCmd;
|
||||
use crate::cli::screenshot::ScreenshotArgs;
|
||||
use crate::cli::session::SessionCmd;
|
||||
@@ -122,6 +124,9 @@ pub enum Command {
|
||||
/// Produce an indented aria-snapshot with @eN refs.
|
||||
Snapshot(SnapshotArgs),
|
||||
|
||||
/// Produce a semantic VOM observation with perception probes.
|
||||
Observe(ObserveArgs),
|
||||
|
||||
/// Read buffered console/log/exception messages.
|
||||
Console(ConsoleArgs),
|
||||
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
//! `bsk observe` — semantic VOM observation with bounded perception probes.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use anyhow::Context;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{ObserveParams, ObserveResult};
|
||||
use clap::Args;
|
||||
|
||||
use crate::cli::TOOL_IPC_TIMEOUT;
|
||||
use crate::cli::dialogs::print_dialog_summaries;
|
||||
use crate::cli::ensure_daemon::ensure_daemon;
|
||||
use crate::cli::error::{CliError, Format};
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct ObserveArgs {
|
||||
/// Session id (must be active).
|
||||
#[arg(long)]
|
||||
pub session: String,
|
||||
|
||||
/// Target tab. Defaults to the Agent Window's active tab.
|
||||
#[arg(long = "tab-id")]
|
||||
pub tab_id: Option<i64>,
|
||||
|
||||
/// Cap on VOM tree depth before truncating.
|
||||
#[arg(long = "max-depth")]
|
||||
pub max_depth: Option<u32>,
|
||||
|
||||
/// Soft cap on rendered tokens (~4 chars/token).
|
||||
#[arg(long = "max-tokens")]
|
||||
pub max_tokens: Option<u32>,
|
||||
}
|
||||
|
||||
pub fn dispatch(args: ObserveArgs, format: Format) -> Result<(), CliError> {
|
||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||
run(info.sock_path, args, format)
|
||||
}
|
||||
|
||||
fn run(sock: PathBuf, args: ObserveArgs, format: Format) -> Result<(), CliError> {
|
||||
let params = ObserveParams {
|
||||
session_id: args.session.clone(),
|
||||
tab_id: args.tab_id,
|
||||
max_depth: args.max_depth,
|
||||
max_tokens: args.max_tokens,
|
||||
};
|
||||
let reply: ObserveResult = call(sock, params)?;
|
||||
match format {
|
||||
Format::Json => {
|
||||
let json = serde_json::to_string_pretty(&reply)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?;
|
||||
println!("{json}");
|
||||
}
|
||||
Format::Human => {
|
||||
if reply.text.is_empty() {
|
||||
println!("(empty observation — page may still be loading)");
|
||||
} else {
|
||||
println!("{}", reply.text);
|
||||
}
|
||||
if reply.truncated {
|
||||
eprintln!(
|
||||
"warning: observation truncated (refs={}, tab={}). Increase --max-depth / --max-tokens if needed.",
|
||||
reply.ref_count, reply.tab_id
|
||||
);
|
||||
}
|
||||
print_dialog_summaries(&reply.dialogs);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn call(sock: PathBuf, params: ObserveParams) -> Result<ObserveResult, CliError> {
|
||||
crate::cli::business_rpc::call::<ObserveParams, ObserveResult>(
|
||||
sock,
|
||||
"observe",
|
||||
Method::ToolObserve,
|
||||
Some(params),
|
||||
TOOL_IPC_TIMEOUT,
|
||||
)
|
||||
}
|
||||
@@ -231,6 +231,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
||||
| Method::ToolConsole
|
||||
| Method::ToolNetwork
|
||||
| Method::ToolSnapshot
|
||||
| Method::ToolObserve
|
||||
| Method::ToolGetHtml
|
||||
| Method::ToolNavigate
|
||||
| Method::ToolNavigateBack
|
||||
@@ -291,16 +292,14 @@ async fn handle_tool_dispatch(
|
||||
};
|
||||
// Pre-flight: if the user has clicked the agent-window mask's
|
||||
// stop button, every session carries a one-shot "pending
|
||||
// interrupt" marker. The marker is consumed by the next
|
||||
// *mutating* tool call (which is rejected with `UserAborted`).
|
||||
// Read-only tools (snapshot / get_html / waits / tab_list) and
|
||||
// session-lifecycle RPCs pass through transparently — gating
|
||||
// them would prevent the agent from observing page state
|
||||
// before asking the user, or from cleanly tearing down the
|
||||
// session. Classification lives on `Method::is_mutating()` so
|
||||
// adding a new tool variant requires an explicit
|
||||
// classification call.
|
||||
if method.is_mutating() && state.session_interrupts.try_consume(&session_id) {
|
||||
// interrupt" marker. The marker is consumed by the next method
|
||||
// that dispatches browser/page input (which is rejected with
|
||||
// `UserAborted`). Passive reads and control-plane RPCs pass through
|
||||
// transparently — gating them would prevent the agent from observing
|
||||
// page state before asking the user, or from cleanly tearing down the
|
||||
// session. Classification lives on `Method::effect()` so adding a new
|
||||
// tool variant requires an explicit classification call.
|
||||
if method.requires_interrupt_gate() && state.session_interrupts.try_consume(&session_id) {
|
||||
return ResponseBody::Err(RpcError {
|
||||
code: ErrorCode::UserAborted,
|
||||
message: "tool dispatch rejected: pending user interrupt. The user explicitly requested to stop. Ask the user how to proceed before issuing further actions.".into(),
|
||||
|
||||
@@ -2,13 +2,13 @@
|
||||
//!
|
||||
//! Holds a single-use marker per `SessionId` indicating that the
|
||||
//! user has clicked the agent-window mask's stop button. The next
|
||||
//! mutating `tool.*` call for that session is rejected with
|
||||
//! `ErrorCode::UserAborted`; read-only tools and session-lifecycle
|
||||
//! browser-input-dispatching `tool.*` call for that session is rejected
|
||||
//! with `ErrorCode::UserAborted`; passive reads and session-lifecycle
|
||||
//! RPCs pass through transparently and do not consume the marker.
|
||||
//!
|
||||
//! The marker is single-use and has no expiry: it sits in the
|
||||
//! registry until consumed by a mutating call or until the session
|
||||
//! is torn down. This lets the user's interrupt survive an LLM
|
||||
//! registry until consumed by an input-dispatching call or until the
|
||||
//! session is torn down. This lets the user's interrupt survive an LLM
|
||||
//! thinking phase of arbitrary length — the v1 time-window
|
||||
//! mechanism dropped interrupts whenever the LLM took longer to
|
||||
//! respond than the window allowed.
|
||||
|
||||
@@ -46,8 +46,8 @@ pub struct DaemonState {
|
||||
/// Per-session "pending interrupt" signal. The WS event handler
|
||||
/// `mark`s the session when the user clicks the agent-window
|
||||
/// mask's stop button; the IPC tool-dispatch handler
|
||||
/// `try_consume`s on the way in so the next mutating tool call
|
||||
/// is rejected with `UserAborted`. Independent of
|
||||
/// `try_consume`s on the way in so the next browser-input-dispatching
|
||||
/// tool call is rejected with `UserAborted`. Independent of
|
||||
/// `SessionRegistry` because the signal is a transient runtime
|
||||
/// control state.
|
||||
pub session_interrupts: Arc<SessionInterruptRegistry>,
|
||||
|
||||
@@ -82,6 +82,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
|
||||
Command::Tab(cmd) => cli::tab::dispatch(cmd, format),
|
||||
Command::Screenshot(args) => cli::screenshot::dispatch(args, format),
|
||||
Command::Snapshot(args) => cli::snapshot::dispatch(args, format),
|
||||
Command::Observe(args) => cli::observe::dispatch(args, format),
|
||||
Command::Console(args) => cli::console::dispatch(args, format),
|
||||
Command::Network(args) => cli::network::dispatch(args, format),
|
||||
Command::GetHtml(args) => cli::get_html::dispatch(args, format),
|
||||
|
||||
@@ -283,11 +283,11 @@ async fn session_user_interrupt_event_cancels_inflight_with_user_aborted() {
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
/// The pending-interrupt marker must reject a *new* mutating tool
|
||||
/// The pending-interrupt marker must reject a *new* browser-input tool
|
||||
/// call that arrives AFTER the user clicked stop, even when no
|
||||
/// tool was inflight at click time. Under v2 single-use marker
|
||||
/// semantics, the marker survives indefinitely until consumed by
|
||||
/// a mutating tool call — there is no time window. This is the
|
||||
/// an input-dispatching tool call — there is no time window. This is the
|
||||
/// core motivating scenario: agents spend most of their time
|
||||
/// between tool calls (LLM thinking), and the stop button must
|
||||
/// catch a tool dispatched arbitrarily later.
|
||||
@@ -426,12 +426,12 @@ async fn read_only_tool_passes_through_without_consuming_interrupt_marker() {
|
||||
let ws_sink = Arc::new(tokio::sync::Mutex::new(ws_sink));
|
||||
|
||||
// Fake extension: replies to session_start, console, and snapshot
|
||||
// (read-only tools must succeed transparently). Records click count
|
||||
// — must remain 0 because the daemon should reject the click
|
||||
// (passive reads must succeed transparently). Records observe count
|
||||
// — must remain 0 because the daemon should reject transient input
|
||||
// without forwarding.
|
||||
let ws_sink_for_responder = Arc::clone(&ws_sink);
|
||||
let observed_click_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let observed_click_count_clone = Arc::clone(&observed_click_count);
|
||||
let observed_observe_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let observed_observe_count_clone = Arc::clone(&observed_observe_count);
|
||||
let responder = tokio::spawn(async move {
|
||||
let mut ws_stream = ws_stream;
|
||||
while let Some(Ok(msg)) = ws_stream.next().await {
|
||||
@@ -458,8 +458,8 @@ async fn read_only_tool_passes_through_without_consuming_interrupt_marker() {
|
||||
"truncated": false
|
||||
})),
|
||||
Method::ToolSnapshot => ResponseBody::Ok(json!({"ok": true})),
|
||||
Method::ToolClick => {
|
||||
observed_click_count_clone
|
||||
Method::ToolObserve => {
|
||||
observed_observe_count_clone
|
||||
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
continue;
|
||||
}
|
||||
@@ -547,26 +547,25 @@ async fn read_only_tool_passes_through_without_consuming_interrupt_marker() {
|
||||
snapshot_outcome
|
||||
);
|
||||
|
||||
// The mutating click that follows MUST still be rejected — the
|
||||
// marker survived the snapshot.
|
||||
let click_outcome = ipc
|
||||
// The transient-input observe that follows MUST still be rejected — the
|
||||
// marker survived the passive reads.
|
||||
let observe_outcome = ipc
|
||||
.call::<_, serde_json::Value>(
|
||||
"click-after-snapshot",
|
||||
Method::ToolClick,
|
||||
"observe-after-snapshot",
|
||||
Method::ToolObserve,
|
||||
Some(json!({
|
||||
"session_id": start.session_id,
|
||||
"ref": "fake-ref-2",
|
||||
})),
|
||||
Duration::from_secs(3),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let err = click_outcome.expect_err("tool.click must be rejected");
|
||||
let err = observe_outcome.expect_err("tool.observe must be rejected");
|
||||
assert_eq!(err.code, ErrorCode::UserAborted, "got {:?}", err);
|
||||
assert_eq!(
|
||||
observed_click_count.load(std::sync::atomic::Ordering::SeqCst),
|
||||
observed_observe_count.load(std::sync::atomic::Ordering::SeqCst),
|
||||
0,
|
||||
"tool.click must NOT have been forwarded after snapshot transparency"
|
||||
"tool.observe must NOT have been forwarded after passive-read transparency"
|
||||
);
|
||||
|
||||
responder.abort();
|
||||
@@ -574,7 +573,7 @@ async fn read_only_tool_passes_through_without_consuming_interrupt_marker() {
|
||||
}
|
||||
|
||||
/// The pending-interrupt marker must survive an arbitrary delay
|
||||
/// between user click and next mutating tool call. This is the
|
||||
/// between user click and next browser-input tool call. This is the
|
||||
/// regression test for v2's core motivating bug: v1's 500ms time
|
||||
/// window dropped interrupts whenever the LLM's thinking phase
|
||||
/// took longer to respond than the window allowed.
|
||||
|
||||
@@ -11,9 +11,9 @@ use bsk::ipc_client::IpcClient;
|
||||
use bsk_protocol::system::{HandshakeParams, HandshakeResult};
|
||||
use bsk_protocol::tools::{
|
||||
ConsoleEntry, ConsoleEntryKind, ConsoleParams, ConsoleResult, GetHtmlParams, GetHtmlResult,
|
||||
NetworkEntry, NetworkEntryKind, NetworkParams, NetworkResult, ScreenshotParams,
|
||||
ScreenshotResult, SessionStartParams, SessionStartResult, SnapshotParams, SnapshotResult,
|
||||
TabInfo, TabListParams, TabListResult, TabScope,
|
||||
NetworkEntry, NetworkEntryKind, NetworkParams, NetworkResult, ObserveParams, ObserveResult,
|
||||
ScreenshotParams, ScreenshotResult, SessionStartParams, SessionStartResult, SnapshotParams,
|
||||
SnapshotResult, TabInfo, TabListParams, TabListResult, TabScope,
|
||||
};
|
||||
use bsk_protocol::{
|
||||
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
|
||||
@@ -462,6 +462,46 @@ async fn snapshot_returns_text_and_ref_count() {
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn observe_returns_semantic_text_and_ref_count() {
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
let mut ws = connect_ext(handle.ws_addr()).await;
|
||||
let _ = do_handshake(&mut ws).await;
|
||||
|
||||
run_extension(ws, |req| {
|
||||
assert_eq!(req.method, Method::ToolObserve);
|
||||
let _: ObserveParams = serde_json::from_value(req.params.clone().unwrap()).unwrap();
|
||||
ResponseBody::Ok(
|
||||
serde_json::to_value(ObserveResult {
|
||||
text: "@vom 1\n @e1 button \"Products\" [hover: Shoes]\n".into(),
|
||||
ref_count: 1,
|
||||
tab_id: 13,
|
||||
truncated: false,
|
||||
dialogs: vec![],
|
||||
})
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let session_id = ipc_session_start(&sock).await;
|
||||
let result: ObserveResult = ipc_tool_call(
|
||||
&sock,
|
||||
Method::ToolObserve,
|
||||
ObserveParams {
|
||||
session_id,
|
||||
tab_id: None,
|
||||
max_depth: None,
|
||||
max_tokens: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("observe ok");
|
||||
assert_eq!(result.ref_count, 1);
|
||||
assert!(result.text.contains("[hover: Shoes]"));
|
||||
assert_eq!(result.tab_id, 13);
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn get_html_round_trips_with_ref_alias() {
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "ObserveParams",
|
||||
"description": "Parameters for `tool.observe` — produce a semantic VOM observation for one tab plus a fresh `@e<N>` ref-store on the extension side.\n\nUnlike `tool.snapshot`, this path may run bounded perception probes such as hover-surface discovery. It must not submit input, click, navigate, or otherwise commit page state.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"session_id"
|
||||
],
|
||||
"properties": {
|
||||
"max_depth": {
|
||||
"description": "Cap the depth of the rendered VOM tree.",
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"max_tokens": {
|
||||
"description": "Soft cap on rendered tokens (approximate; extension uses a best-effort heuristic based on character count).",
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"session_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"tab_id": {
|
||||
"description": "Optional target tab. Defaults to the Agent Window's currently active tab.",
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "int64"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "ObserveResult",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"ref_count",
|
||||
"tab_id",
|
||||
"text"
|
||||
],
|
||||
"properties": {
|
||||
"dialogs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/JavaScriptDialogInfo"
|
||||
}
|
||||
},
|
||||
"ref_count": {
|
||||
"description": "Number of `@e<N>` refs registered for this session by this observation. Equivalent to the size of the new ref-store map.",
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"tab_id": {
|
||||
"description": "Tab the observation was computed for.",
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
},
|
||||
"text": {
|
||||
"description": "Semantic VOM observation text. Refs are rendered as `@e<N>` so the agent can copy them into subsequent interaction tools.",
|
||||
"type": "string"
|
||||
},
|
||||
"truncated": {
|
||||
"description": "Whether the rendered tree was truncated because of `max_depth` / `max_tokens`. Agents may re-run with looser caps.",
|
||||
"default": false,
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"JavaScriptDialogHandledAction": {
|
||||
"description": "How the extension resolved the dialog so CDP could continue.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"accepted",
|
||||
"dismissed"
|
||||
]
|
||||
},
|
||||
"JavaScriptDialogInfo": {
|
||||
"description": "One observed + handled JavaScript dialog during a tool call.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"handled",
|
||||
"message",
|
||||
"sequence",
|
||||
"tab_id",
|
||||
"type"
|
||||
],
|
||||
"properties": {
|
||||
"default_prompt": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"handled": {
|
||||
"$ref": "#/definitions/JavaScriptDialogHandledAction"
|
||||
},
|
||||
"has_browser_handler": {
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"message": {
|
||||
"type": "string"
|
||||
},
|
||||
"sequence": {
|
||||
"description": "Monotonic per-tab sequence for ordering within a session.",
|
||||
"type": "integer",
|
||||
"format": "uint64",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"tab_id": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
},
|
||||
"type": {
|
||||
"$ref": "#/definitions/JavaScriptDialogType"
|
||||
},
|
||||
"url": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"JavaScriptDialogType": {
|
||||
"description": "Native JS dialog kind reported by CDP.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"alert",
|
||||
"confirm",
|
||||
"prompt",
|
||||
"beforeunload"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -78,6 +78,8 @@ fn main() {
|
||||
|
||||
dump!(SnapshotParams, "tool_snapshot_params");
|
||||
dump!(SnapshotResult, "tool_snapshot_result");
|
||||
dump!(ObserveParams, "tool_observe_params");
|
||||
dump!(ObserveResult, "tool_observe_result");
|
||||
dump!(GetHtmlParams, "tool_get_html_params");
|
||||
dump!(GetHtmlResult, "tool_get_html_result");
|
||||
dump!(ScreenshotParams, "tool_screenshot_params");
|
||||
|
||||
@@ -2,6 +2,22 @@
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Observable browser-side effect class for a protocol method.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum MethodEffect {
|
||||
/// Reads browser/session state without dispatching page input.
|
||||
PassiveRead,
|
||||
/// Dispatches temporary input such as hover probes. It must not submit,
|
||||
/// navigate, or persist page state, but it can trigger page event handlers.
|
||||
TransientInput,
|
||||
/// Drives browser/page state such as clicking, filling, navigation, tabs,
|
||||
/// or arbitrary page script.
|
||||
BrowserMutation,
|
||||
/// Control-plane/session lifecycle operation. These are deliberately not
|
||||
/// gated by the pending browser-action interrupt path.
|
||||
ControlPlane,
|
||||
}
|
||||
|
||||
/// Namespaced method string (`system.handshake`, `tool.tab_list`, …).
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum Method {
|
||||
@@ -58,6 +74,8 @@ pub enum Method {
|
||||
ToolSelect,
|
||||
#[serde(rename = "tool.snapshot")]
|
||||
ToolSnapshot,
|
||||
#[serde(rename = "tool.observe")]
|
||||
ToolObserve,
|
||||
#[serde(rename = "tool.get_html")]
|
||||
ToolGetHtml,
|
||||
#[serde(rename = "tool.screenshot")]
|
||||
@@ -86,13 +104,13 @@ pub enum Method {
|
||||
}
|
||||
|
||||
impl Method {
|
||||
/// Whether this RPC may modify browser state.
|
||||
/// Browser-side effect classification for this method.
|
||||
///
|
||||
/// Used by the daemon's pending-interrupt machinery: when the
|
||||
/// user has clicked the agent-window mask's stop button, the
|
||||
/// next *mutating* tool call for that session is rejected with
|
||||
/// `ErrorCode::UserAborted`. Read-only tools and session-
|
||||
/// lifecycle RPCs pass through transparently.
|
||||
/// next tool call that dispatches browser/page input for that
|
||||
/// session is rejected with `ErrorCode::UserAborted`. Passive reads
|
||||
/// and control-plane RPCs pass through transparently.
|
||||
///
|
||||
/// **Compile-time enforcement.** The match below is exhaustive
|
||||
/// (no `_ =>` fallthrough). Adding a new `Method` variant is a
|
||||
@@ -101,9 +119,12 @@ impl Method {
|
||||
///
|
||||
/// **Judgment calls** (read these before adding new variants):
|
||||
///
|
||||
/// * `tool.evaluate` is classified as mutating because the
|
||||
/// * `tool.evaluate` is classified as browser-mutating because the
|
||||
/// daemon cannot statically distinguish a `document.title`
|
||||
/// read from a `form.submit()` write.
|
||||
/// * `tool.observe` is classified as transient input: its bounded
|
||||
/// hover probes do not commit browser state, but they dispatch real
|
||||
/// page input events and therefore must be gated like automation.
|
||||
/// * `tool.wait_*` are classified as read-only: they do not
|
||||
/// initiate any browser action; they observe state only.
|
||||
/// * `session.*` and `tool.session_*` are NOT gated. Blocking
|
||||
@@ -111,9 +132,9 @@ impl Method {
|
||||
/// tearing down after observing the user's interrupt.
|
||||
/// * `cancel` is NOT gated. It's a control-plane operation
|
||||
/// (stops another in-flight RPC), not a browser action.
|
||||
pub fn is_mutating(&self) -> bool {
|
||||
pub fn effect(&self) -> MethodEffect {
|
||||
match self {
|
||||
// Mutating tool calls — gated by pending-interrupt.
|
||||
// Browser/page mutations — gated by pending-interrupt.
|
||||
Method::ToolTabCreate
|
||||
| Method::ToolTabClose
|
||||
| Method::ToolTabBorrow
|
||||
@@ -130,10 +151,13 @@ impl Method {
|
||||
| Method::ToolEvaluate
|
||||
// May navigate via optional `url` and changes Agent Window
|
||||
// chrome; gate behind pending-interrupt like other writes.
|
||||
| Method::ToolRecordStart => true,
|
||||
| Method::ToolRecordStart => MethodEffect::BrowserMutation,
|
||||
|
||||
// Read-only tool calls — transparent.
|
||||
//
|
||||
// Transient input — no committed browser action, but still page
|
||||
// input. It must be stopped by pending user interrupts.
|
||||
Method::ToolObserve => MethodEffect::TransientInput,
|
||||
|
||||
// Passive reads — transparent.
|
||||
// `record_stop` / `record_await` observe / finish a recording
|
||||
// without driving new automation gestures, so they stay
|
||||
// ungated (teardown after interrupt must still work).
|
||||
@@ -147,7 +171,7 @@ impl Method {
|
||||
| Method::ToolWaitMs
|
||||
| Method::ToolRequestHelp
|
||||
| Method::ToolRecordStop
|
||||
| Method::ToolRecordAwait => false,
|
||||
| Method::ToolRecordAwait => MethodEffect::PassiveRead,
|
||||
|
||||
// Session lifecycle — not gated.
|
||||
Method::SessionStart
|
||||
@@ -155,16 +179,29 @@ impl Method {
|
||||
| Method::SessionStopAll
|
||||
| Method::SessionList
|
||||
| Method::ToolSessionStart
|
||||
| Method::ToolSessionStop => false,
|
||||
| Method::ToolSessionStop => MethodEffect::ControlPlane,
|
||||
|
||||
// System / control — not gated.
|
||||
Method::SystemHandshake
|
||||
| Method::SystemPing
|
||||
| Method::SystemStatus
|
||||
| Method::BrowserList
|
||||
| Method::Cancel => false,
|
||||
| Method::Cancel => MethodEffect::ControlPlane,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this RPC drives browser/page state in the traditional sense.
|
||||
pub fn is_mutating(&self) -> bool {
|
||||
self.effect() == MethodEffect::BrowserMutation
|
||||
}
|
||||
|
||||
/// Whether a pending user interrupt should reject this method.
|
||||
pub fn requires_interrupt_gate(&self) -> bool {
|
||||
matches!(
|
||||
self.effect(),
|
||||
MethodEffect::TransientInput | MethodEffect::BrowserMutation
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -209,6 +246,7 @@ mod tests {
|
||||
fn is_mutating_classifies_read_only_tools_as_non_mutating() {
|
||||
assert!(!Method::ToolTabList.is_mutating());
|
||||
assert!(!Method::ToolSnapshot.is_mutating());
|
||||
assert!(!Method::ToolObserve.is_mutating());
|
||||
assert!(!Method::ToolGetHtml.is_mutating());
|
||||
assert!(!Method::ToolScreenshot.is_mutating());
|
||||
assert!(!Method::ToolConsole.is_mutating());
|
||||
@@ -264,4 +302,20 @@ mod tests {
|
||||
assert!(!Method::BrowserList.is_mutating());
|
||||
assert!(!Method::Cancel.is_mutating());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn effect_classifies_observe_as_transient_input() {
|
||||
assert_eq!(Method::ToolSnapshot.effect(), MethodEffect::PassiveRead);
|
||||
assert_eq!(Method::ToolObserve.effect(), MethodEffect::TransientInput);
|
||||
assert_eq!(Method::ToolClick.effect(), MethodEffect::BrowserMutation);
|
||||
assert_eq!(Method::Cancel.effect(), MethodEffect::ControlPlane);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn interrupt_gate_includes_transient_input() {
|
||||
assert!(!Method::ToolSnapshot.requires_interrupt_gate());
|
||||
assert!(Method::ToolObserve.requires_interrupt_gate());
|
||||
assert!(Method::ToolClick.requires_interrupt_gate());
|
||||
assert!(!Method::Cancel.requires_interrupt_gate());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! Read-only observation tools (`tool.snapshot`, `tool.get_html`, `tool.screenshot`).
|
||||
//! Observation tools (`tool.snapshot`, `tool.observe`, `tool.get_html`, `tool.screenshot`).
|
||||
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -50,6 +50,50 @@ pub struct SnapshotResult {
|
||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// observe
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Parameters for `tool.observe` — produce a semantic VOM observation
|
||||
/// for one tab plus a fresh `@e<N>` ref-store on the extension side.
|
||||
///
|
||||
/// Unlike `tool.snapshot`, this path may run bounded perception probes
|
||||
/// such as hover-surface discovery. It must not submit input, click,
|
||||
/// navigate, or otherwise commit page state.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct ObserveParams {
|
||||
pub session_id: String,
|
||||
/// Optional target tab. Defaults to the Agent Window's currently
|
||||
/// active tab.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub tab_id: Option<i64>,
|
||||
/// Cap the depth of the rendered VOM tree.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_depth: Option<u32>,
|
||||
/// Soft cap on rendered tokens (approximate; extension uses a
|
||||
/// best-effort heuristic based on character count).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_tokens: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct ObserveResult {
|
||||
/// Semantic VOM observation text. Refs are rendered as `@e<N>` so
|
||||
/// the agent can copy them into subsequent interaction tools.
|
||||
pub text: String,
|
||||
/// Number of `@e<N>` refs registered for this session by this
|
||||
/// observation. Equivalent to the size of the new ref-store map.
|
||||
pub ref_count: u32,
|
||||
/// Tab the observation was computed for.
|
||||
pub tab_id: i64,
|
||||
/// Whether the rendered tree was truncated because of `max_depth` /
|
||||
/// `max_tokens`. Agents may re-run with looser caps.
|
||||
#[serde(default)]
|
||||
pub truncated: bool,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// get_html
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -227,4 +271,19 @@ mod tests {
|
||||
let round: ScreenshotResult = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, r);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn observe_result_round_trips_with_ref_count() {
|
||||
let r = ObserveResult {
|
||||
text: "@vom 1\n @e1 button \"submit\"\n".into(),
|
||||
ref_count: 1,
|
||||
tab_id: 42,
|
||||
truncated: false,
|
||||
dialogs: Vec::new(),
|
||||
};
|
||||
let v = serde_json::to_value(&r).unwrap();
|
||||
assert_eq!(v.get("ref_count").and_then(|v| v.as_u64()), Some(1));
|
||||
let round: ObserveResult = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, r);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -386,7 +386,7 @@ describe("renderVom single-layer page", () => {
|
||||
],
|
||||
});
|
||||
|
||||
expect(out.text).toContain('@e1 button "Products" [→ Shoes | Bags | Accessories]');
|
||||
expect(out.text).toContain('@e1 button "Products" [hover: Shoes | Bags | Accessories]');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
|
||||
@@ -633,7 +633,7 @@ function renderNodeLine(
|
||||
if (surface && surface.subItems.length > 0) {
|
||||
const items = surface.subItems.slice(0, MAX_SURFACE_ITEMS).join(" | ");
|
||||
const suffix = surface.subItems.length > MAX_SURFACE_ITEMS ? " | …" : "";
|
||||
line += ` [→ ${items}${suffix}]`;
|
||||
line += ` [${surface.triggerAction}: ${items}${suffix}]`;
|
||||
}
|
||||
|
||||
return line;
|
||||
|
||||
Reference in New Issue
Block a user