feat(vom): update vom algorithm, fix overlay problem with background state machine

This commit is contained in:
Ljy-0827
2026-07-27 13:46:33 +08:00
parent e14ae9cea0
commit 4fb2e331a6
36 changed files with 2276 additions and 452 deletions
@@ -274,6 +274,35 @@ describe("ToolDispatcher", () => {
expect(onSessionsChanged).toHaveBeenCalledTimes(2);
});
it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => {
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 1),
remove: vi.fn(async () => {}),
ensureActiveTab: vi.fn(async () => {}),
},
});
const onBrowserControlResumed = vi.fn();
const dispatcher = new ToolDispatcher({ transport, sessions, onBrowserControlResumed });
dispatcher.start();
deliver(makeRequest("tool.session_start", { session_id: "aa11" }));
await flushMicrotasks();
sent.length = 0;
deliver({ ...makeRequest("tool.snapshot", { session_id: "aa11" }), id: "r-passive" });
await flushMicrotasks();
expect(onBrowserControlResumed).not.toHaveBeenCalled();
deliver({
...makeRequest("tool.tab_create", { session_id: "aa11", url: "https://example.test/" }),
id: "r-control",
});
await flushMicrotasks();
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
});
it("disconnects the transport when send() fails so keepalive can rebuild", async () => {
const { transport, deliver } = fakeTransport();
const sessions = new SessionManager({
@@ -7,6 +7,7 @@ import {
buildVomScene,
type CdpAxNode,
handleGetHtml,
handleObserve,
handleScreenshot,
handleSnapshot,
parsePngDimensions,
@@ -1412,7 +1413,7 @@ describe("buildVomScene", () => {
expect(scene.surfaces).toEqual([
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
]);
expect(renderVom(scene).text).toContain('@e1 button "Products" [→ Shoes | Bags]');
expect(renderVom(scene).text).toContain('@e1 button "Products" [hover: Shoes | Bags]');
});
it("enriches names and active scope signals from AX properties", () => {
@@ -1741,6 +1742,225 @@ describe("handleSnapshot", () => {
expect(ctx.refStore.resolve("e2")).toBeNull();
});
it("keeps snapshots static without conditional surface probing", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
const root: CdpAxNode = {
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
name: { type: "computedString", value: "Example" },
backendDOMNodeId: 100,
childIds: ["2"],
};
const button: CdpAxNode = {
nodeId: "2",
parentId: "1",
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Products" },
backendDOMNodeId: 200,
};
const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"];
const i = (s: string) => strings.indexOf(s);
const send = vi.fn(async (_tabId: number, method: string) => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
return {
strings,
documents: [
{
nodes: {
parentIndex: [-1, 0],
nodeName: [i("body"), i("button")],
backendNodeId: [100, 200],
attributes: [[], []],
},
layout: {
nodeIndex: [0, 1],
styles: [
[i("static"), i("auto"), i("auto")],
[i("static"), i("auto"), i("auto")],
],
bounds: [
[0, 0, 1000, 800],
[20, 20, 120, 40],
],
paintOrders: [0, 1],
},
},
],
};
}
if (method === "Runtime.evaluate") return { result: { value: [] } };
throw new Error(`unexpected CDP method ${method}`);
});
const res = await handleSnapshot(
sm,
{ session_id: "aa11" },
{
cdp: {
send: send as unknown as <T = unknown>(
tabId: number,
method: string,
params?: object,
) => Promise<T>,
trackSessionTab: vi.fn(),
},
tabsApi: {
get: vi.fn(
async (tabId: number) =>
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
},
},
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(send).not.toHaveBeenCalledWith(
4,
"Runtime.evaluate",
expect.objectContaining({ returnByValue: true }),
);
});
it("enables conditional surface probing for semantic observe", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
const root: CdpAxNode = {
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
name: { type: "computedString", value: "Example" },
backendDOMNodeId: 100,
childIds: ["2"],
};
const button: CdpAxNode = {
nodeId: "2",
parentId: "1",
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Products" },
backendDOMNodeId: 200,
};
const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"];
const i = (s: string) => strings.indexOf(s);
const send = vi.fn(async (_tabId: number, method: string) => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
return {
strings,
documents: [
{
nodes: {
parentIndex: [-1, 0],
nodeName: [i("body"), i("button")],
backendNodeId: [100, 200],
attributes: [[], []],
},
layout: {
nodeIndex: [0, 1],
styles: [
[i("static"), i("auto"), i("auto")],
[i("static"), i("auto"), i("auto")],
],
bounds: [
[0, 0, 1000, 800],
[20, 20, 120, 40],
],
paintOrders: [0, 1],
},
},
],
};
}
if (method === "Runtime.evaluate") return { result: { value: [] } };
throw new Error(`unexpected CDP method ${method}`);
});
const res = await handleObserve(
sm,
{ session_id: "aa11" },
{
cdp: {
send: send as unknown as <T = unknown>(
tabId: number,
method: string,
params?: object,
) => Promise<T>,
trackSessionTab: vi.fn(),
},
tabsApi: {
get: vi.fn(
async (tabId: number) =>
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
},
},
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(send).toHaveBeenCalledWith(
4,
"Runtime.evaluate",
expect.objectContaining({ returnByValue: true }),
);
});
it("allows passive snapshots of explicit user-window tabs", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
const root: CdpAxNode = {
nodeId: "1",
role: { type: "role", value: "RootWebArea" },
name: { type: "computedString", value: "User tab" },
backendDOMNodeId: 100,
childIds: ["2"],
};
const button: CdpAxNode = {
nodeId: "2",
parentId: "1",
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Read" },
backendDOMNodeId: 200,
};
const deps = makeDeps([root, button]);
deps.tabsApi.get = vi.fn(
async (tabId: number) => ({ id: tabId, windowId: 200, active: true }) as chrome.tabs.Tab,
);
const res = await handleSnapshot(sm, { session_id: "aa11", tab_id: 9 }, deps);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.tab_id).toBe(9);
expect(res.text).toContain('@e1 button "Read"');
expect(deps.send).toHaveBeenCalledWith(9, "Accessibility.enable", {});
});
it("rejects semantic observe on explicit user-window tabs before CDP probing", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
const deps = makeDeps([]);
deps.tabsApi.get = vi.fn(
async (tabId: number) => ({ id: tabId, windowId: 200, active: true }) as chrome.tabs.Tab,
);
const res = await handleObserve(sm, { session_id: "aa11", tab_id: 9 }, deps);
expect(res).toMatchObject({
code: "permission_denied",
data: { reason: "agent_window_scope" },
});
expect(deps.send).not.toHaveBeenCalled();
});
it("resets the RefStore on every fresh snapshot", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
@@ -1,6 +1,12 @@
import { describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import { lookupSession, parseBufferedReadBounds } from "../shared";
import {
cdpBlockedUrlReason,
enforceCdpAccessibleTarget,
lookupSession,
parseBufferedReadBounds,
resolveCdpAccessibleTargetTab,
} from "../shared";
function fakeAgentWindow() {
return {
@@ -67,3 +73,101 @@ describe("parseBufferedReadBounds", () => {
});
});
});
describe("CDP target URL guard", () => {
it("allows ordinary pages and about:blank", () => {
expect(cdpBlockedUrlReason("https://example.test/path")).toBeNull();
expect(cdpBlockedUrlReason("http://example.test/path")).toBeNull();
expect(cdpBlockedUrlReason("about:blank")).toBeNull();
});
it("blocks browser and extension internal pages", () => {
expect(cdpBlockedUrlReason("chrome-extension://abc/options.html")).toBe("chrome-extension:");
expect(cdpBlockedUrlReason("chrome://extensions")).toBe("chrome:");
expect(cdpBlockedUrlReason("about:newtab")).toBe("about:");
});
it("returns a structured permission error before page CDP access", () => {
expect(
enforceCdpAccessibleTarget(
{
tabId: 7,
windowId: 100,
active: true,
url: "chrome-extension://other-extension/page.html",
},
"snapshot",
),
).toMatchObject({
code: "permission_denied",
data: { reason: "restricted_tab_url" },
message: expect.stringContaining("snapshot cannot access tab 7"),
});
});
it("falls back from a restricted default active tab to an accessible agent tab", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
const ctx = await sm.start("aa11");
const tabsApi = {
get: vi.fn(),
query: vi.fn(async (query: chrome.tabs.QueryInfo) => {
if (query.active) {
return [
{
id: 7,
windowId: ctx.agentWindowId,
active: true,
url: "chrome-extension://other-extension/page.html",
} as chrome.tabs.Tab,
];
}
return [
{
id: 7,
windowId: ctx.agentWindowId,
active: true,
url: "chrome-extension://other-extension/page.html",
} as chrome.tabs.Tab,
{
id: 8,
windowId: ctx.agentWindowId,
active: false,
url: "https://example.test/",
} as chrome.tabs.Tab,
];
}),
};
await expect(
resolveCdpAccessibleTargetTab(sm, ctx, undefined, tabsApi, "snapshot"),
).resolves.toMatchObject({
tabId: 8,
url: "https://example.test/",
});
});
it("does not replace an explicitly requested restricted tab", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow() });
const ctx = await sm.start("aa11");
const tabsApi = {
get: vi.fn(
async () =>
({
id: 7,
windowId: ctx.agentWindowId,
active: true,
url: "chrome-extension://other-extension/page.html",
}) as chrome.tabs.Tab,
),
query: vi.fn(),
};
await expect(
resolveCdpAccessibleTargetTab(sm, ctx, 7, tabsApi, "snapshot"),
).resolves.toMatchObject({
code: "permission_denied",
data: { reason: "restricted_tab_url" },
});
expect(tabsApi.query).not.toHaveBeenCalled();
});
});
+41
View File
@@ -11,6 +11,7 @@ import type {
NavigateForwardParams,
NavigateParams,
NetworkParams,
ObserveParams,
PressParams,
ProtocolFrame,
RecordAwaitParams,
@@ -42,6 +43,7 @@ import {
type CdpRunner,
chromeTabsCaptureApi,
handleGetHtml,
handleObserve,
handleScreenshot,
handleSnapshot,
} from "./observation";
@@ -86,6 +88,8 @@ export interface DispatcherDeps {
* `chrome.storage.session` "sessions live" flag (review M4/M5 I3).
*/
onSessionsChanged?: () => void;
/** Invoked before a tool that dispatches page input or mutates browser state is forwarded. */
onBrowserControlResumed?: (sessionId: string) => void;
/** User approval for `tool.tab_borrow` (overlay in content script). */
approveBorrow?: BorrowConfirmationApprover;
/** i18n notification copy for `tool.request_help` (resolved per-call). */
@@ -114,6 +118,7 @@ export class ToolDispatcher {
private readonly sessions: SessionManager;
private readonly cdp?: DispatcherCdpRunner;
private readonly onSessionsChanged?: () => void;
private readonly onBrowserControlResumed?: (sessionId: string) => void;
private readonly approveBorrow?: BorrowConfirmationApprover;
private readonly helpNotificationCopy?: () => { title: string; body: string };
private subscription: { dispose(): void } | null = null;
@@ -130,6 +135,7 @@ export class ToolDispatcher {
this.sessions = deps.sessions;
this.cdp = deps.cdp;
this.onSessionsChanged = deps.onSessionsChanged;
this.onBrowserControlResumed = deps.onBrowserControlResumed;
this.approveBorrow = deps.approveBorrow;
this.helpNotificationCopy = deps.helpNotificationCopy;
}
@@ -193,6 +199,8 @@ export class ToolDispatcher {
let body: ResponseFrame;
let startedSession: string | null = null;
try {
const sessionId = sessionIdForBrowserControlMethod(req);
if (sessionId) this.onBrowserControlResumed?.(sessionId);
const result = await Promise.race([this.invoke(req, ac.signal), abortPromise(ac.signal)]);
if (isRpcError(result)) {
body = { id: req.id, error: result };
@@ -304,6 +312,12 @@ export class ToolDispatcher {
req.params as SnapshotParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
);
case "tool.observe":
return handleObserve(
this.sessions,
req.params as ObserveParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
);
case "tool.get_html":
return handleGetHtml(
this.sessions,
@@ -457,6 +471,33 @@ function isRpcError(v: unknown): v is RpcError {
);
}
function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
switch (req.method) {
case "tool.tab_create":
case "tool.tab_close":
case "tool.tab_select":
case "tool.tab_borrow":
case "tool.tab_return":
case "tool.navigate":
case "tool.navigate_back":
case "tool.navigate_forward":
case "tool.reload":
case "tool.click":
case "tool.fill":
case "tool.press":
case "tool.select":
case "tool.evaluate":
case "tool.observe":
case "tool.request_help":
case "tool.record_start": {
const sessionId = (req.params as { session_id?: unknown } | undefined)?.session_id;
return typeof sessionId === "string" && sessionId.length > 0 ? sessionId : null;
}
default:
return null;
}
}
/**
* Resolves never; rejects with `AbortLikeError` as soon as the signal
* fires (or immediately if it is already aborted). Used by the
+64 -13
View File
@@ -1,5 +1,5 @@
// Read-only observation handlers — `tool.screenshot`, `tool.snapshot`,
// and `tool.get_html` (design §7). Each handler resolves the target
// Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`,
// and semantic `tool.observe` (design §7). Each handler resolves the target
// tab (defaulting to the Agent Window's active tab when omitted) and
// returns a payload that mirrors the bsk-protocol Rust structs.
@@ -15,6 +15,8 @@ import type { SessionManager } from "@/session-manager/manager";
import type {
GetHtmlParams,
GetHtmlResult,
ObserveParams,
ObserveResult,
RpcError,
ScreenshotParams,
ScreenshotResult,
@@ -26,11 +28,13 @@ import { nodeBoundingRect, scrollNodeIntoView } from "./element-geometry";
import { rpcError } from "./errors";
import {
type ChromeTabsApi,
enforceToolTargetScope,
isRpcError,
lookupSession,
resolveTargetTab,
resolveCdpAccessibleTargetTab,
type CdpRunner as SharedCdpRunner,
normaliseRef as sharedNormaliseRef,
type ToolEffect,
} from "./shared";
import { resolveSnapshotRef } from "./snapshot-ref";
import {
@@ -167,7 +171,13 @@ export async function handleScreenshot(
const ctxOrErr = lookupSession(manager, params, "screenshot");
if (isRpcError(ctxOrErr)) return ctxOrErr;
const ctx = ctxOrErr;
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
const target = await resolveCdpAccessibleTargetTab(
manager,
ctx,
params.tab_id,
deps.tabsApi,
"screenshot",
);
if (isRpcError(target)) return target;
const dialogCursor = deps.cdp ? markDialogCursor(deps.cdp, target.tabId) : 0;
const withShotDialogs = <T extends object>(result: T) =>
@@ -181,6 +191,7 @@ export async function handleScreenshot(
const node = resolveSnapshotRef(ctx, ref, target.tabId);
if (isRpcError(node)) return node;
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
const captured = await captureElementScreenshot(deps.cdp, target.tabId, node.backendNodeId);
if (isRpcError(captured)) return captured;
return withShotDialogs({
@@ -664,6 +675,7 @@ function inputStateFor(
value: string | undefined,
): VomNode["inputState"] {
if (!capturedNode || !FORM_CONTROL_TAGS.has(normalizeTag(capturedNode.tag))) return undefined;
if (capturedNode.formState) return capturedNode.formState;
const formValue = capturedNode.formValue;
if (formValue !== undefined) {
if (formValue === "") return "empty";
@@ -1099,7 +1111,13 @@ export async function handleGetHtml(
const ctxOrErr = lookupSession(manager, params, "get_html");
if (isRpcError(ctxOrErr)) return ctxOrErr;
const ctx = ctxOrErr;
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
const target = await resolveCdpAccessibleTargetTab(
manager,
ctx,
params.tab_id,
deps.tabsApi,
"get_html",
);
if (isRpcError(target)) return target;
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
@@ -1108,6 +1126,7 @@ export async function handleGetHtml(
try {
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
let html: string;
if (params.ref) {
const resolved = resolveSnapshotRef(ctx, params.ref, target.tabId);
@@ -1178,28 +1197,44 @@ async function fallbackCapturedViewModel(
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
}
async function captureForVom(cdp: CdpRunner, tabId: number): Promise<CapturedViewModel> {
async function captureForVom(
cdp: CdpRunner,
tabId: number,
conditionalSurfaceProbe: boolean,
): Promise<CapturedViewModel> {
try {
return await captureViewModel(cdp, tabId);
return await captureViewModel(cdp, tabId, { conditionalSurfaceProbe });
} catch {
return fallbackCapturedViewModel(cdp, tabId);
}
}
export async function handleSnapshot(
async function handleVomObservation(
manager: SessionManager,
params: SnapshotParams,
params: SnapshotParams | ObserveParams,
toolName: "snapshot" | "observe",
effect: ToolEffect,
conditionalSurfaceProbe: boolean,
deps: SnapshotDeps = getDefaultDeps(),
): Promise<SnapshotResult | RpcError> {
const ctxOrErr = lookupSession(manager, params, "snapshot");
): Promise<SnapshotResult | ObserveResult | RpcError> {
const ctxOrErr = lookupSession(manager, params, toolName);
if (isRpcError(ctxOrErr)) return ctxOrErr;
const ctx = ctxOrErr;
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
const target = await resolveCdpAccessibleTargetTab(
manager,
ctx,
params.tab_id,
deps.tabsApi,
toolName,
);
if (isRpcError(target)) return target;
const denied = enforceToolTargetScope(ctx, target, effect, toolName);
if (denied) return denied;
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
try {
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
await deps.cdp.send<unknown>(target.tabId, "Accessibility.enable", {});
const result = await deps.cdp.send<{ nodes: CdpAxNode[] }>(
target.tabId,
@@ -1207,7 +1242,7 @@ export async function handleSnapshot(
{},
);
const axNodes = result.nodes ?? [];
const captured = await captureForVom(deps.cdp, target.tabId);
const captured = await captureForVom(deps.cdp, target.tabId, conditionalSurfaceProbe);
const scene = buildVomScene(axNodes, captured);
const rendered = renderVom(scene, {
maxDepth: params.max_depth,
@@ -1232,3 +1267,19 @@ export async function handleSnapshot(
};
}
}
export async function handleSnapshot(
manager: SessionManager,
params: SnapshotParams,
deps: SnapshotDeps = getDefaultDeps(),
): Promise<SnapshotResult | RpcError> {
return handleVomObservation(manager, params, "snapshot", "passive_read", false, deps);
}
export async function handleObserve(
manager: SessionManager,
params: ObserveParams,
deps: SnapshotDeps = getDefaultDeps(),
): Promise<ObserveResult | RpcError> {
return handleVomObservation(manager, params, "observe", "transient_input", true, deps);
}
+4 -4
View File
@@ -369,7 +369,7 @@ async function rearmRecording(
deps: RecordDeps,
): Promise<boolean> {
// Do NOT toggle automation-bypass here: each retry used to increment the
// content-script counter, and a single stop decrement left the ControlOverlay
// content-script counter, and a single stop decrement left the control mask
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
// already hides the control chrome while activeRecord is set.
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
@@ -561,9 +561,9 @@ export async function handleRecordStart(
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
if (isRpcError(target)) return target;
// Register the recording *before* navigate so content-script syncAgentOverlay
// on the destination page can RECORD_QUERY → rearm → show RecordOverlay
// instead of flashing ControlOverlay ("Agent 正在控制").
// Register the recording *before* navigate so the destination content script's
// overlay.ready flow can RECORD_QUERY → rearm → show RecordOverlay
// instead of flashing the control mask ("Agent 正在控制").
const requestId = makeRequestId(target.tabId);
let resolveFinish!: (trace: Trace) => void;
let rejectFinish!: (err: Error) => void;
+115 -3
View File
@@ -32,6 +32,13 @@ export const chromeTabsApi: ChromeTabsApi = {
query: (q) => chrome.tabs.query(q),
};
export interface ResolvedTargetTab {
tabId: number;
windowId: number;
active: boolean;
url?: string;
}
export type { DialogCursor };
/**
@@ -44,6 +51,7 @@ export type { DialogCursor };
*/
export interface CdpRunner {
send<T = unknown>(tabId: number, method: string, params?: object): Promise<T>;
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
trackSessionTab?(sessionId: string, tabId: number): void;
onEvent?(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): {
dispose(): void;
@@ -146,7 +154,7 @@ export async function resolveTargetTab(
ctx: SessionContext,
tabId: number | undefined,
api: ChromeTabsApi,
): Promise<{ tabId: number; windowId: number; active: boolean } | RpcError> {
): Promise<ResolvedTargetTab | RpcError> {
if (tabId !== undefined) {
if (!Number.isSafeInteger(tabId) || tabId <= 0) {
return {
@@ -176,7 +184,7 @@ export async function resolveTargetTab(
message: `tab ${tabId} not found in session scope`,
};
}
return { tabId: tab.id, windowId: tab.windowId, active: tab.active === true };
return { tabId: tab.id, windowId: tab.windowId, active: tab.active === true, url: tab.url };
}
const tabs = await api.query({ active: true, windowId: ctx.agentWindowId });
const first = tabs.find((t) => typeof t.id === "number");
@@ -186,7 +194,12 @@ export async function resolveTargetTab(
message: `no active tab in Agent Window ${ctx.agentWindowId}`,
};
}
return { tabId: first.id, windowId: ctx.agentWindowId, active: first.active === true };
return {
tabId: first.id,
windowId: ctx.agentWindowId,
active: first.active === true,
url: first.url,
};
}
export function isRpcError(v: unknown): v is RpcError {
@@ -202,6 +215,91 @@ export function isRpcError(v: unknown): v is RpcError {
/** Re-export so M6/M7 tools keep a stable import path. */
export { normaliseRef };
export type ToolEffect = "passive_read" | "transient_input" | "browser_mutation";
const CDP_BLOCKED_PROTOCOLS = new Set([
"chrome:",
"chrome-extension:",
"devtools:",
"edge:",
"brave:",
"vivaldi:",
"opera:",
]);
export function cdpBlockedUrlReason(url: string | undefined): string | null {
if (!url) return null;
let parsed: URL;
try {
parsed = new URL(url);
} catch {
return null;
}
if (CDP_BLOCKED_PROTOCOLS.has(parsed.protocol)) return parsed.protocol;
if (parsed.protocol === "about:" && parsed.pathname !== "blank") return "about:";
return null;
}
/**
* Page CDP tools cannot inspect browser/extension internal pages. Check this
* at the tool boundary so Chrome's low-level "Cannot access..." errors do not
* leak as ambiguous page-read failures.
*/
export function enforceCdpAccessibleTarget(
target: ResolvedTargetTab,
toolName: string,
): RpcError | null {
const reason = cdpBlockedUrlReason(target.url);
if (!reason) return null;
return rpcError(
"permission_denied",
"restricted_tab_url",
`${toolName} cannot access tab ${target.tabId} because its URL is ${target.url}; navigate the Agent Window to a web page first`,
);
}
function resolvedTargetFromChromeTab(
tab: chrome.tabs.Tab,
fallbackWindowId: number,
): ResolvedTargetTab | null {
if (typeof tab.id !== "number") return null;
return {
tabId: tab.id,
windowId: typeof tab.windowId === "number" ? tab.windowId : fallbackWindowId,
active: tab.active === true,
url: tab.url,
};
}
/**
* Resolve a target for page CDP reads. Explicit tab_ids stay exact and are
* rejected when browser security blocks the URL. For default targeting, skip a
* restricted Agent Window active tab if another CDP-accessible tab exists in
* the same Agent Window.
*/
export async function resolveCdpAccessibleTargetTab(
manager: SessionManager,
ctx: SessionContext,
tabId: number | undefined,
api: ChromeTabsApi,
toolName: string,
): Promise<ResolvedTargetTab | RpcError> {
const target = await resolveTargetTab(manager, ctx, tabId, api);
if (isRpcError(target)) return target;
const restricted = enforceCdpAccessibleTarget(target, toolName);
if (!restricted) return target;
if (tabId !== undefined) return restricted;
const tabs = await api.query({ windowId: ctx.agentWindowId });
for (const tab of tabs) {
const candidate = resolvedTargetFromChromeTab(tab, ctx.agentWindowId);
if (!candidate) continue;
if (!enforceCdpAccessibleTarget(candidate, toolName)) return candidate;
}
return restricted;
}
/**
* Sandbox guard: M7 write tools (click / fill / press / navigate*)
* MUST refuse to touch a tab outside the session's Agent Window
@@ -224,3 +322,17 @@ export function enforceAgentWindow(
}
return null;
}
/**
* Unified target-scope policy by tool effect. Passive reads may inspect user
* tabs; any tool that dispatches page input must stay inside the Agent Window.
*/
export function enforceToolTargetScope(
ctx: SessionContext,
target: { tabId: number; windowId: number },
effect: ToolEffect,
toolName: string,
): RpcError | null {
if (effect === "passive_read") return null;
return enforceAgentWindow(ctx, target, toolName);
}
@@ -59,6 +59,23 @@ function makeCdp(snapshot: unknown) {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
}
if (method === "Runtime.evaluate") {
return {
result: {
value: {
controls: [
{
state: "filled",
sensitive: true,
defaultValue: "",
placeholder: "secret",
},
],
childFrames: [],
},
},
};
}
throw new Error(`unexpected ${method}`);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
@@ -80,6 +97,89 @@ describe("captureViewModel", () => {
expect(input?.attrs).toEqual({ type: "password" });
});
it("does not run conditional surface probes by default", async () => {
const cdp = makeCdp(fakeSnapshotReply());
const result = await captureViewModel(cdp, 4);
expect(result.surfaceProbes).toEqual([]);
expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything());
});
it("batch-enriches form controls without per-node object resolution", async () => {
const cdp = makeCdp(fakeSnapshotReply());
const result = await captureViewModel(cdp, 4);
const input = result.nodes.find((node) => node.backendNodeId === 13);
expect(input).toMatchObject({
formState: "filled",
formDefaultValue: "",
formPlaceholder: "secret",
});
expect(input?.formValue).toBeUndefined();
expect(cdp.send).toHaveBeenCalledWith(
4,
"Runtime.evaluate",
expect.objectContaining({ returnByValue: true }),
);
expect(cdp.send).not.toHaveBeenCalledWith(4, "DOM.resolveNode", expect.anything());
expect(cdp.send).not.toHaveBeenCalledWith(4, "Runtime.callFunctionOn", expect.anything());
});
it("runs hover surface probes when explicitly enabled", async () => {
let runtimeCalls = 0;
const cdp = {
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "Runtime.evaluate") {
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
if (expression.includes("input,textarea,select")) {
return { result: { value: { controls: [], childFrames: [] } } };
}
runtimeCalls += 1;
return runtimeCalls === 1
? {
result: {
value: [
{
triggerSel: ".menu",
affectedSub: " .items",
label: "Products",
x: 50,
y: 20,
},
],
},
}
: { result: { value: ["Shoes", "Bags"] } };
}
if (method === "Input.dispatchMouseEvent") return {};
throw new Error(`unexpected ${method}`);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
expect(result.surfaceProbes).toEqual([
{ triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] },
]);
expect(cdp.send).toHaveBeenCalledWith(
4,
"Input.dispatchMouseEvent",
expect.objectContaining({ type: "mouseMoved", x: 50, y: 20 }),
);
expect(cdp.send).toHaveBeenCalledWith(
4,
"Input.dispatchMouseEvent",
expect.objectContaining({ type: "mouseMoved", x: -10, y: -10 }),
);
});
it("excludes the agent's own overlay shadow host and its inlined shadow subtree", async () => {
// DOMSnapshot inlines an open shadow root's content as descendants of the
// host. The agent's WXT overlay host carries a fixed full-viewport
@@ -349,8 +449,11 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { nodes } = await captureViewModel(cdp, 4);
// overlay div bounds y=0 minus scroll pageY=200 → y=-200
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(-200);
const div = nodes.find((n) => n.backendNodeId === 12);
// local rect preserves viewport-relative scroll math.
expect(div?.localRect?.y).toBe(-200);
// exported rect is clipped to the top-level viewport before VOM consumes it.
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
});
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
@@ -557,8 +660,187 @@ describe("captureViewModel", () => {
expect(iframeNodes.size).toBe(1);
expect(iframeNodes.has(13)).toBe(true);
const subNodes = iframeNodes.get(13)!;
expect(subNodes.find((n) => n.backendNodeId === 101)?.tag).toBe("input");
expect(subNodes.find((n) => n.backendNodeId === 101)?.attrs.type).toBe("text");
const input = subNodes.find((n) => n.backendNodeId === 101);
expect(input?.tag).toBe("input");
expect(input?.attrs.type).toBe("text");
expect(input?.ownerFrameBackendNodeId).toBe(13);
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
});
it("clips iframe sub-document rects to the iframe viewport in top coordinates", async () => {
const S = [
"html",
"body",
"iframe",
"input",
"position",
"static",
"pointer-events",
"auto",
"type",
"text",
];
const i = (s: string) => S.indexOf(s);
const snapshot = {
strings: S,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
nodes: {
parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("iframe")],
backendNodeId: [10, 11, 13],
attributes: [[], [], []],
contentDocumentIndex: { index: [2], value: [1] },
},
layout: {
nodeIndex: [1, 2],
styles: [
[i("static"), i("auto")],
[i("static"), i("auto")],
],
bounds: [
[0, 0, 1000, 800],
[950, 760, 200, 100],
],
paintOrders: [0, 1],
},
},
{
scrollOffsetX: 0,
scrollOffsetY: 0,
nodes: {
parentIndex: [-1, 0],
nodeName: [i("body"), i("input")],
backendNodeId: [100, 101],
attributes: [[], [i("type"), i("text")]],
},
layout: {
nodeIndex: [1],
styles: [[i("static"), i("auto")]],
bounds: [[20, 20, 120, 60]],
paintOrders: [0],
},
},
],
};
const cdp = {
send: vi.fn(async (_t: number, method: string) => {
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { iframeNodes } = await captureViewModel(cdp, 4);
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
expect(input?.rect).toEqual({ x: 970, y: 780, w: 30, h: 20 });
});
it("recursively normalizes nested iframe sub-documents", async () => {
const S = [
"html",
"body",
"iframe",
"input",
"position",
"static",
"pointer-events",
"auto",
"type",
"text",
];
const i = (s: string) => S.indexOf(s);
const snapshot = {
strings: S,
documents: [
{
nodes: {
parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("iframe")],
backendNodeId: [10, 11, 13],
attributes: [[], [], []],
contentDocumentIndex: { index: [2], value: [1] },
},
layout: {
nodeIndex: [1, 2],
styles: [
[i("static"), i("auto")],
[i("static"), i("auto")],
],
bounds: [
[0, 0, 1000, 800],
[10, 20, 300, 200],
],
paintOrders: [0, 1],
},
},
{
nodes: {
parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("iframe")],
backendNodeId: [20, 21, 23],
attributes: [[], [], []],
contentDocumentIndex: { index: [2], value: [2] },
},
layout: {
nodeIndex: [1, 2],
styles: [
[i("static"), i("auto")],
[i("static"), i("auto")],
],
bounds: [
[0, 0, 300, 200],
[5, 6, 100, 80],
],
paintOrders: [0, 1],
},
},
{
nodes: {
parentIndex: [-1, 0],
nodeName: [i("body"), i("input")],
backendNodeId: [30, 31],
attributes: [[], [i("type"), i("text")]],
},
layout: {
nodeIndex: [1],
styles: [[i("static"), i("auto")]],
bounds: [[1, 2, 40, 20]],
paintOrders: [0],
},
},
],
};
const cdp = {
send: vi.fn(async (_t: number, method: string) => {
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { iframeNodes } = await captureViewModel(cdp, 4);
const nestedIframe = iframeNodes.get(13)?.find((n) => n.backendNodeId === 23);
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31);
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
expect(input?.ownerFrameBackendNodeId).toBe(23);
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
});
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => {
+286 -76
View File
@@ -16,9 +16,14 @@ const STYLE_COL = Object.fromEntries(
export interface CapturedNode {
backendNodeId: number;
parentBackendNodeId: number | null;
/** Owning iframe backend node id; `null` for the top-level document. */
ownerFrameBackendNodeId?: number | null;
tag: string;
attrs: Record<string, string>;
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
rect: Rect | null;
/** Frame-local viewport-relative CSS px before top-level translation/clipping. */
localRect?: Rect | null;
paintOrder: number;
position: string;
pointerEvents: string;
@@ -31,6 +36,7 @@ export interface CapturedNode {
*/
cursor?: string;
textContent?: string;
formState?: "empty" | "filled" | "default";
formValue?: string;
formDefaultValue?: string;
formPlaceholder?: string;
@@ -53,6 +59,10 @@ export interface CapturedViewModel {
excludedBackendNodeIds: Set<number>;
}
export interface CaptureViewModelOptions {
conditionalSurfaceProbe?: boolean;
}
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
interface SparseArray {
index: number[];
@@ -102,48 +112,112 @@ function isFormControlTag(tag: string): boolean {
return tag === "input" || tag === "textarea" || tag === "select";
}
async function enrichFormControlState(cdp: CdpRunner, tabId: number, nodes: CapturedNode[]) {
const MAX_FORM_ENRICH_CONTROLS = 250;
interface CapturedFormState {
value?: string;
defaultValue?: string;
placeholder?: string;
state?: "empty" | "filled" | "default";
sensitive?: boolean;
}
interface CapturedFrameFormState {
controls: CapturedFormState[];
childFrames: CapturedFrameFormState[];
}
function formStateBatchExpression(maxControls: number): string {
return `(() => {
const maxControls = ${JSON.stringify(maxControls)};
let remaining = maxControls;
const controlSelector = "input,textarea,select";
const controlState = (el) => {
const tag = el.tagName.toLowerCase();
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
const sensitive = type === "password" || type === "credit-card";
const rawValue = typeof el.value === "string" ? el.value : "";
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
return {
state,
sensitive,
defaultValue,
placeholder,
...(sensitive ? {} : { value: rawValue }),
};
};
const collect = (doc) => {
const controls = [];
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
if (remaining <= 0) break;
controls.push(controlState(el));
remaining -= 1;
}
const childFrames = [];
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
let childDoc = null;
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
childFrames.push(childDoc && remaining > 0 ? collect(childDoc) : { controls: [], childFrames: [] });
}
return { controls, childFrames };
};
return collect(document);
})()`;
}
function flattenFrameFormStates(
frame: CapturedFrameFormState | undefined,
out: CapturedFormState[][] = [],
): CapturedFormState[][] {
if (!frame) return out;
out.push(Array.isArray(frame.controls) ? frame.controls : []);
for (const child of Array.isArray(frame.childFrames) ? frame.childFrames : []) {
flattenFrameFormStates(child, out);
}
return out;
}
function applyFormStates(nodes: CapturedNode[], states: CapturedFormState[]): void {
let index = 0;
for (const node of nodes) {
if (!isFormControlTag(node.tag)) continue;
let objectId: string | undefined;
try {
const resolved = await cdp.send<{ object?: { objectId?: string } }>(
tabId,
"DOM.resolveNode",
{
backendNodeId: node.backendNodeId,
},
);
objectId = resolved.object?.objectId;
if (!objectId) continue;
const result = await cdp.send<{
result?: { value?: { value?: string; defaultValue?: string; placeholder?: string } };
}>(tabId, "Runtime.callFunctionOn", {
objectId,
returnByValue: true,
functionDeclaration: `function(){
return {
value: typeof this.value === "string" ? this.value : "",
defaultValue: typeof this.defaultValue === "string" ? this.defaultValue : "",
placeholder: typeof this.placeholder === "string" ? this.placeholder : ""
};
}`,
});
const value = result.result?.value;
if (!value) continue;
node.formValue = value.value ?? "";
node.formDefaultValue = value.defaultValue ?? "";
node.formPlaceholder = value.placeholder ?? "";
} catch {
// Best-effort enrichment. DOMSnapshot/AX data still carries the node.
} finally {
if (objectId) {
await cdp.send(tabId, "Runtime.releaseObject", { objectId }).catch(() => {});
}
const state = states[index];
index += 1;
if (!state) continue;
node.formState = state.state;
node.formDefaultValue = state.defaultValue ?? "";
node.formPlaceholder = state.placeholder ?? "";
if (!state.sensitive && state.value !== undefined) {
node.formValue = state.value;
}
}
}
async function enrichFormControlStates(
cdp: CdpRunner,
tabId: number,
frameNodeGroups: CapturedNode[][],
): Promise<void> {
const hasControls = frameNodeGroups.some((nodes) =>
nodes.some((node) => isFormControlTag(node.tag)),
);
if (!hasControls) return;
try {
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
returnByValue: true,
});
const frameStates = flattenFrameFormStates(runtimeValue<CapturedFrameFormState>(result));
for (let i = 0; i < frameNodeGroups.length; i += 1) {
applyFormStates(frameNodeGroups[i], frameStates[i] ?? []);
}
} catch {
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
}
}
interface RuntimeEvaluateReply {
result?: {
value?: unknown;
@@ -169,6 +243,14 @@ interface ParseDocumentResult {
excludedBackendNodeIds: Set<number>;
}
interface FrameContext {
ownerFrameBackendNodeId: number | null;
originInTop: { x: number; y: number };
clipRectInTop: Rect;
scrollX: number;
scrollY: number;
}
function str(strings: string[], idx: number | undefined): string {
if (idx === undefined || idx < 0) return "";
return strings[idx] ?? "";
@@ -183,6 +265,49 @@ function devicePixelRatio(metrics: LayoutMetricsReply): number {
return dpr >= 1 ? dpr : 1;
}
function viewportRect(viewport: Viewport): Rect {
return { x: 0, y: 0, w: Math.max(0, viewport.width), h: Math.max(0, viewport.height) };
}
function intersectRects(a: Rect, b: Rect): Rect | null {
const x1 = Math.max(a.x, b.x);
const y1 = Math.max(a.y, b.y);
const x2 = Math.min(a.x + a.w, b.x + b.w);
const y2 = Math.min(a.y + a.h, b.y + b.h);
if (x2 <= x1 || y2 <= y1) return null;
return { x: x1, y: y1, w: x2 - x1, h: y2 - y1 };
}
function rectInTop(localRect: Rect, context: FrameContext): Rect | null {
const translated = {
x: context.originInTop.x + localRect.x,
y: context.originInTop.y + localRect.y,
w: localRect.w,
h: localRect.h,
};
return intersectRects(translated, context.clipRectInTop);
}
function unclipRectInTop(localRect: Rect, context: FrameContext): Rect {
return {
x: context.originInTop.x + localRect.x,
y: context.originInTop.y + localRect.y,
w: localRect.w,
h: localRect.h,
};
}
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
const out = new Map<number, number>();
if (!sparse?.index || !sparse.value) return out;
for (let i = 0; i < sparse.index.length; i++) {
const nodeIndex = sparse.index[i];
const docIndex = sparse.value[i];
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
}
return out;
}
function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<number>): void {
if (!node) return;
if (typeof node.backendNodeId === "number") {
@@ -235,14 +360,24 @@ function hoverCssScanExpression(): string {
const candidates = [];
const seenLabels = new Set();
const isUnsafeTrigger = (el) => {
if (!(el instanceof HTMLElement)) return true;
const tag = el.tagName.toLowerCase();
if (["input", "textarea", "select", "option"].includes(tag)) return true;
if (el.isContentEditable) return true;
if (el.hasAttribute("disabled") || el.hasAttribute("inert")) return true;
if ((el.getAttribute("aria-disabled") || "").toLowerCase() === "true") return true;
return false;
};
for (const pair of pairs) {
let elements;
try { elements = Array.from(document.querySelectorAll(pair.triggerSel)); } catch { continue; }
for (const el of elements) {
if (!(el instanceof Element)) continue;
if (isUnsafeTrigger(el)) continue;
const rect = el.getBoundingClientRect();
const style = getComputedStyle(el);
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none") continue;
if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue;
const label = (el.textContent || "").replace(/\\s+/g, " ").trim();
if (!label || seenLabels.has(label)) continue;
seenLabels.add(label);
@@ -304,6 +439,20 @@ async function wait(ms: number): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, ms));
}
async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
await cdp
.send(tabId, "Input.dispatchMouseEvent", {
type: "mouseMoved",
x: -10,
y: -10,
})
.catch(() =>
cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 }).catch(() => {
// best effort
}),
);
}
async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<CapturedSurfaceProbe[]> {
const started = Date.now();
try {
@@ -328,7 +477,6 @@ async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<Captur
expression: hoverCollectExpression(candidate),
returnByValue: true,
});
await cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 });
const subItems = (runtimeValue<string[]>(collected) ?? [])
.map((item) => item.replace(/\s+/g, " ").trim())
.filter(Boolean);
@@ -341,6 +489,8 @@ async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise<Captur
});
} catch {
continue;
} finally {
await clearHover(cdp, tabId);
}
}
return results;
@@ -391,15 +541,13 @@ export async function collectOverlayExcludedBackendIds(
* @param doc - the raw DOMSnapshot document object
* @param strings - the shared string table for the whole snapshot
* @param dpr - device-pixel-ratio from Page.getLayoutMetrics
* @param scrollX - horizontal scroll offset to subtract (CSS px)
* @param scrollY - vertical scroll offset to subtract (CSS px)
* @param context - frame coordinate context; output rects are top viewport-relative
*/
function parseDocumentNodes(
doc: SnapshotDocument,
strings: string[],
dpr: number,
scrollX: number,
scrollY: number,
context: FrameContext,
): ParseDocumentResult {
const dn = doc.nodes;
const dl = doc.layout;
@@ -477,6 +625,7 @@ function parseDocumentNodes(
}
let rect: Rect | null = null;
let localRect: Rect | null = null;
let paintOrder = 0;
let position = "static";
let pointerEvents = "auto";
@@ -485,12 +634,13 @@ function parseDocumentNodes(
if (li !== undefined) {
const b = dl?.bounds?.[li];
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
rect = {
x: b[0] / dpr - scrollX,
y: b[1] / dpr - scrollY,
localRect = {
x: b[0] / dpr - context.scrollX,
y: b[1] / dpr - context.scrollY,
w: b[2] / dpr,
h: b[3] / dpr,
};
rect = rectInTop(localRect, context);
}
paintOrder = dl?.paintOrders?.[li] ?? 0;
const styleRow = dl?.styles?.[li] ?? [];
@@ -508,9 +658,11 @@ function parseDocumentNodes(
nodes.push({
backendNodeId,
parentBackendNodeId,
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
tag,
attrs,
rect,
localRect,
paintOrder,
position,
pointerEvents,
@@ -521,7 +673,81 @@ function parseDocumentNodes(
return { nodes, excludedBackendNodeIds };
}
export async function captureViewModel(cdp: CdpRunner, tabId: number): Promise<CapturedViewModel> {
interface ParseFrameDocumentsResult {
iframeNodes: CapturedIframeNodes;
excludedBackendNodeIds: Set<number>;
}
function parseChildFrameDocuments(
documents: SnapshotDocument[],
strings: string[],
dpr: number,
parentDocIndex: number,
parentNodes: CapturedNode[],
parentContext: FrameContext,
visited = new Set<number>(),
): ParseFrameDocumentsResult {
const iframeNodes: CapturedIframeNodes = new Map();
const excludedBackendNodeIds = new Set<number>();
const parentDoc = documents[parentDocIndex];
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
if (cdi.size === 0) return { iframeNodes, excludedBackendNodeIds };
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
for (const [nodeArrayIdx, childDocIndex] of cdi) {
if (visited.has(childDocIndex)) continue;
const childDoc = documents[childDocIndex];
if (!childDoc) continue;
const iframeBackendId = parentBackendIds[nodeArrayIdx];
if (iframeBackendId === undefined) continue;
const iframeNode = parentNodeByBackendId.get(iframeBackendId);
if (!iframeNode?.localRect) continue;
const iframeRectInTop = unclipRectInTop(iframeNode.localRect, parentContext);
const childClip = intersectRects(iframeRectInTop, parentContext.clipRectInTop);
if (!childClip) {
iframeNodes.set(iframeBackendId, []);
continue;
}
const childContext: FrameContext = {
ownerFrameBackendNodeId: iframeBackendId,
originInTop: { x: iframeRectInTop.x, y: iframeRectInTop.y },
clipRectInTop: childClip,
scrollX: childDoc.scrollOffsetX ?? 0,
scrollY: childDoc.scrollOffsetY ?? 0,
};
const nextVisited = new Set(visited);
nextVisited.add(childDocIndex);
const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext);
iframeNodes.set(iframeBackendId, parsed.nodes);
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
const nested = parseChildFrameDocuments(
documents,
strings,
dpr,
childDocIndex,
parsed.nodes,
childContext,
nextVisited,
);
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
iframeNodes.set(nestedFrameId, nestedNodes);
}
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
}
return { iframeNodes, excludedBackendNodeIds };
}
export async function captureViewModel(
cdp: CdpRunner,
tabId: number,
options: CaptureViewModelOptions = {},
): Promise<CapturedViewModel> {
const metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
const dpr = devicePixelRatio(metrics);
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
@@ -552,42 +778,26 @@ export async function captureViewModel(cdp: CdpRunner, tabId: number): Promise<C
};
}
const mainParsed = parseDocumentNodes(doc0, strings, dpr, scrollX, scrollY);
const topContext: FrameContext = {
ownerFrameBackendNodeId: null,
originInTop: { x: 0, y: 0 },
clipRectInTop: viewportRect(viewport),
scrollX,
scrollY,
};
const mainParsed = parseDocumentNodes(doc0, strings, dpr, topContext);
const nodes = mainParsed.nodes;
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
// Build a map from node-array-index → sub-document-index using the sparse
// contentDocumentIndex field. Chrome emits this only for <iframe> nodes.
const iframeNodes: CapturedIframeNodes = new Map();
if (documents.length > 1) {
const cdi = doc0.nodes?.contentDocumentIndex;
if (cdi?.index && cdi.value) {
for (let k = 0; k < cdi.index.length; k++) {
const nodeArrayIdx = cdi.index[k];
const docIdx = cdi.value[k];
const subDoc = documents[docIdx];
if (!subDoc) continue;
const iframeBid = doc0.nodes?.backendNodeId?.[nodeArrayIdx];
if (iframeBid === undefined) continue;
// Sub-documents have their own scroll offsets; bounds are relative
// to the iframe's own coordinate space (not the main viewport).
const subScrollX = subDoc.scrollOffsetX ?? 0;
const subScrollY = subDoc.scrollOffsetY ?? 0;
const subParsed = parseDocumentNodes(subDoc, strings, dpr, subScrollX, subScrollY);
iframeNodes.set(iframeBid, subParsed.nodes);
for (const id of subParsed.excludedBackendNodeIds) {
excludedBackendNodeIds.add(id);
}
}
}
const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext);
const iframeNodes = frameParsed.iframeNodes;
for (const id of frameParsed.excludedBackendNodeIds) {
excludedBackendNodeIds.add(id);
}
await enrichFormControlState(cdp, tabId, nodes);
for (const subNodes of iframeNodes.values()) {
await enrichFormControlState(cdp, tabId, subNodes);
}
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()]);
const surfaceProbes = await probeHoverSurfaces(cdp, tabId);
const surfaceProbes = options.conditionalSurfaceProbe ? await probeHoverSurfaces(cdp, tabId) : [];
return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds };
}