From df5ed1a4956ef861165ef3311c8f13226166aeb1 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Tue, 8 Sep 2026 19:22:46 +0800 Subject: [PATCH 1/5] fix(observation): validate document consistency during capture --- .../__tests__/chromium-cdp.test.ts | 18 +++++ .../__tests__/frame-graph.test.ts | 18 +++++ .../src/browser-driver/chromium-cdp.ts | 10 +++ .../src/browser-driver/frame-graph.ts | 3 + apps/extension/src/tools/shared.ts | 1 + .../src/tools/vom/document-identity.ts | 69 +++++++++++++++++++ apps/extension/src/tools/vom/facts.ts | 9 +++ apps/extension/src/tools/vom/normalize.ts | 7 ++ 8 files changed, 135 insertions(+) create mode 100644 apps/extension/src/tools/vom/document-identity.ts diff --git a/apps/extension/src/browser-driver/__tests__/chromium-cdp.test.ts b/apps/extension/src/browser-driver/__tests__/chromium-cdp.test.ts index fccd4e2..41b6719 100644 --- a/apps/extension/src/browser-driver/__tests__/chromium-cdp.test.ts +++ b/apps/extension/src/browser-driver/__tests__/chromium-cdp.test.ts @@ -29,6 +29,24 @@ function fakeApi() { } describe("ChromiumCdp", () => { + it("changes attachment identity only across real debugger attachments", async () => { + const { api, onEvent, onDetach } = fakeApi(); + const cdp = new ChromiumCdp(api); + expect(cdp.getAttachmentId(4)).toBeUndefined(); + await cdp.ensureAttached(4); + const first = cdp.getAttachmentId(4); + expect(first).toBeTruthy(); + await cdp.ensureAttached(4); + onEvent.fire({ tabId: 4 }, "Target.attachedToTarget", { sessionId: "child" }); + expect(cdp.getAttachmentId(4)).toBe(first); + onDetach.fire({ tabId: 4 }, "target_closed"); + expect(cdp.getAttachmentId(4)).toBeUndefined(); + await cdp.ensureAttached(4); + expect(cdp.getAttachmentId(4)).not.toBe(first); + await cdp.detach(4); + expect(cdp.getAttachmentId(4)).toBeUndefined(); + }); + it("discovers multiple iframe targets and recursively routes nested OOPIF commands", async () => { const { api, onEvent } = fakeApi(); (api.sendCommand as ReturnType).mockImplementation( diff --git a/apps/extension/src/browser-driver/__tests__/frame-graph.test.ts b/apps/extension/src/browser-driver/__tests__/frame-graph.test.ts index b9d4b40..3838d2b 100644 --- a/apps/extension/src/browser-driver/__tests__/frame-graph.test.ts +++ b/apps/extension/src/browser-driver/__tests__/frame-graph.test.ts @@ -2,6 +2,24 @@ import { describe, expect, it } from "vitest"; import { buildFrameGraph, type CdpFrameTreeNode, type CdpFrameTreeSource } from "../frame-graph"; describe("buildFrameGraph", () => { + it("preserves the owning target's loader identity", () => { + const graph = buildFrameGraph([ + { + target: { tabId: 4 }, + tree: { + frame: { id: "root", loaderId: "root-loader" }, + childFrames: [{ frame: { id: "child", loaderId: "stale-placeholder" } }], + }, + }, + { + target: { tabId: 4, sessionId: "child-session" }, + tree: { frame: { id: "child", loaderId: "child-loader" } }, + }, + ]); + expect(graph?.frames.find((frame) => frame.frameId === "root")?.loaderId).toBe("root-loader"); + expect(graph?.frames.find((frame) => frame.frameId === "child")?.loaderId).toBe("child-loader"); + }); + it("keeps sibling frames distinct and routes nested OOPIFs to their child sessions", () => { const graph = buildFrameGraph([ { diff --git a/apps/extension/src/browser-driver/chromium-cdp.ts b/apps/extension/src/browser-driver/chromium-cdp.ts index 8af2abc..ddbf5ee 100644 --- a/apps/extension/src/browser-driver/chromium-cdp.ts +++ b/apps/extension/src/browser-driver/chromium-cdp.ts @@ -164,6 +164,7 @@ async function settleBeforeDeadline(promises: Promise[], deadline: number) export class ChromiumCdp { private readonly api: CdpDebuggerApi; private readonly attachedTabs = new Set(); + private readonly attachmentIds = new Map(); private readonly attachInFlight = new Map>(); private readonly detachInFlight = new Map>(); private readonly tabOwners = new Map>(); @@ -198,6 +199,11 @@ export class ChromiumCdp { this.bindFrameTargetHandler(); } + /** Identity of this actual debugger attachment, independent of frame topology. */ + getAttachmentId(tabId: number): string | undefined { + return this.attachmentIds.get(tabId); + } + /** Attach to `tabId` if we haven't already in this driver. */ async ensureAttached(tabId: number): Promise { // Returning a tab clears the cache before Chrome finishes detaching. @@ -217,6 +223,7 @@ export class ChromiumCdp { await this.enableConsoleDomains(tabId); await this.enableNetworkDomainBestEffort(tabId); this.attachedTabs.add(tabId); + this.attachmentIds.set(tabId, crypto.randomUUID()); await this.enableFrameDiscovery({ tabId }).catch((err) => { console.debug("[bsk cdp] frame discovery unavailable", { tabId, err }); }); @@ -444,6 +451,7 @@ export class ChromiumCdp { this.attachInFlight.delete(tabId); if (!this.attachedTabs.has(tabId)) return; this.attachedTabs.delete(tabId); + this.attachmentIds.delete(tabId); this.clearDialogState(tabId); this.clearConsoleState(tabId); this.clearNetworkState(tabId); @@ -502,6 +510,7 @@ export class ChromiumCdp { this.attachInFlight.clear(); this.tabOwners.clear(); this.attachedTabs.clear(); + this.attachmentIds.clear(); this.dialogBuffers.clear(); this.dialogSequences.clear(); this.consoleBuffers.clear(); @@ -848,6 +857,7 @@ export class ChromiumCdp { const listener = (source: chrome.debugger.Debuggee, _reason: string) => { if (typeof source.tabId === "number") { this.attachedTabs.delete(source.tabId); + this.attachmentIds.delete(source.tabId); this.attachInFlight.delete(source.tabId); this.tabOwners.delete(source.tabId); this.clearDialogState(source.tabId); diff --git a/apps/extension/src/browser-driver/frame-graph.ts b/apps/extension/src/browser-driver/frame-graph.ts index 1dc2ead..2367a5f 100644 --- a/apps/extension/src/browser-driver/frame-graph.ts +++ b/apps/extension/src/browser-driver/frame-graph.ts @@ -8,6 +8,7 @@ export interface CdpFrame { parentFrameId?: string; ownerBackendNodeId?: number; url?: string; + loaderId?: string; target: CdpTarget; } @@ -21,6 +22,7 @@ export interface CdpFrameTreeNode { id: string; parentId?: string; url?: string; + loaderId?: string; }; childFrames?: CdpFrameTreeNode[]; } @@ -48,6 +50,7 @@ function mergeFrame( frameId: node.frame.id, ...(parentFrameId ? { parentFrameId } : {}), ...(url ? { url } : {}), + ...(node.frame.loaderId ? { loaderId: node.frame.loaderId } : {}), target, }); } diff --git a/apps/extension/src/tools/shared.ts b/apps/extension/src/tools/shared.ts index 82aed79..7ae9fda 100644 --- a/apps/extension/src/tools/shared.ts +++ b/apps/extension/src/tools/shared.ts @@ -55,6 +55,7 @@ export interface CdpRunner { sendToTarget?(target: CdpTarget, method: string, params?: object): Promise; detach?(tabId: number): Promise; getFrameGraph?(tabId: number): Promise; + getAttachmentId?(tabId: number): string | undefined; ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise; trackSessionTab?(sessionId: string, tabId: number): void; releaseSessionTab?(sessionId: string, tabId: number): Promise; diff --git a/apps/extension/src/tools/vom/document-identity.ts b/apps/extension/src/tools/vom/document-identity.ts new file mode 100644 index 0000000..365c2e5 --- /dev/null +++ b/apps/extension/src/tools/vom/document-identity.ts @@ -0,0 +1,69 @@ +import type { CdpFrame } from "@/browser-driver/frame-graph"; +import type { CdpRunner } from "../shared"; +import { sendToCdpTarget } from "../shared"; +import { + isAbortError as isCaptureAbort, + throwIfAborted as throwCaptureAborted, +} from "./capture-abort"; +import type { DocumentIdentity } from "./facts"; + +/** Fresh, frame-scoped identity read. An isolated world supplies the owning + * document without querying each node or installing a page observer. */ +export async function readDocumentIdentity( + cdp: CdpRunner, + frame: CdpFrame, + signal?: AbortSignal, +): Promise { + const attachmentId = cdp.getAttachmentId?.(frame.target.tabId); + if (!attachmentId || !frame.loaderId) return undefined; + let objectId: string | undefined; + const send = (method: string, params: object) => { + throwCaptureAborted(signal); + return sendToCdpTarget(cdp, frame.target, method, params); + }; + try { + const world = await send<{ executionContextId: number }>("Page.createIsolatedWorld", { + frameId: frame.frameId, + worldName: "bsk-document-identity", + }); + const element = await send<{ result?: { objectId?: string } }>("Runtime.evaluate", { + expression: "document.documentElement", + contextId: world.executionContextId, + }); + objectId = element.result?.objectId; + if (!objectId) return undefined; + const reply = await send<{ node?: { backendNodeId?: number } }>("DOM.describeNode", { + objectId, + depth: 0, + }); + const backendNodeId = reply.node?.backendNodeId; + if (!backendNodeId || cdp.getAttachmentId?.(frame.target.tabId) !== attachmentId) + return undefined; + return { + attachmentId, + target: frame.target, + frameId: frame.frameId, + loaderId: frame.loaderId, + documentElementBackendNodeId: backendNodeId, + }; + } catch (error) { + if (isCaptureAbort(error)) throw error; + return undefined; + } finally { + if (objectId) + await sendToCdpTarget(cdp, frame.target, "Runtime.releaseObject", { objectId }).catch( + () => {}, + ); + } +} + +export function sameDocument(a: DocumentIdentity, b: DocumentIdentity): boolean { + return ( + a.attachmentId === b.attachmentId && + a.target.tabId === b.target.tabId && + a.target.sessionId === b.target.sessionId && + a.frameId === b.frameId && + a.loaderId === b.loaderId && + a.documentElementBackendNodeId === b.documentElementBackendNodeId + ); +} diff --git a/apps/extension/src/tools/vom/facts.ts b/apps/extension/src/tools/vom/facts.ts index e257d01..59be289 100644 --- a/apps/extension/src/tools/vom/facts.ts +++ b/apps/extension/src/tools/vom/facts.ts @@ -1,3 +1,4 @@ +import type { CdpTarget } from "@/browser-driver/frame-graph"; import { isOverlayHostNode } from "@/lib/overlay-bridge"; import { createCaptureCheckpoint } from "./capture-abort"; import type { CapturedNode } from "./capture-types"; @@ -92,3 +93,11 @@ export async function buildDocumentIndex( } return { nodes, excludedBackendNodeIds }; } + +export interface DocumentIdentity { + attachmentId: string; + target: CdpTarget; + frameId: string; + loaderId: string; + documentElementBackendNodeId: number; +} diff --git a/apps/extension/src/tools/vom/normalize.ts b/apps/extension/src/tools/vom/normalize.ts index 66bbc4d..d785ee4 100644 --- a/apps/extension/src/tools/vom/normalize.ts +++ b/apps/extension/src/tools/vom/normalize.ts @@ -21,6 +21,7 @@ export interface FrameContext { export interface NormalizedDocument { nodes: NodeFacts[]; index: DocumentIndex; + documentElementBackendNodeId?: number; } /** Interpret one document using a supplied projection. No live reads or frame scheduling. */ @@ -74,5 +75,11 @@ export async function normalizeDocument( (node) => !node.tag.startsWith("#") && !index.excludedBackendNodeIds.has(node.backendNodeId), ), index, + documentElementBackendNodeId: nodes.find( + (node) => + node.nodeType === 1 && + node.parentBackendNodeId !== null && + index.nodes.get(node.parentBackendNodeId)?.nodeType === 9, + )?.backendNodeId, }; } From 4e3365450a4f12c3159c0b405dbe140c24dbca5c Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Wed, 9 Sep 2026 00:53:45 +0800 Subject: [PATCH 2/5] refactor(observation): integrate capture orchestration and document consistency --- .../src/tools/__tests__/observation.test.ts | 459 ++++--- apps/extension/src/tools/observation.ts | 235 +--- .../vom/__tests__/capture-coordinator.test.ts | 1119 +++++++++++++++++ .../src/tools/vom/__tests__/capture.test.ts | 416 ++---- .../tools/vom/__tests__/frame-capture.test.ts | 289 ----- .../vom/__tests__/frame-document.test.ts | 41 +- .../src/tools/vom/capture-coordinator.ts | 443 +++++++ apps/extension/src/tools/vom/capture.ts | 361 +----- .../src/tools/vom/document-identity.ts | 2 + apps/extension/src/tools/vom/facts.ts | 54 +- apps/extension/src/tools/vom/frame-capture.ts | 183 --- .../extension/src/tools/vom/frame-document.ts | 37 +- apps/extension/src/tools/vom/normalize.ts | 222 +++- apps/extension/src/tools/vom/snapshot.ts | 67 + 14 files changed, 2323 insertions(+), 1605 deletions(-) create mode 100644 apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts delete mode 100644 apps/extension/src/tools/vom/__tests__/frame-capture.test.ts create mode 100644 apps/extension/src/tools/vom/capture-coordinator.ts delete mode 100644 apps/extension/src/tools/vom/frame-capture.ts diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index fcd61cc..3e5e935 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -4,9 +4,9 @@ import { CAPTURE_SUPPRESS, type CaptureSuppressMessage } from "@/lib/capture-sup import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "@/lib/overlay-bridge"; import { SessionManager } from "@/session-manager/manager"; import type { CdpRunner } from "@/tools/shared"; +import type { BuildVomSceneOptions, VomFrameDocument } from "../observation"; import { buildFrameVomScene, - buildVomScene, type CdpAxNode, captureVomObservation, handleGetHtml, @@ -17,7 +17,35 @@ import { type ScreenshotDeps, stripDataUrlPrefix, } from "../observation"; -import type { CapturedNode, CapturedViewModel } from "../vom/capture"; +import type { CapturedNode, CapturedSceneInput } from "../vom/facts"; + +interface SceneFixture extends CapturedSceneInput { + documents?: VomFrameDocument[]; +} + +// Only supplies the common root fixture; child documents declare their own ownership. +function buildTestScene( + axNodes: CdpAxNode[], + captured: SceneFixture, + options: BuildVomSceneOptions = {}, +) { + const rootFrameId = captured.rootFrameId ?? "root"; + return buildFrameVomScene( + [ + { + frameId: rootFrameId, + contextScopeId: rootFrameId, + target: { tabId: 7 }, + url: options.pageUrl, + domNodes: captured.nodes, + axNodes, + }, + ...(captured.documents ?? []), + ], + captured, + options, + ); +} function fakeAgentWindow(ids: number[]) { let i = 0; @@ -526,10 +554,10 @@ describe("handleScreenshot overlay suppression", () => { }); // --------------------------------------------------------------------------- -// buildVomScene +// Semantic scene fixtures // --------------------------------------------------------------------------- -describe("buildVomScene", () => { +describe("buildFrameVomScene", () => { it("merges multiple sibling and nested frames without mixing their targets", () => { const node = ( backendNodeId: number, @@ -567,25 +595,10 @@ describe("buildVomScene", () => { node(300, null, "nested", "body"), node(301, 300, "nested", "input", { placeholder: "Nested field" }), ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1200, height: 800 }, nodes: mainNodes, - iframeNodes: new Map([ - [10, firstNodes], - [20, secondNodes], - [210, nestedNodes], - ]), - frameNodes: new Map([ - ["main", mainNodes], - ["first", firstNodes], - ["second", secondNodes], - ["nested", nestedNodes], - ]), - frameOwnerBackendNodeIds: new Map([ - ["first", 10], - ["second", 20], - ["nested", 210], - ]), + rootFrameId: "main", excludedBackendNodeIds: new Set(), }; @@ -716,7 +729,6 @@ describe("buildVomScene", () => { { nodes: owners, viewport: { width: 800, height: 600 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), }, ); @@ -738,11 +750,10 @@ describe("buildVomScene", () => { position: "static", pointerEvents: "auto", }; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 800, height: 600 }, nodes: [], - iframeNodes: new Map(), - frameNodes: new Map([["child", [childNode]]]), + rootFrameId: "main", excludedBackendNodeIds: new Set(), }; @@ -804,14 +815,10 @@ describe("buildVomScene", () => { pointerEvents: "auto", }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 800, height: 600 }, nodes: mainNodes, - iframeNodes: new Map(), - frameNodes: new Map([ - ["main", mainNodes], - ["child", []], - ]), + rootFrameId: "main", excludedBackendNodeIds: new Set(), }; @@ -887,9 +894,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 200, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -915,11 +921,11 @@ describe("buildVomScene", () => { ], }; - const scene = buildVomScene(axNodes, captured); + const scene = buildTestScene(axNodes, captured); expect(scene.viewport).toEqual({ width: 1000, height: 800 }); expect(scene.nodes[0]).toEqual( expect.objectContaining({ - id: 100, + backendNodeId: 100, parentId: null, role: "RootWebArea", name: "Example", @@ -931,14 +937,14 @@ describe("buildVomScene", () => { ); expect(scene.nodes[1]).toEqual( expect.objectContaining({ - id: 200, - parentId: 100, + backendNodeId: 200, + parentId: scene.nodes.find((node) => node.backendNodeId === 100)!.id, role: "button", name: "Submit", tag: "button", rect: { x: 20, y: 20, w: 120, h: 40 }, - domParentId: 100, - domAncestorIds: [100], + domParentId: scene.nodes.find((node) => node.backendNodeId === 100)!.id, + domAncestorIds: [scene.nodes.find((node) => node.backendNodeId === 100)!.id], attrs: {}, }), ); @@ -967,9 +973,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 300, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1005,10 +1010,10 @@ describe("buildVomScene", () => { ], }; - const scene = buildVomScene(axNodes, captured, { pageUrl: "https://app.example.test/home" }); + const scene = buildTestScene(axNodes, captured, { pageUrl: "https://app.example.test/home" }); - expect(scene.nodes.find((node) => node.id === 200)?.href).toBe("docs.example.org"); - expect(scene.nodes.find((node) => node.id === 300)?.href).toBeUndefined(); + expect(scene.nodes.find((node) => node.backendNodeId === 200)?.href).toBe("docs.example.org"); + expect(scene.nodes.find((node) => node.backendNodeId === 300)?.href).toBeUndefined(); }); it("uses the nearest backend AX ancestor as parentId", () => { @@ -1033,9 +1038,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 30, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 400, height: 300 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1061,10 +1065,12 @@ describe("buildVomScene", () => { ], }; - const scene = buildVomScene(axNodes, captured); + const scene = buildTestScene(axNodes, captured); - expect(scene.nodes.map((node) => node.id)).toEqual([10, 30]); - expect(scene.nodes.find((node) => node.id === 30)?.parentId).toBe(10); + expect(scene.nodes.map((node) => node.backendNodeId)).toEqual([10, 30]); + expect(scene.nodes.find((node) => node.backendNodeId === 30)?.parentId).toBe( + scene.nodes.find((node) => node.backendNodeId === 10)!.id, + ); }); it("maps iframe sub-document controls to VomNodes", () => { @@ -1082,7 +1088,7 @@ describe("buildVomScene", () => { backendDOMNodeId: 20, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, nodes: [ { @@ -1106,10 +1112,15 @@ describe("buildVomScene", () => { pointerEvents: "auto", }, ], - iframeNodes: new Map([ - [ - 20, - [ + documents: [ + { + frameId: "child", + contextScopeId: "child", + parentFrameId: "root", + ownerBackendNodeId: 20, + target: { tabId: 7 }, + axNodes: [], + domNodes: [ { backendNodeId: 101, parentBackendNodeId: null, @@ -1131,26 +1142,25 @@ describe("buildVomScene", () => { pointerEvents: "auto", }, ], - ], - ]), + }, + ], excludedBackendNodeIds: new Set(), }; - const iframeControls = buildVomScene(axNodes, captured).nodes.filter((node) => - [101, 102].includes(node.id), - ); + const scene = buildTestScene(axNodes, captured); + const iframeControls = scene.nodes.filter((node) => [101, 102].includes(node.backendNodeId!)); expect(iframeControls).toEqual([ expect.objectContaining({ - id: 101, - parentId: 20, + backendNodeId: 101, + parentId: scene.nodes.find((node) => node.backendNodeId === 20)!.id, role: "textbox", name: "请输入手机号", sensitive: false, }), expect.objectContaining({ - id: 102, - parentId: 20, + backendNodeId: 102, + parentId: scene.nodes.find((node) => node.backendNodeId === 20)!.id, role: "textbox", name: "密码", sensitive: true, @@ -1171,11 +1181,9 @@ describe("buildVomScene", () => { parentId: "1", role: { type: "role", value: "Iframe" }, backendDOMNodeId: 20, - childIds: ["3"], }, { nodeId: "3", - parentId: "2", role: { type: "role", value: "textbox" }, name: { type: "x", value: "输入密码" }, value: { value: "iframe-secret" }, @@ -1183,7 +1191,7 @@ describe("buildVomScene", () => { properties: [{ name: "inputType", value: { value: "password" } }], }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, nodes: [ { @@ -1207,10 +1215,15 @@ describe("buildVomScene", () => { pointerEvents: "auto", }, ], - iframeNodes: new Map([ - [ - 20, - [ + documents: [ + { + frameId: "child", + contextScopeId: "child", + parentFrameId: "root", + ownerBackendNodeId: 20, + target: { tabId: 7 }, + axNodes: [axNodes[2]], + domNodes: [ { backendNodeId: 102, parentBackendNodeId: null, @@ -1222,25 +1235,27 @@ describe("buildVomScene", () => { pointerEvents: "auto", }, ], - ], - ]), + }, + ], excludedBackendNodeIds: new Set(), }; - const passwordNodes = buildVomScene(axNodes, captured).nodes.filter( + const passwordNodes = buildTestScene(axNodes.slice(0, 2), captured).nodes.filter( (node) => node.role === "textbox" && node.name === "输入密码", ); expect(passwordNodes).toHaveLength(1); - expect(passwordNodes[0]).toEqual(expect.objectContaining({ id: 202, sensitive: true })); + expect(passwordNodes[0]).toEqual( + expect.objectContaining({ backendNodeId: 202, sensitive: true }), + ); expect(passwordNodes[0].value).toBeUndefined(); - const rendered = renderVom(buildVomScene(axNodes, captured)).text; + const rendered = renderVom(buildTestScene(axNodes.slice(0, 2), captured)).text; expect(rendered).toContain('textbox "输入密码" [filled] ="•••"'); expect(rendered).not.toContain("iframe-secret"); }); it("keeps unnamed iframe controls but skips unnamed iframe links", () => { - const scene = buildVomScene( + const scene = buildTestScene( [ { nodeId: "1", @@ -1279,10 +1294,15 @@ describe("buildVomScene", () => { pointerEvents: "auto", }, ], - iframeNodes: new Map([ - [ - 20, - [ + documents: [ + { + frameId: "child", + contextScopeId: "child", + parentFrameId: "root", + ownerBackendNodeId: 20, + target: { tabId: 7 }, + axNodes: [], + domNodes: [ { backendNodeId: 201, parentBackendNodeId: null, @@ -1304,25 +1324,25 @@ describe("buildVomScene", () => { pointerEvents: "auto", }, ], - ], - ]), + }, + ], excludedBackendNodeIds: new Set(), }, ); - expect(scene.nodes.find((node) => node.id === 201)).toEqual( + expect(scene.nodes.find((node) => node.backendNodeId === 201)).toEqual( expect.objectContaining({ - id: 201, - parentId: 20, + backendNodeId: 201, + parentId: scene.nodes.find((node) => node.backendNodeId === 20)!.id, role: "button", }), ); - expect(scene.nodes.find((node) => node.id === 201)).not.toHaveProperty("name"); - expect(scene.nodes.find((node) => node.id === 202)).toBeUndefined(); + expect(scene.nodes.find((node) => node.backendNodeId === 201)).not.toHaveProperty("name"); + expect(scene.nodes.find((node) => node.backendNodeId === 202)).toBeUndefined(); }); it("preserves captured-only iframe anchors for iframe sub-document controls", () => { - const scene = buildVomScene( + const scene = buildTestScene( [ { nodeId: "1", @@ -1355,10 +1375,15 @@ describe("buildVomScene", () => { pointerEvents: "auto", }, ], - iframeNodes: new Map([ - [ - 20, - [ + documents: [ + { + frameId: "child", + contextScopeId: "child", + parentFrameId: "root", + ownerBackendNodeId: 20, + target: { tabId: 7 }, + axNodes: [], + domNodes: [ { backendNodeId: 201, parentBackendNodeId: null, @@ -1380,16 +1405,16 @@ describe("buildVomScene", () => { pointerEvents: "auto", }, ], - ], - ]), + }, + ], excludedBackendNodeIds: new Set(), }, ); - expect(scene.nodes.find((node) => node.id === 20)).toEqual( + expect(scene.nodes.find((node) => node.backendNodeId === 20)).toEqual( expect.objectContaining({ - id: 20, - parentId: 10, + backendNodeId: 20, + parentId: scene.nodes.find((node) => node.backendNodeId === 10)!.id, role: "Iframe", name: "支付验证", }), @@ -1410,9 +1435,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 10, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 320, height: 240 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1433,9 +1457,9 @@ describe("buildVomScene", () => { ], }; - expect(buildVomScene(axNodes, captured).nodes[0]).toEqual( + expect(buildTestScene(axNodes, captured).nodes[0]).toEqual( expect.objectContaining({ - id: 10, + backendNodeId: 10, sensitive: true, }), ); @@ -1450,9 +1474,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 10, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1469,9 +1492,9 @@ describe("buildVomScene", () => { ], }; - expect(buildVomScene(axNodes, captured).nodes[0]).toEqual( + expect(buildTestScene(axNodes, captured).nodes[0]).toEqual( expect.objectContaining({ - id: 10, + backendNodeId: 10, role: "textbox", name: "+86", placeholder: "输入手机号", @@ -1489,9 +1512,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 10, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1508,8 +1530,13 @@ describe("buildVomScene", () => { ], }; - expect(buildVomScene(axNodes, captured).nodes[0]).toEqual( - expect.objectContaining({ id: 10, role: "textbox", name: "Email", value: "a@b.com" }), + expect(buildTestScene(axNodes, captured).nodes[0]).toEqual( + expect.objectContaining({ + backendNodeId: 10, + role: "textbox", + name: "Email", + value: "a@b.com", + }), ); }); @@ -1528,9 +1555,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 20, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1568,9 +1594,9 @@ describe("buildVomScene", () => { ], }; - expect(buildVomScene(axNodes, captured).nodes.find((node) => node.id === 20)).toEqual( - expect.objectContaining({ role: "textbox", name: "验证码" }), - ); + expect( + buildTestScene(axNodes, captured).nodes.find((node) => node.backendNodeId === 20), + ).toEqual(expect.objectContaining({ role: "textbox", name: "验证码" })); }); it("uses preceding AX static text as a form control label", () => { @@ -1594,9 +1620,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 20, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1613,13 +1638,13 @@ describe("buildVomScene", () => { ], }; - expect(buildVomScene(axNodes, captured).nodes.find((node) => node.id === 20)).toEqual( - expect.objectContaining({ role: "textbox", name: "验证码", inputState: "empty" }), - ); + expect( + buildTestScene(axNodes, captured).nodes.find((node) => node.backendNodeId === 20), + ).toEqual(expect.objectContaining({ role: "textbox", name: "验证码", inputState: "empty" })); }); it("marks runtime default input values without treating them as ordinary filled input", () => { - const scene = buildVomScene( + const scene = buildTestScene( [ { nodeId: "1", @@ -1629,7 +1654,6 @@ describe("buildVomScene", () => { ], { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1673,9 +1697,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 20, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1725,9 +1748,14 @@ describe("buildVomScene", () => { ], }; - const scene = buildVomScene(axNodes, captured); - expect(scene.nodes.find((n) => n.id === 20)).toEqual( - expect.objectContaining({ id: 20, role: "button", name: "close", cursor: "pointer" }), + const scene = buildTestScene(axNodes, captured); + expect(scene.nodes.find((n) => n.backendNodeId === 20)).toEqual( + expect.objectContaining({ + backendNodeId: 20, + role: "button", + name: "close", + cursor: "pointer", + }), ); const rendered = renderVom(scene); expect(rendered.text).toContain('@e1 button "close"'); @@ -1761,9 +1789,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 30, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1802,9 +1829,9 @@ describe("buildVomScene", () => { ], }; - const scene = buildVomScene(axNodes, captured); - expect(scene.nodes.find((n) => n.id === 20)).toBeUndefined(); - expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("link"); + const scene = buildTestScene(axNodes, captured); + expect(scene.nodes.find((n) => n.backendNodeId === 20)).toBeUndefined(); + expect(scene.nodes.find((n) => n.backendNodeId === 30)?.role).toBe("link"); }); it("promotes only the outermost clickable in a nested pointer chain", () => { @@ -1829,9 +1856,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 30, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1870,11 +1896,15 @@ describe("buildVomScene", () => { ], }; - const scene = buildVomScene(axNodes, captured); - expect(scene.nodes.find((n) => n.id === 20)).toEqual( - expect.objectContaining({ id: 20, role: "button", attrs: { "aria-label": "收藏" } }), + const scene = buildTestScene(axNodes, captured); + expect(scene.nodes.find((n) => n.backendNodeId === 20)).toEqual( + expect.objectContaining({ + backendNodeId: 20, + role: "button", + attrs: { "aria-label": "收藏" }, + }), ); - expect(scene.nodes.find((n) => n.id === 30)).toBeUndefined(); + expect(scene.nodes.find((n) => n.backendNodeId === 30)).toBeUndefined(); const rendered = renderVom(scene); expect(rendered.text).toContain('@e1 button "收藏"'); expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([ @@ -1898,9 +1928,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 20, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1948,10 +1977,10 @@ describe("buildVomScene", () => { ], }; - const scene = buildVomScene(axNodes, captured); + const scene = buildTestScene(axNodes, captured); expect(scene.activeScopeBlocks).toEqual([ { - triggerId: 20, + triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id, label: "Reviews (12)", lines: ["Jane - ear cups are small", "Bob - great sound"], }, @@ -1975,11 +2004,10 @@ describe("buildVomScene", () => { backendDOMNodeId: 20, }, ]; - const scene = buildVomScene( + const scene = buildTestScene( axNodes, { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -2012,7 +2040,11 @@ describe("buildVomScene", () => { ); expect(scene.surfaces).toEqual([ - { triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] }, + { + triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id, + triggerAction: "hover", + subItems: ["Shoes", "Bags"], + }, ]); expect(renderVom(scene).text).toContain('@e1 button "Products" [hover first: Shoes | Bags]'); }); @@ -2033,11 +2065,10 @@ describe("buildVomScene", () => { backendDOMNodeId: 21, }, ]; - const scene = buildVomScene( + const scene = buildTestScene( axNodes, { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -2098,11 +2129,10 @@ describe("buildVomScene", () => { backendDOMNodeId: 21, }, ]; - const scene = buildVomScene( + const scene = buildTestScene( axNodes, { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -2147,7 +2177,11 @@ describe("buildVomScene", () => { ); expect(scene.surfaces).toEqual([ - { triggerId: 21, triggerAction: "hover", subItems: ["My profile"] }, + { + triggerId: scene.nodes.find((node) => node.backendNodeId === 21)!.id, + triggerAction: "hover", + subItems: ["My profile"], + }, ]); }); @@ -2167,11 +2201,10 @@ describe("buildVomScene", () => { backendDOMNodeId: 21, }, ]; - const scene = buildVomScene( + const scene = buildTestScene( axNodes, { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -2222,7 +2255,11 @@ describe("buildVomScene", () => { ); expect(scene.surfaces).toEqual([ - { triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] }, + { + triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id, + triggerAction: "hover", + subItems: ["My profile", "Sign out"], + }, ]); expect(renderVom(scene).text).toContain( '@e1 button "image" [hover first: My profile | Sign out]', @@ -2245,11 +2282,10 @@ describe("buildVomScene", () => { backendDOMNodeId: 42, }, ]; - const scene = buildVomScene( + const scene = buildTestScene( axNodes, { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -2319,9 +2355,8 @@ describe("buildVomScene", () => { properties: [{ name: "inputType", value: { value: "credit-card" } }], }, ]; - const scene = buildVomScene(axNodes, { + const scene = buildTestScene(axNodes, { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -2368,7 +2403,7 @@ describe("buildVomScene", () => { ], }); - expect(scene.nodes.find((node) => node.id === 20)).toEqual( + expect(scene.nodes.find((node) => node.backendNodeId === 20)).toEqual( expect.objectContaining({ name: "Products [expanded]", attrs: expect.objectContaining({ @@ -2378,13 +2413,17 @@ describe("buildVomScene", () => { }), ); expect(scene.activeScopeBlocks).toEqual([ - { triggerId: 20, label: "Products [expanded]", lines: ["Shoes Bags"] }, + { + triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id, + label: "Products [expanded]", + lines: ["Shoes Bags"], + }, ]); - expect(scene.nodes.find((node) => node.id === 30)?.sensitive).toBe(true); + expect(scene.nodes.find((node) => node.backendNodeId === 30)?.sensitive).toBe(true); }); it("aggregates AX virtual text into unnamed structural nodes", () => { - const scene = buildVomScene( + const scene = buildTestScene( [ { nodeId: "1", @@ -2408,7 +2447,6 @@ describe("buildVomScene", () => { ], { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -2435,13 +2473,12 @@ describe("buildVomScene", () => { }, ); - expect(scene.nodes.find((node) => node.id === 20)?.name).toBe("Inline only text"); + expect(scene.nodes.find((node) => node.backendNodeId === 20)?.name).toBe("Inline only text"); }); it("marks captured dialog elements as modal without AX role or aria-modal", () => { - const scene = buildVomScene([], { + const scene = buildTestScene([], { viewport: { width: 640, height: 480 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -2459,7 +2496,7 @@ describe("buildVomScene", () => { expect(scene.nodes[0]).toEqual( expect.objectContaining({ - id: 50, + backendNodeId: 50, modal: true, }), ); @@ -2488,9 +2525,8 @@ describe("buildVomScene", () => { backendDOMNodeId: 202, }, ]; - const captured: CapturedViewModel = { + const captured: SceneFixture = { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set([200, 201, 202]), nodes: [ { @@ -2506,9 +2542,9 @@ describe("buildVomScene", () => { ], }; - const scene = buildVomScene(axNodes, captured); - expect(scene.nodes.find((n) => n.id === 202)).toBeUndefined(); - expect(scene.nodes.find((n) => n.id === 2)).toEqual( + const scene = buildTestScene(axNodes, captured); + expect(scene.nodes.find((n) => n.backendNodeId === 202)).toBeUndefined(); + expect(scene.nodes.find((n) => n.backendNodeId === 2)).toEqual( expect.objectContaining({ role: "button", name: "Real Button" }), ); }); @@ -2536,13 +2572,12 @@ describe("buildVomScene", () => { backendDOMNodeId: 203, }, ]; - const scene = buildVomScene(axNodes, { + const scene = buildTestScene(axNodes, { viewport: { width: 1000, height: 800 }, - iframeNodes: new Map(), excludedBackendNodeIds: new Set([200]), nodes: [], }); - expect(scene.nodes.find((n) => n.id === 203)).toBeUndefined(); + expect(scene.nodes.find((n) => n.backendNodeId === 203)).toBeUndefined(); }); }); @@ -2655,6 +2690,7 @@ describe("handleSnapshot", () => { strings, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0], nodeName: [i("body"), i("button")], @@ -2750,6 +2786,7 @@ describe("handleSnapshot", () => { strings, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0], nodeName: [i("body"), i("button")], @@ -2848,6 +2885,7 @@ describe("handleSnapshot", () => { strings, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0], nodeName: [i("body"), i("button")], @@ -3116,6 +3154,7 @@ describe("handleSnapshot", () => { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1, 2], nodeName: [i("html"), i("body"), i("div"), i("input")], @@ -3173,6 +3212,7 @@ describe("handleSnapshot", () => { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1, 1, 1, 1], nodeName: [i("html"), i("body"), i("input"), i("input"), i("input"), i("input")], @@ -3324,12 +3364,17 @@ describe("handleSnapshot", () => { if (method === "Runtime.releaseObject") return {}; } if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {}; - if (method === "DOMSnapshot.captureSnapshot") return snapshot; + if (method === "DOMSnapshot.captureSnapshot") + return { ...snapshot, documents: [snapshot.documents[0]] }; if (method === "Accessibility.getFullAXTree") return { nodes: mainAx }; throw new Error(`unexpected root CDP method ${method}`); }); const sendToTarget = vi.fn(async (_target, method: string) => { - if (method === "Accessibility.enable") return {}; + if (method === "Page.getLayoutMetrics" && ownerGeometry !== "unavailable") + return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } }; + if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") + return { ...snapshot, documents: [snapshot.documents[1]] }; if (method === "Accessibility.getFullAXTree") return { nodes: childAx }; throw new Error(`unexpected child CDP method ${method}`); }); @@ -3361,6 +3406,66 @@ describe("handleSnapshot", () => { }; } + it.each([ + false, + true, + ])("preserves external-link hints without graph metadata (snapshot URL: %s)", async (hasSnapshotUrl) => { + const snapshot = { + strings: ["body", "a", "href", "//outside.test/docs", "https://page.test/start"], + documents: [ + { + frameId: "main", + ...(hasSnapshotUrl ? { documentURL: 4 } : {}), + nodes: { + backendNodeId: [1, 2], + parentIndex: [-1, 0], + nodeName: [0, 1], + attributes: [[], [2, 3]], + }, + layout: { + nodeIndex: [0, 1], + bounds: [ + [0, 0, 1000, 800], + [10, 20, 100, 40], + ], + }, + }, + ], + }; + const ax: CdpAxNode[] = [ + { + nodeId: "root", + backendDOMNodeId: 1, + role: { type: "role", value: "RootWebArea" }, + childIds: ["link"], + }, + { + nodeId: "link", + parentId: "root", + backendDOMNodeId: 2, + role: { type: "role", value: "link" }, + name: { type: "computedString", value: "Docs" }, + }, + ]; + const deps = makeOverlayDeps(ax, snapshot, VP_METRICS); + const cdp: CdpRunner = { + ...deps.cdp, + getFrameGraph: async () => { + throw new Error("graph unavailable"); + }, + }; + const result = await captureVomObservation( + cdp, + 4, + hasSnapshotUrl ? undefined : "https://page.test/start", + ); + expect(result.text).toContain("outside.test"); + expect(result.text).toContain("Docs"); + expect( + deps.send.mock.calls.filter(([, method]) => method === "DOMSnapshot.captureSnapshot"), + ).toHaveLength(1); + }); + it("exposes the frame-aware observation result for recording", async () => { const ax: CdpAxNode[] = [ { @@ -3430,6 +3535,7 @@ describe("handleSnapshot", () => { strings, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1, 1, 3], nodeName: [index("html"), index("body"), index("button"), index("div"), index("#text")], @@ -3754,6 +3860,7 @@ describe("handleSnapshot", () => { strings: S, documents: [ { + frameId: "root", scrollOffsetX: 0, scrollOffsetY: 0, nodes: { @@ -3780,6 +3887,7 @@ describe("handleSnapshot", () => { }, // documents[1]: cross-origin login form { + frameId: "child", scrollOffsetX: 0, scrollOffsetY: 0, nodes: { @@ -3958,6 +4066,7 @@ describe("handleSnapshot", () => { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1, 0, 3, 4, 5], nodeName: [ diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index ea1f0a1..00e6c42 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -1,3 +1,4 @@ +import type { CapturedSceneInput } from "./vom/facts"; // Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`, // and semantic `tool.observe` (design §7). Each handler resolves the target // tab (defaulting to the Agent Window's active tab when omitted) and @@ -35,7 +36,6 @@ import { attachDialogs, markDialogCursor } from "./dialogs"; import { rpcError } from "./errors"; import { resolveNodeGeometry } from "./frame-geometry"; import { screenshotPageRect } from "./geometry/coordinate-types"; -import { cssViewport, GeometryContext } from "./geometry/frame-context"; import { type ChromeTabsApi, enforceToolTargetScope, @@ -49,15 +49,9 @@ import { type ToolEffect, } from "./shared"; import { resolveSnapshotRef } from "./snapshot-ref"; -import { - type CapturedNode, - type CapturedSurfaceProbe, - type CapturedViewModel, - captureViewModel, - collectOverlayExcludedBackendIds, - probeHoverSurfaces, -} from "./vom/capture"; -import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture"; +import { type CapturedNode, type CapturedSurfaceProbe, probeHoverSurfaces } from "./vom/capture"; +import { captureObservationFacts, semanticCapture } from "./vom/capture-coordinator"; +import type { FrameDocument as CapturedFrameDocument } from "./vom/frame-document"; import { withOverlayBypass } from "./vom/hover-perception"; import { probeTooltipNames } from "./vom/name-enrichment"; import { @@ -531,7 +525,7 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A function buildConditionalSurfaces( nodes: VomNode[], - captured: CapturedViewModel, + captured: CapturedSceneInput, probes: CapturedSurfaceProbe[], ): CondSurface[] { if (probes.length === 0) return []; @@ -661,163 +655,9 @@ export interface BuildVomSceneOptions { export type VomFrameDocument = CapturedFrameDocument; -function legacyFrameDocuments( - axNodes: CdpAxNode[], - captured: CapturedViewModel, - pageUrl?: string, - rootTarget: VomFrameDocument["target"] = { tabId: 0 }, -): VomFrameDocument[] { - const rootFrameId = captured.rootFrameId ?? "root"; - const documents: VomFrameDocument[] = [ - { - frameId: rootFrameId, - contextScopeId: rootFrameId, - target: rootTarget, - ...(pageUrl ? { url: pageUrl } : {}), - axNodes: [], - domNodes: captured.nodes.map((node) => ({ ...node, frameId: node.frameId ?? rootFrameId })), - excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(rootFrameId), - }, - ]; - const pending = [...captured.iframeNodes.entries()]; - let progress = true; - let nextSyntheticFrame = 1; - while (pending.length > 0 && progress) { - progress = false; - for (let index = pending.length - 1; index >= 0; index -= 1) { - const [ownerBackendNodeId, domNodes] = pending[index]; - const parent = documents.find((document) => - document.domNodes.some((node) => node.backendNodeId === ownerBackendNodeId), - ); - if (!parent) continue; - const frameId = - domNodes.find((node) => node.frameId)?.frameId ?? `legacy-frame-${nextSyntheticFrame++}`; - documents.push({ - frameId, - parentFrameId: parent.frameId, - ownerBackendNodeId, - contextScopeId: frameId, - target: parent.target, - axNodes: [], - domNodes: domNodes.map((node) => ({ ...node, frameId: node.frameId ?? frameId })), - excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(frameId), - }); - pending.splice(index, 1); - progress = true; - } - } - - const documentByFrameId = new Map(documents.map((document) => [document.frameId, document])); - const backendFrame = new Map(); - const childFrameByOwner = new Map(); - for (const document of documents) { - if (document.ownerBackendNodeId !== undefined) { - childFrameByOwner.set(document.ownerBackendNodeId, document.frameId); - } - for (const node of document.domNodes) backendFrame.set(node.backendNodeId, document.frameId); - } - const axById = new Map(axNodes.map((node) => [node.nodeId, node])); - const ownership = new Map(); - const resolving = new Set(); - const frameForAx = (node: CdpAxNode): string => { - const cached = ownership.get(node.nodeId); - if (cached) return cached; - let frameId: string | undefined; - if (node.frameId && documentByFrameId.has(node.frameId)) frameId = node.frameId; - if (!frameId && typeof node.backendDOMNodeId === "number") { - frameId = backendFrame.get(node.backendDOMNodeId); - } - if (!frameId && node.parentId && !resolving.has(node.nodeId)) { - const parent = axById.get(node.parentId); - if (parent) { - frameId = - typeof parent.backendDOMNodeId === "number" - ? childFrameByOwner.get(parent.backendDOMNodeId) - : undefined; - if (!frameId) { - resolving.add(node.nodeId); - frameId = frameForAx(parent); - resolving.delete(node.nodeId); - } - } - } - frameId ??= rootFrameId; - ownership.set(node.nodeId, frameId); - return frameId; - }; - for (const node of axNodes) frameForAx(node); - for (const node of axNodes) { - const frameId = ownership.get(node.nodeId) ?? rootFrameId; - const document = documentByFrameId.get(frameId) ?? documents[0]; - document.axNodes.push({ - ...node, - frameId, - ...(node.parentId && ownership.get(node.parentId) === frameId - ? { parentId: node.parentId } - : { parentId: undefined }), - ...(node.childIds - ? { childIds: node.childIds.filter((childId) => ownership.get(childId) === frameId) } - : {}), - }); - } - return documents; -} - -function withLegacyBackendIds(scene: VomScene): VomScene { - const used = new Set(); - const idMap = new Map(); - let nextVirtualId = -1; - for (const node of scene.nodes) { - const preferred = node.backendNodeId; - const id = preferred !== undefined && !used.has(preferred) ? preferred : nextVirtualId--; - used.add(id); - idMap.set(node.id, id); - } - return { - ...scene, - nodes: scene.nodes.map((node) => ({ - ...node, - id: idMap.get(node.id) as number, - parentId: node.parentId === null ? null : (idMap.get(node.parentId) ?? null), - ...(node.domParentId !== undefined - ? { domParentId: node.domParentId === null ? null : (idMap.get(node.domParentId) ?? null) } - : {}), - ...(node.domAncestorIds - ? { domAncestorIds: node.domAncestorIds.flatMap((id) => idMap.get(id) ?? []) } - : {}), - })), - ...(scene.surfaces - ? { - surfaces: scene.surfaces.map((surface) => ({ - ...surface, - triggerId: idMap.get(surface.triggerId) ?? surface.triggerId, - })), - } - : {}), - ...(scene.activeScopeBlocks - ? { - activeScopeBlocks: scene.activeScopeBlocks.map((block) => ({ - ...block, - triggerId: idMap.get(block.triggerId) ?? block.triggerId, - })), - } - : {}), - }; -} - -export function buildVomScene( - axNodes: CdpAxNode[], - captured: CapturedViewModel, - options: BuildVomSceneOptions = {}, -): VomScene { - return withLegacyBackendIds( - buildFrameVomScene(legacyFrameDocuments(axNodes, captured, options.pageUrl), captured, options), - ); -} - export function buildFrameVomScene( documents: VomFrameDocument[], - captured: CapturedViewModel, + captured: CapturedSceneInput, options: BuildVomSceneOptions = {}, ): VomScene { const scene = buildSemanticVomScene({ @@ -833,7 +673,7 @@ export function buildFrameVomScene( function attachCapturedSceneAnnotations( scene: VomScene, documents: VomFrameDocument[], - captured: CapturedViewModel, + captured: CapturedSceneInput, surfaceProbes: CapturedSurfaceProbe[], ): VomScene { const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId); @@ -983,45 +823,6 @@ export async function handleGetHtml( } } -function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel { - return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() }; -} - -async function fallbackCapturedViewModel( - cdp: CdpRunner, - tabId: number, - geometry: GeometryContext, - signal?: AbortSignal, -): Promise { - throwIfAborted(signal, "observation"); - let viewport = { width: 0, height: 0 }; - try { - const metrics = await geometry.layoutMetrics({ tabId }); - throwIfAborted(signal, "observation"); - const source = cssViewport(metrics); - viewport = { width: source.width, height: source.height }; - } catch (error) { - if (isAbortError(error)) throw error; - } - const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal); - throwIfAborted(signal, "observation"); - return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds }; -} - -async function captureForVom( - cdp: CdpRunner, - tabId: number, - options: CaptureVomObservationOptions, - geometry: GeometryContext, -): Promise { - try { - return await captureViewModel(cdp, tabId, { signal: options.signal, geometry }); - } catch (error) { - if (isAbortError(error)) throw error; - return fallbackCapturedViewModel(cdp, tabId, geometry, options.signal); - } -} - export interface HoverProbeOutcome { /** Whether any active hover was dispatched during this observation. */ performed: boolean; @@ -1054,7 +855,7 @@ const NO_HOVER_PROBES: HoverProbeOutcome = { async function runHoverProbes( cdp: CdpRunner, tabId: number, - captured: CapturedViewModel, + captured: CapturedSceneInput, documents: VomFrameDocument[], staticSemantics: ReturnType, options: CaptureVomObservationOptions, @@ -1093,21 +894,9 @@ export async function captureVomObservation( throwIfAborted(options.signal, "observation"); await cdp.ensureAttachedToUrl?.(tabId, url); throwIfAborted(options.signal, "observation"); - const geometry = new GeometryContext(cdp, tabId, undefined, options.signal); - const captured = await captureForVom(cdp, tabId, options, geometry); + const facts = await captureObservationFacts(cdp, tabId, options.signal, url); + const { captured, documents: normalizedDocuments } = semanticCapture(facts); throwIfAborted(options.signal, "observation"); - const documents = await captureFrameData( - cdp, - tabId, - captured, - options.signal, - geometry, - ); - throwIfAborted(options.signal, "observation"); - const normalizedDocuments = - documents.length === 1 && captured.iframeNodes.size > 0 - ? legacyFrameDocuments(documents[0].axNodes, captured, url, documents[0].target) - : documents; const semanticGraph = buildSemanticGraph({ documents: normalizedDocuments, viewport: captured.viewport, @@ -1137,8 +926,8 @@ export async function captureVomObservation( ); // Reserve space using the renderer's character-based token estimate. Like VOM // headers, this integrity notice remains visible even under a tiny token budget. - const geometryNotice = captured.frameGeometryIssues?.length - ? "@warning geometry incomplete: some page or frame content has no top-level coordinates.\n" + const geometryNotice = facts.issues.some((issue) => issue.stage === "geometry") + ? "@warning iframe geometry incomplete: some frame content has no top-level coordinates.\n" : ""; const rendered = renderVom(decoratedScene, { maxDepth: options.maxDepth, diff --git a/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts b/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts new file mode 100644 index 0000000..2a8471c --- /dev/null +++ b/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts @@ -0,0 +1,1119 @@ +import { describe, expect, it, vi } from "vitest"; +import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph"; +import { cdpTargetKey } from "@/browser-driver/frame-graph"; +import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge"; +import type { CdpRunner } from "../../shared"; +import { captureObservationFacts, semanticCapture } from "../capture-coordinator"; +import { buildSemanticGraph } from "../semantic-graph/build"; +import { REQUESTED_STYLES } from "../snapshot"; + +function fixture( + options: { + frames?: CdpFrame[]; + after?: Record< + string, + { loader?: string; element?: number; missing?: boolean; unreadable?: boolean } + >; + fail?: string; + missingIdentity?: boolean; + omitDocument?: string; + attachmentChanged?: boolean; + overlay?: string; + } = {}, +) { + const frames = options.frames ?? [ + { frameId: "main", loaderId: "main-loader", target: { tabId: 4 } }, + { + frameId: "same", + parentFrameId: "main", + ownerBackendNodeId: 3, + loaderId: "same-loader", + target: { tabId: 4 }, + }, + { + frameId: "nested", + parentFrameId: "same", + ownerBackendNodeId: 13, + loaderId: "nested-loader", + target: { tabId: 4 }, + }, + { + frameId: "remote", + parentFrameId: "main", + ownerBackendNodeId: 4, + loaderId: "remote-loader", + target: { tabId: 4, sessionId: "remote" }, + }, + ]; + const elements = new Map( + frames.map((frame, i) => [frame.frameId, frame.target.sessionId ? 1 : i * 10 + 1]), + ); + const contexts = new Map(frames.map((frame, i) => [i + 1, frame.frameId])); + const reads = new Map(); + const logs: Array<{ target: CdpTarget; method: string; params: Record }> = []; + let snapshots = 0; + const targetCount = new Set(frames.map((frame) => cdpTargetKey(frame.target))).size; + const send = vi.fn( + async (target: CdpTarget, method: string, params: object = {}): Promise => { + logs.push({ target, method, params: params as Record }); + const args = params as Record; + if (options.fail === `${target.sessionId ?? "main"}:${method}`) + throw new Error("fixture failure"); + let result: unknown = {}; + if (method === "Page.createIsolatedWorld") + result = { + executionContextId: frames.findIndex((frame) => frame.frameId === args.frameId) + 1, + }; + if (method === "Runtime.evaluate") + result = { result: { objectId: contexts.get(Number(args.contextId)) } }; + if (method === "DOM.describeNode") { + const id = String(args.objectId); + const count = reads.get(id) ?? 0; + reads.set(id, count + 1); + result = + options.missingIdentity || (count > 0 && options.after?.[id]?.unreadable) + ? {} + : { + node: { + backendNodeId: + count > 0 + ? (options.after?.[id]?.element ?? elements.get(id)) + : elements.get(id), + }, + }; + } + if (method === "Page.getFrameTree") { + const own = frames.filter( + (frame) => + cdpTargetKey(frame.target) === cdpTargetKey(target) && + !options.after?.[frame.frameId]?.missing, + ); + const trees = new Map( + own.map((frame) => [ + frame.frameId, + { + frame: { + id: frame.frameId, + loaderId: options.after?.[frame.frameId]?.loader ?? frame.loaderId, + }, + childFrames: [] as unknown[], + }, + ]), + ); + for (const frame of own) + if (frame.parentFrameId && trees.has(frame.parentFrameId)) + trees.get(frame.parentFrameId)!.childFrames.push(trees.get(frame.frameId)); + result = { frameTree: trees.get(own[0]?.frameId) }; + } + if (method === "Page.getLayoutMetrics") + result = { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } }; + if (method === "DOMSnapshot.captureSnapshot") { + snapshots++; + expect((params as { computedStyles: unknown }).computedStyles).toEqual(REQUESTED_STYLES); + result = { + strings: [ + "#document", + "html", + "button", + OVERLAY_HOST_MARKER_ATTR, + "", + "visible", + "1", + "static", + "auto", + ], + documents: frames + .filter( + (frame) => + cdpTargetKey(frame.target) === cdpTargetKey(target) && + frame.frameId !== options.omitDocument, + ) + .map((frame) => { + const element = elements.get(frame.frameId)!; + const localFrames = frames.filter( + (f) => + cdpTargetKey(f.target) === cdpTargetKey(target) && + f.frameId !== options.omitDocument, + ); + const children = localFrames.filter((f) => f.parentFrameId === frame.frameId); + + return { + frameId: frame.frameId, + nodes: { + backendNodeId: [ + element - 1, + element + 10000, + element, + element + 1, + ...children.map((f) => f.ownerBackendNodeId!), + ], + nodeName: [0, 1, 1, 2, ...children.map(() => 2)], + nodeType: [9, 10, 1, 1, ...children.map(() => 1)], + parentIndex: [-1, 0, 0, 2, ...children.map(() => 2)], + attributes: [ + [], + [], + [], + frame.frameId === options.overlay ? [3, 4] : [], + ...children.map(() => []), + ], + contentDocumentIndex: { + index: children.map((_, i) => 4 + i), + value: children.map((f) => localFrames.indexOf(f)), + }, + }, + layout: { + nodeIndex: [2, 3], + bounds: [ + [0, 0, 1000, 800], + [10, 20, 100, 40], + ], + styles: [ + [7, 8, 8, 5, 6], + [7, 8, 8, 5, 6], + ], + }, + }; + }), + }; + } + if (method === "Accessibility.getFullAXTree") { + const frameId = String(args.frameId); + result = { + nodes: [ + { + nodeId: `${frameId}-button`, + frameId, + backendDOMNodeId: elements.get(frameId)! + 1, + role: { value: "button" }, + name: { value: frameId }, + }, + ], + }; + } + return result as T; + }, + ); + const cdp: CdpRunner = { + getAttachmentId: () => + options.attachmentChanged && snapshots === targetCount ? "new-attachment" : "attachment", + getFrameGraph: async () => ({ rootFrameId: frames[0].frameId, frames }), + send: (tabId, method, params) => + (send as NonNullable)({ tabId }, method, params), + sendToTarget: send as NonNullable, + }; + return { cdp, logs, elements }; +} + +describe("captureObservationFacts", () => { + it("collects each target once and scopes equal backend IDs", async () => { + const { cdp, logs } = fixture(); + const facts = await captureObservationFacts(cdp, 4); + expect(logs.filter((call) => call.method === "DOMSnapshot.captureSnapshot")).toHaveLength(2); + expect(logs.filter((call) => call.method === "Accessibility.enable")).toHaveLength(2); + expect(logs.filter((call) => call.method === "Accessibility.getFullAXTree")).toHaveLength(4); + expect(facts.documents.every((doc) => doc.identity)).toBe(true); + expect(logs.filter((call) => call.method === "DOM.describeNode")).toHaveLength(8); + expect(logs.filter((call) => call.method === "Runtime.releaseObject")).toHaveLength(8); + const main = facts.documents.find((doc) => doc.frame.frameId === "main")!; + const remote = facts.documents.find((doc) => doc.frame.frameId === "remote")!; + expect(main.index.nodes.get(2)).not.toBe(remote.index.nodes.get(2)); + expect(main.index.nodes.get(2)?.parentBackendNodeId).toBe(1); + expect(main.index.nodes.get(10001)?.nodeType).toBe(10); + expect(remote.axNodes[0].frameId).toBe("remote"); + expect(main.index.nodes.get(2)).toBe(main.domNodes.find((node) => node.backendNodeId === 2)); + expect(facts.finishedAt).toBeGreaterThanOrEqual(facts.startedAt); + }); + + it.each([ + { loader: "new-loader" }, + { element: 99 }, + { missing: true }, + ])("isolates changed child identity %j and dependent descendants", async (change) => { + const { cdp } = fixture({ after: { same: change } }); + const facts = await captureObservationFacts(cdp, 4); + expect(facts.documents.map((doc) => doc.frame.frameId)).toEqual(["main", "remote"]); + expect(facts.issues).toContainEqual( + expect.objectContaining({ frameId: "same", reason: "document-changed" }), + ); + }); + + it.each([ + { after: { main: { element: 99 } } }, + { attachmentChanged: true }, + ])("rejects a root replacement or reattachment %j", async (options) => { + await expect(captureObservationFacts(fixture(options).cdp, 4)).rejects.toThrow( + "document identity", + ); + }); + + it.each([ + "main", + "same", + ])("does not publish a previously identified %s document after identity reads fail", async (frameId) => { + const { cdp } = fixture({ after: { [frameId]: { unreadable: true } } }); + if (frameId === "main") + await expect(captureObservationFacts(cdp, 4)).rejects.toThrow("document identity"); + else { + const facts = await captureObservationFacts(cdp, 4); + expect(facts.documents.map((doc) => doc.frame.frameId)).toEqual(["main", "remote"]); + expect(facts.issues).toContainEqual( + expect.objectContaining({ frameId, reason: "identity-unverified" }), + ); + } + }); + + it("keeps newly discovered snapshot documents unverified without reading a synthetic before identity", async () => { + const { cdp, logs } = fixture(); + const originalGraph = await cdp.getFrameGraph!(4); + cdp.getFrameGraph = async () => ({ + ...originalGraph, + frames: originalGraph.frames.slice(0, 1), + }); + const facts = await captureObservationFacts(cdp, 4); + expect(facts.documents.map((doc) => doc.frame.frameId)).toEqual(["main", "same", "nested"]); + expect(facts.documents[0].identity).toBeDefined(); + for (const doc of facts.documents.slice(1)) { + expect(doc.identity).toBeUndefined(); + expect(facts.issues).toContainEqual( + expect.objectContaining({ frameId: doc.frame.frameId, reason: "identity-unverified" }), + ); + } + expect( + logs.filter((call) => call.method === "DOM.describeNode").map((call) => call.params.objectId), + ).toEqual(["main", "main"]); + }); + + it.each([ + "frame-id", + "snapshot-element", + ])("rejects root replacement evidenced by %s even if the later graph is stale", async (mode) => { + const { cdp } = fixture(); + const original = cdp.sendToTarget!; + cdp.sendToTarget = async (target, method, params) => { + const result = await original(target, method, params); + if (method === "DOMSnapshot.captureSnapshot" && !target.sessionId) { + const snapshot = result as { + documents: { frameId: string; nodes: { backendNodeId: number[] } }[]; + }; + if (mode === "frame-id") snapshot.documents[0].frameId = "replacement"; + else snapshot.documents[0].nodes.backendNodeId[2] = 999; + } + return result as never; + }; + cdp.send = (tabId, method, params) => cdp.sendToTarget!({ tabId }, method, params); + await expect(captureObservationFacts(cdp, 4)).rejects.toThrow("document identity"); + }); + + it("starts after-identity reads only after every target finishes and propagates cancellation during identity cleanup", async () => { + const { cdp, logs } = fixture(); + const controller = new AbortController(); + const original = cdp.sendToTarget!; + let identityCleanup = 0; + cdp.sendToTarget = async (target, method, params) => { + if (method === "Page.getFrameTree") { + expect(logs.filter((call) => call.method === "Accessibility.getFullAXTree")).toHaveLength( + 4, + ); + } + const result = await original(target, method, params); + if ( + method === "Runtime.releaseObject" && + (params as { objectId?: string })?.objectId === "remote" + ) { + if (++identityCleanup === 2) controller.abort(); + } + return result as never; + }; + cdp.send = (tabId, method, params) => cdp.sendToTarget!({ tabId }, method, params); + await expect(captureObservationFacts(cdp, 4, controller.signal)).rejects.toMatchObject({ + name: "AbortError", + }); + expect(identityCleanup).toBe(2); + }); + + it("does not retry failed or missing documents and retains valid AX-only semantics", async () => { + const { cdp, logs } = fixture({ + fail: "remote:DOMSnapshot.captureSnapshot", + omitDocument: "same", + }); + const facts = await captureObservationFacts(cdp, 4); + const remote = facts.documents.find((doc) => doc.frame.frameId === "remote")!; + expect(remote.domNodes).toHaveLength(0); + expect(remote.axNodes).toHaveLength(1); + expect(logs.filter((call) => call.method === "DOMSnapshot.captureSnapshot")).toHaveLength(2); + expect(facts.issues).toContainEqual( + expect.objectContaining({ frameId: "same", stage: "dom", reason: "capture-unavailable" }), + ); + expect(facts.issues).toContainEqual( + expect.objectContaining({ target: { tabId: 4, sessionId: "remote" }, stage: "dom" }), + ); + }); + + it("keeps missing identity explicit without manufacturing a verified document", async () => { + const facts = await captureObservationFacts(fixture({ missingIdentity: true }).cdp, 4); + expect(facts.documents.every((doc) => !doc.identity)).toBe(true); + expect(facts.issues.filter((issue) => issue.reason === "identity-unverified")).toHaveLength(4); + }); + + it("excludes overlay AX in its own document without excluding equal IDs elsewhere", async () => { + const facts = await captureObservationFacts(fixture({ overlay: "remote" }).cdp, 4); + const { documents } = semanticCapture(facts); + const graph = buildSemanticGraph({ + documents, + viewport: facts.viewport, + rootFrameId: facts.rootFrameId, + }); + expect( + [...graph.nodes.values()].find( + (node) => node.frameId === "remote" && node.backendNodeId === 2, + )?.excluded, + ).toBe(true); + expect( + [...graph.nodes.values()].find((node) => node.frameId === "main" && node.backendNodeId === 2) + ?.excluded, + ).toBe(false); + }); + + it.each([ + "cancel", + "worker-abort", + ] as const)("joins target cleanup and stops claiming targets after %s", async (mode) => { + const frames = Array.from({ length: 5 }, (_, i) => ({ + frameId: `f${i}`, + target: { tabId: 4, ...(i ? { sessionId: `s${i}` } : {}) }, + })); + const { cdp } = fixture({ frames }); + const original = cdp.sendToTarget!; + const pending = new Map void; reject: (error: Error) => void }>(); + const calls: Array<{ target: string; method: string }> = []; + let release: (() => void) | undefined; + let released = false; + const send: NonNullable = async ( + target: CdpTarget, + method: string, + params?: object, + ) => { + calls.push({ target: target.sessionId ?? "main", method }); + if (method === "DOMSnapshot.captureSnapshot" && target.sessionId) { + await new Promise((resolve, reject) => + pending.set(target.sessionId!, { resolve, reject }), + ); + } + if (method === "Runtime.releaseObjectGroup") { + await new Promise((resolve) => { + release = resolve; + }); + released = true; + } + const result = await original(target, method, params); + if (method === "DOMSnapshot.captureSnapshot" && !target.sessionId) + (result as { strings: string[] }).strings[2] = "input"; + return result; + }; + cdp.sendToTarget = send; + cdp.send = (tabId, method, params) => send({ tabId }, method, params); + const controller = new AbortController(); + let settled = false; + const capture = captureObservationFacts(cdp, 4, controller.signal); + const rejected = expect(capture).rejects.toMatchObject({ name: "AbortError" }); + void capture.then( + () => { + settled = true; + }, + () => { + settled = true; + }, + ); + await vi.waitFor(() => { + expect(pending.size).toBe(3); + expect(release).toBeDefined(); + }); + if (mode === "cancel") controller.abort(); + // Cancellation may race with an ordinary transport rejection. Without an + // external signal, an escaping AbortError still terminates the worker pool. + pending + .get("s1")! + .reject( + mode === "cancel" + ? new Error("transport closed") + : new DOMException("target aborted", "AbortError"), + ); + pending.get("s2")!.resolve(); + pending.get("s3")!.resolve(); + await vi.waitFor(() => { + if (mode === "worker-abort") + expect( + calls.some( + (call) => call.target === "s3" && call.method === "Accessibility.getFullAXTree", + ), + ).toBe(true); + else + expect( + calls.some( + (call) => call.target === "s3" && call.method === "DOMSnapshot.captureSnapshot", + ), + ).toBe(true); + }); + // Allow worker rejection and the outer promise chain to run while cleanup + // is deliberately held. The old fail-fast pool settles here. + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(settled).toBe(false); + expect(released).toBe(false); + expect(calls.some((call) => call.target === "s4")).toBe(false); + if (mode === "cancel") + expect( + calls.some( + (call) => call.method === "Accessibility.enable" || call.method === "DOM.getDocument", + ), + ).toBe(false); + release!(); + await rejected; + expect(released).toBe(true); + expect(calls.some((call) => call.target === "s4")).toBe(false); + }); + + it("bounds collection across many targets and stops scheduling after cancellation", async () => { + const frames = Array.from({ length: 12 }, (_, i) => ({ + frameId: `f${i}`, + target: { tabId: 4, ...(i ? { sessionId: `s${i}` } : {}) }, + })); + const { cdp } = fixture({ frames }); + let active = 0, + peak = 0; + const original = cdp.sendToTarget!; + const send: NonNullable = async (target, method, params) => { + active++; + peak = Math.max(peak, active); + try { + await new Promise((resolve) => setTimeout(resolve, 1)); + return await original(target, method, params); + } finally { + active--; + } + }; + cdp.sendToTarget = send; + cdp.send = (tabId, method, params) => send({ tabId }, method, params); + await captureObservationFacts(cdp, 4); + // This fixture has no frame edges, so no concurrent owner requests. + expect(peak).toBeLessThanOrEqual(4); + expect(peak).toBeGreaterThan(1); + const controller = new AbortController(); + controller.abort(); + await expect(captureObservationFacts(cdp, 4, controller.signal)).rejects.toMatchObject({ + name: "AbortError", + }); + expect(active).toBe(0); + }); +}); + +function childSnapshot(frameId: string, backendNodeId: number) { + const strings = [frameId, "body", "button", "static", "auto", "pointer"]; + return { + strings, + documents: [ + { + frameId, + nodes: { + parentIndex: [-1, 0], + nodeName: [1, 2], + backendNodeId: [backendNodeId - 1, backendNodeId], + attributes: [[], []], + }, + layout: { + nodeIndex: [0, 1], + styles: [ + [3, 4, 4], + [3, 4, 5], + ], + bounds: [ + [0, 0, 300, 200], + [10, 20, 100, 40], + ], + paintOrders: [0, 1], + }, + }, + ], + }; +} + +describe("OOPIF capture", () => { + it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => { + const sendToTarget = vi.fn(async (target, method) => { + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } }; + } + if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") { + return target.sessionId === "left-session" + ? childSnapshot("left", 101) + : childSnapshot("right", 201); + } + if (method === "Accessibility.getFullAXTree") return { nodes: [] }; + throw new Error(`unexpected ${method}`); + }); + const cdp: CdpRunner = { + send: vi.fn(async (_tabId, method, params) => { + if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("main", 1); + if (method === "Accessibility.getFullAXTree") return { nodes: [] }; + if (method === "DOM.getBoxModel") { + const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId; + const x = backendNodeId === 10 ? 50 : 500; + return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } }; + } + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } }; + } + throw new Error(`unexpected root ${method}`); + }) as CdpRunner["send"], + sendToTarget: sendToTarget as unknown as NonNullable, + getFrameGraph: vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "left", + parentFrameId: "main", + ownerBackendNodeId: 10, + target: { tabId: 4, sessionId: "left-session" }, + }, + { + frameId: "right", + parentFrameId: "main", + ownerBackendNodeId: 20, + target: { tabId: 4, sessionId: "right-session" }, + }, + ], + })), + }; + + const { documents: trees } = semanticCapture(await captureObservationFacts(cdp, 4)); + + expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]); + expect( + trees + .find((doc) => doc.frameId === "left") + ?.domNodes?.find((node) => node.backendNodeId === 101)?.rect, + ).toEqual({ x: 60, y: 120, w: 100, h: 40 }); + expect( + trees + .find((doc) => doc.frameId === "right") + ?.domNodes?.find((node) => node.backendNodeId === 201)?.rect, + ).toEqual({ x: 510, y: 120, w: 100, h: 40 }); + expect( + new Map( + trees + .filter((doc) => doc.ownerBackendNodeId !== undefined) + .map((doc) => [doc.frameId, doc.ownerBackendNodeId]), + ), + ).toEqual( + new Map([ + ["left", 10], + ["right", 20], + ]), + ); + expect( + new Map( + trees.filter((doc) => doc.parentFrameId).map((doc) => [doc.frameId, doc.parentFrameId]), + ), + ).toEqual( + new Map([ + ["left", "main"], + ["right", "main"], + ]), + ); + }); + + it("retains nested owner failures when merging a captured OOPIF", async () => { + const child = childSnapshot("child", 101); + const nested = childSnapshot("nested", 201); + const document = child.documents[0]; + const snapshot = { + strings: child.strings, + documents: [ + { + ...document, + nodes: { ...document.nodes, contentDocumentIndex: { index: [1], value: [1] } }, + }, + nested.documents[0], + ], + }; + const reply = async (_target: unknown, method: string) => { + if (method === "Page.getLayoutMetrics") + return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200 } }; + if (method === "DOMSnapshot.captureSnapshot") + return typeof _target === "number" ? childSnapshot("main", 11) : snapshot; + if (method === "DOM.getBoxModel") + return { model: { content: [50, 100, 350, 100, 350, 300, 50, 300] } }; + if (method === "DOM.resolveNode") throw new Error("nested owner replaced"); + return {}; + }; + const cdp: CdpRunner = { + send: vi.fn(reply) as CdpRunner["send"], + sendToTarget: vi.fn(reply) as NonNullable, + getFrameGraph: vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "nested", + parentFrameId: "child", + ownerBackendNodeId: 101, + target: { tabId: 4, sessionId: "child-session" }, + }, + { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 10, + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })), + }; + const captured = await captureObservationFacts(cdp, 4); + expect( + captured.issues + .filter((issue) => issue.projectionIssue) + .map((issue) => issue.projectionIssue), + ).toEqual([ + { + status: "unavailable", + source: { target: { tabId: 4, sessionId: "child-session" }, frameId: "nested" }, + ownerBackendNodeId: 101, + }, + ]); + expect( + captured.documents.find((doc) => doc.frame.frameId === "nested")?.domNodes[1], + ).toMatchObject({ + rect: null, + localRect: { x: 10, y: 20, w: 100, h: 40 }, + }); + }); + + it("keeps OOPIF semantics when viewport projection is unavailable", async () => { + const cdp: CdpRunner = { + send: vi.fn(async (_tabId, method) => { + if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("main", 1); + if (method === "Accessibility.getFullAXTree") return { nodes: [] }; + if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable"); + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } }; + } + throw new Error(`unexpected root ${method}`); + }) as CdpRunner["send"], + sendToTarget: vi.fn(async (_target, method) => { + if (method === "Page.getLayoutMetrics") throw new Error("viewport unavailable"); + if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("child", 101); + if (method === "Accessibility.getFullAXTree") { + return { + nodes: [ + { + nodeId: "button", + backendDOMNodeId: 101, + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Continue" }, + }, + ], + }; + } + throw new Error(`unexpected ${method}`); + }) as unknown as NonNullable, + getFrameGraph: vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 10, + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })), + }; + + const { documents } = semanticCapture(await captureObservationFacts(cdp, 4)); + const childDocument = documents.find((document) => document.frameId === "child"); + + expect(childDocument?.axNodes).toEqual([ + expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }), + ]); + expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual( + expect.objectContaining({ rect: null, localRect: { x: 10, y: 20, w: 100, h: 40 } }), + ); + }); +}); + +// Six sibling frames and one nested frame exercise scheduling through the real capture entry. +function siblingCaptureFixture( + beforeReply: (method: string, params: Record) => Promise = async () => {}, +) { + const document = (id: number, owners: number[], childIndexes: number[]) => ({ + frameId: `frame-${id}`, + nodes: { + parentIndex: [-1, ...owners.map(() => 0)], + nodeName: [0, ...owners.map(() => 1)], + backendNodeId: [1000 + id, ...owners], + attributes: [[], ...owners.map(() => [])], + contentDocumentIndex: { index: owners.map((_, i) => i + 1), value: childIndexes }, + }, + layout: { + nodeIndex: [0, ...owners.map((_, i) => i + 1)], + bounds: [[0, 0, 200, 100], ...owners.map(() => [0, 0, 200, 100])], + }, + }); + const snapshot = { + strings: ["body", "iframe"], + documents: [ + document(0, [100, 101, 102, 103, 104, 105], [1, 2, 3, 4, 5, 6]), + document(1, [200], [7]), + ...Array.from({ length: 6 }, (_, i) => document(i + 2, [], [])), + ], + }; + let active = 0; + let peak = 0; + const send = vi.fn(async (_tabId: number, method: string, params: object = {}) => { + const args = params as Record; + active++; + peak = Math.max(peak, active); + try { + await beforeReply(method, args); + if (method === "DOMSnapshot.captureSnapshot") return snapshot; + if (method === "Page.getLayoutMetrics") + return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } }; + if (method === "DOM.getBoxModel") + return { model: { content: [0, 0, 200, 0, 200, 100, 0, 100] } }; + if (method === "DOM.resolveNode") return { object: { objectId: String(args.backendNodeId) } }; + if (method === "Runtime.callFunctionOn") + return { result: { value: { width: 200, height: 100 } } }; + return {}; + } finally { + active--; + } + }); + return { + snapshot, + cdp: { send: send as CdpRunner["send"] }, + send, + peak: () => peak, + active: () => active, + measured: () => + send.mock.calls + .filter(([, method]) => method === "DOM.getBoxModel") + .map(([, , params]) => (params as { backendNodeId: number }).backendNodeId), + }; +} + +describe("snapshot document provenance", () => { + it.each([ + "new-child", + "new-root", + "stale-owner", + ])("retains current snapshot evidence with %s graph data", async (mode) => { + const f = siblingCaptureFixture(); + const cdp: CdpRunner = { + ...f.cdp, + getFrameGraph: async () => ({ + rootFrameId: mode === "new-root" ? "old-root" : "frame-0", + frames: [ + { frameId: mode === "new-root" ? "old-root" : "frame-0", target: { tabId: 4 } }, + ...(mode === "stale-owner" + ? [ + { + frameId: "frame-1", + parentFrameId: "frame-0", + ownerBackendNodeId: 999, + target: { tabId: 4 }, + }, + ] + : []), + ], + }), + }; + const facts = await captureObservationFacts(cdp, 4); + expect(facts.rootFrameId).toBe("frame-0"); + expect(facts.documents.find((doc) => doc.frame.frameId === "frame-1")).toMatchObject({ + frame: { ownerBackendNodeId: 100 }, + domNodes: [ + expect.objectContaining({ rect: { x: 0, y: 0, w: 200, h: 100 } }), + expect.anything(), + ], + }); + expect( + facts.documents.find((doc) => doc.frame.frameId === "frame-7")?.domNodes[0].rect, + ).not.toBeNull(); + expect(f.measured()).not.toContain(999); + expect( + f.send.mock.calls.filter(([, method]) => method === "DOMSnapshot.captureSnapshot"), + ).toHaveLength(1); + expect( + f.send.mock.calls.some( + ([, method, params]) => + method === "Accessibility.getFullAXTree" && + (params as { frameId?: string }).frameId === "frame-7", + ), + ).toBe(true); + }); + + it("uses snapshot URLs before graph URLs and limits the caller URL to the root", async () => { + const f = siblingCaptureFixture(); + f.snapshot.strings.push("https://current.test/page", "https://child.test/page"); + Object.assign(f.snapshot.documents[0], { documentURL: 2 }); + Object.assign(f.snapshot.documents[1], { documentURL: 3 }); + const facts = await captureObservationFacts( + { + ...f.cdp, + getFrameGraph: async () => ({ + rootFrameId: "frame-0", + frames: [{ frameId: "frame-0", target: { tabId: 4 }, url: "https://old.test/" }], + }), + }, + 4, + undefined, + "https://caller.test/", + ); + expect(facts.documents.find((doc) => doc.frame.frameId === "frame-0")?.frame.url).toBe( + "https://current.test/page", + ); + expect(facts.documents.find((doc) => doc.frame.frameId === "frame-1")?.frame.url).toBe( + "https://child.test/page", + ); + expect( + facts.documents.find((doc) => doc.frame.frameId === "frame-2")?.frame.url, + ).toBeUndefined(); + }); + + it("accepts a migrated snapshot claim without merging AX from the old target", async () => { + const f = siblingCaptureFixture(); + const cdp: CdpRunner = { + ...f.cdp, + getFrameGraph: async () => ({ + rootFrameId: "frame-0", + frames: [ + { frameId: "frame-0", target: { tabId: 4 } }, + { + frameId: "frame-1", + parentFrameId: "frame-0", + ownerBackendNodeId: 100, + target: { tabId: 4, sessionId: "old" }, + }, + ], + }), + sendToTarget: async (_target: CdpTarget, method: string): Promise => + (method === "Accessibility.getFullAXTree" + ? { nodes: [{ nodeId: "stale", role: { value: "button" }, name: { value: "stale" } }] } + : {}) as T, + }; + const facts = await captureObservationFacts(cdp, 4); + const current = facts.documents.find((doc) => doc.frame.frameId === "frame-1"); + expect(current?.frame.target).toEqual({ tabId: 4 }); + expect(current?.domNodes.length).toBeGreaterThan(0); + expect(current?.axNodes).toEqual([]); + }); + + it.each([ + 0, 1, + ])("rejects duplicate snapshot frame identity at document %i without overwriting", async (index) => { + const f = siblingCaptureFixture(); + f.snapshot.documents.push(f.snapshot.documents[index]); + if (index === 0) { + await expect(captureObservationFacts(f.cdp, 4)).rejects.toThrow( + "root document ownership is ambiguous", + ); + } else { + const facts = await captureObservationFacts(f.cdp, 4); + expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-1"); + expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-7"); + expect( + facts.documents.find((doc) => doc.frame.frameId === "frame-2")?.domNodes.length, + ).toBeGreaterThan(0); + } + }); + + it("retains the caller URL in the root AX-only fallback", async () => { + const cdp: CdpRunner = { + getFrameGraph: async () => { + throw new Error("no graph"); + }, + send: async (_tabId: number, method: string): Promise => { + if (method === "DOMSnapshot.captureSnapshot") throw new Error("no snapshot"); + return ( + method === "Accessibility.getFullAXTree" + ? { nodes: [{ nodeId: "root", role: { value: "RootWebArea" } }] } + : {} + ) as T; + }, + }; + const facts = await captureObservationFacts(cdp, 4, undefined, "https://page.test/"); + expect(facts.documents[0].frame.url).toBe("https://page.test/"); + expect(facts.documents[0].axNodes).toHaveLength(1); + }); + + it("retains disconnected document data without inventing top-level geometry", async () => { + const f = siblingCaptureFixture(); + f.snapshot.documents[0].nodes.contentDocumentIndex = { index: [], value: [] }; + const facts = await captureObservationFacts(f.cdp, 4); + expect( + facts.documents.find((doc) => doc.frame.frameId === "frame-1")?.domNodes[0], + ).toMatchObject({ rect: null, localRect: { x: 0, y: 0, w: 200, h: 100 } }); + expect(f.measured()).toEqual([]); + expect(facts.issues).toContainEqual( + expect.objectContaining({ frameId: "frame-1", stage: "ownership" }), + ); + }); + + it.each([ + false, + true, + ])("isolates competing target claims independent of batch order %s", async (reverse) => { + const f = siblingCaptureFixture(); + const remote = { + frameId: "frame-1", + parentFrameId: "frame-0", + ownerBackendNodeId: 100, + target: { tabId: 4, sessionId: "remote" }, + }; + const main = { frameId: "frame-0", target: { tabId: 4 } }; + const cdp: CdpRunner = { + ...f.cdp, + getFrameGraph: async () => ({ + rootFrameId: "frame-0", + frames: reverse ? [remote, main] : [main, remote], + }), + sendToTarget: async (_target: CdpTarget, method: string): Promise => + (method === "DOMSnapshot.captureSnapshot" + ? { ...f.snapshot, documents: [f.snapshot.documents[1]] } + : method === "Page.getLayoutMetrics" + ? { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } } + : {}) as T, + }; + const facts = await captureObservationFacts(cdp, 4); + expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-1"); + expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-7"); + expect( + facts.documents.find((doc) => doc.frame.frameId === "frame-2")?.domNodes[0].rect, + ).not.toBeNull(); + expect(facts.issues).toContainEqual( + expect.objectContaining({ frameId: "frame-1", stage: "ownership" }), + ); + }); +}); + +describe("sibling frame measurement scheduling", () => { + it("bounds concurrent reads, fills free slots and preserves breadth-first output despite reordered replies", async () => { + const pending = new Map void>(); + const fixture = siblingCaptureFixture(async (method, params) => { + if (method === "DOM.getBoxModel" && Number(params.backendNodeId) < 200) + await new Promise((resolve) => pending.set(Number(params.backendNodeId), resolve)); + }); + const capture = captureObservationFacts(fixture.cdp, 4); + await vi.waitFor(() => expect(pending.size).toBe(4)); + expect(fixture.measured()).toEqual([100, 101, 102, 103]); + pending.get(103)!(); + await vi.waitFor(() => expect(pending.has(104)).toBe(true)); + pending.get(104)!(); + await vi.waitFor(() => expect(pending.has(105)).toBe(true)); + expect(fixture.measured()).not.toContain(200); + for (const id of [105, 102, 101, 100]) pending.get(id)!(); + const captured = await capture; + expect( + captured.issues + .filter((issue) => issue.stage === "geometry") + .map((issue) => issue.projectionIssue), + ).toEqual([]); + expect(fixture.peak()).toBe(4); + expect(fixture.active()).toBe(0); + expect(fixture.measured()).toEqual([100, 101, 102, 103, 104, 105, 200]); + expect(captured.documents.map((doc) => doc.frame.frameId)).toEqual( + Array.from({ length: 8 }, (_, i) => `frame-${i}`), + ); + expect( + captured.documents.find((doc) => doc.frame.frameId === "frame-7")?.domNodes[0].rect, + ).toEqual({ x: 0, y: 0, w: 200, h: 100 }); + for (const method of [ + "DOM.getBoxModel", + "DOM.resolveNode", + "Runtime.callFunctionOn", + "Runtime.releaseObject", + ]) + expect(fixture.send.mock.calls.filter(([, name]) => name === method)).toHaveLength(7); + }); + + it("does not measure descendants of a failed owner and retains sibling geometry and local nodes", async () => { + const fixture = siblingCaptureFixture(async (method, params) => { + if (method === "DOM.resolveNode" && params.backendNodeId === 100) + throw new Error("owner replaced"); + }); + const captured = await captureObservationFacts(fixture.cdp, 4); + expect( + captured.issues + .filter((issue) => issue.stage === "geometry") + .map((issue) => issue.projectionIssue), + ).toEqual([ + { + status: "unavailable", + source: { target: { tabId: 4 }, frameId: "frame-1" }, + ownerBackendNodeId: 100, + }, + { + status: "blocked", + source: { target: { tabId: 4 }, frameId: "frame-7" }, + cause: captured.issues.find((issue) => issue.frameId === "frame-1")?.projectionIssue, + }, + ]); + expect(fixture.measured()).not.toContain(200); + expect( + captured.documents.find((doc) => doc.frame.frameId === "frame-7")?.domNodes[0], + ).toMatchObject({ + tag: "body", + rect: null, + localRect: { x: 0, y: 0, w: 200, h: 100 }, + }); + expect( + captured.documents.find((doc) => doc.frame.frameId === "frame-2")?.domNodes[0].rect, + ).toEqual({ x: 0, y: 0, w: 200, h: 100 }); + expect(captured.documents.map((doc) => doc.frame.frameId)).toEqual( + Array.from({ length: 8 }, (_, i) => `frame-${i}`), + ); + }); + + it("stops scheduling on cancellation and waits for resolved objects to be released", async () => { + const controller = new AbortController(); + const resolutions = new Map void>(); + const releases = new Map void>(); + const fixture = siblingCaptureFixture(async (method, params) => { + if (method === "DOM.resolveNode") + await new Promise((resolve) => + resolutions.set(String(params.backendNodeId), resolve), + ); + if (method === "Runtime.releaseObject") + await new Promise((resolve) => releases.set(String(params.objectId), resolve)); + }); + let settled = false; + const capture = captureObservationFacts(fixture.cdp, 4, controller.signal); + const rejected = expect(capture).rejects.toMatchObject({ name: "AbortError" }); + void capture.then( + () => { + settled = true; + }, + () => { + settled = true; + }, + ); + await vi.waitFor(() => expect(resolutions.size).toBe(4)); + controller.abort(); + for (const resolve of resolutions.values()) resolve(); + await vi.waitFor(() => expect(releases.size).toBe(4)); + expect(settled).toBe(false); + expect(fixture.measured()).toEqual([100, 101, 102, 103]); + for (const release of releases.values()) release(); + await rejected; + expect(fixture.active()).toBe(0); + expect(fixture.send.mock.calls.some(([, method]) => method === "Runtime.callFunctionOn")).toBe( + false, + ); + }); +}); diff --git a/apps/extension/src/tools/vom/__tests__/capture.test.ts b/apps/extension/src/tools/vom/__tests__/capture.test.ts index 1748dad..96813a0 100644 --- a/apps/extension/src/tools/vom/__tests__/capture.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, vi } from "vitest"; import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge"; -import type { CdpRunner } from "../../shared"; -import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture"; +import { collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture"; +import { captureObservationFacts, semanticCapture } from "../capture-coordinator"; // Minimal but format-accurate captureSnapshot reply: a body with one // fixed full-screen overlay div carrying a password input. @@ -27,6 +27,7 @@ function fakeSnapshotReply() { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1, 2], nodeType: [1, 1, 1, 1], @@ -74,6 +75,7 @@ function hoverTriggerSnapshotReply() { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1], nodeType: [1, 1, 1], @@ -118,6 +120,7 @@ function twoHoverTriggerSnapshotReply() { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1, 1], nodeType: [1, 1, 1, 1], @@ -171,6 +174,7 @@ function nestedHoverTriggerSnapshotReply() { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1, 2, 3], nodeType: [1, 1, 1, 1, 1], @@ -211,11 +215,7 @@ function makeCdp(snapshot: unknown) { if (method === "DOMSnapshot.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") return snapshot; if (method === "Page.getLayoutMetrics") { - return { - visualViewport: { clientWidth: 1000 }, - cssVisualViewport: { clientWidth: 1000 }, - cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, - }; + return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } }; } if (method === "Runtime.evaluate") { return { @@ -248,13 +248,11 @@ function makeCdp(snapshot: unknown) { }; } -describe("captureViewModel", () => { +describe("snapshot capture and hover", () => { it("parses nodes, attrs, rects, paint order and styles", async () => { - const { nodes, viewport, iframeNodes } = await captureViewModel( - makeCdp(fakeSnapshotReply()), - 4, - ); - expect(iframeNodes.size).toBe(0); + const { nodes, viewport } = semanticCapture( + await captureObservationFacts(makeCdp(fakeSnapshotReply()), 4), + ).captured; expect(viewport).toEqual({ width: 1000, height: 800 }); const div = nodes.find((n) => n.backendNodeId === 12); expect(div).toMatchObject({ tag: "div", position: "fixed", parentBackendNodeId: 11 }); @@ -266,35 +264,11 @@ describe("captureViewModel", () => { it("never hovers the page while capturing", async () => { const cdp = makeCdp(fakeSnapshotReply()); - await captureViewModel(cdp, 4); + semanticCapture(await captureObservationFacts(cdp, 4)).captured; expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything()); }); - it("batch-enriches form controls without per-node object resolution", async () => { - const cdp = makeCdp(fakeSnapshotReply()); - const result = await captureViewModel(cdp, 4); - const input = result.nodes.find((node) => node.backendNodeId === 13); - - expect(input).toMatchObject({ - formState: "filled", - formPlaceholder: "secret", - attrs: { type: "password" }, - }); - expect(input?.formValue).toBeUndefined(); - expect(input?.formDefaultValue).toBeUndefined(); - expect(cdp.send).toHaveBeenCalledWith( - 4, - "Runtime.evaluate", - expect.objectContaining({ - serializationOptions: expect.objectContaining({ serialization: "deep" }), - }), - ); - expect(cdp.send).toHaveBeenCalledWith(4, "Runtime.releaseObjectGroup", expect.anything()); - expect(cdp.send).not.toHaveBeenCalledWith(4, "DOM.resolveNode", expect.anything()); - expect(cdp.send).not.toHaveBeenCalledWith(4, "Runtime.callFunctionOn", expect.anything()); - }); - it("runs hover surface probes when explicitly enabled", async () => { let hoverStateCalls = 0; const cdp = { @@ -303,8 +277,6 @@ describe("captureViewModel", () => { if (method === "DOMSnapshot.captureSnapshot") return hoverTriggerSnapshotReply(); if (method === "Page.getLayoutMetrics") { return { - visualViewport: { clientWidth: 1000 }, - cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; } @@ -336,7 +308,7 @@ describe("captureViewModel", () => { }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const captured = await captureViewModel(cdp, 4); + const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured; const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes); expect(surfaceProbes).toEqual([ @@ -368,8 +340,6 @@ describe("captureViewModel", () => { if (method === "DOMSnapshot.captureSnapshot") return twoHoverTriggerSnapshotReply(); if (method === "Page.getLayoutMetrics") { return { - visualViewport: { clientWidth: 1000 }, - cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; } @@ -398,7 +368,7 @@ describe("captureViewModel", () => { }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const captured = await captureViewModel(cdp, 4); + const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured; const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes); expect(surfaceProbes).toEqual([ @@ -418,8 +388,6 @@ describe("captureViewModel", () => { if (method === "DOMSnapshot.captureSnapshot") return nestedHoverTriggerSnapshotReply(); if (method === "Page.getLayoutMetrics") { return { - visualViewport: { clientWidth: 1000 }, - cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; } @@ -445,7 +413,7 @@ describe("captureViewModel", () => { }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const captured = await captureViewModel(cdp, 4); + const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured; await probeHoverSurfaces(cdp, 4, captured.nodes); expect(hoverMoves).toBe(1); @@ -477,6 +445,7 @@ describe("captureViewModel", () => { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1, 0, 3, 4], nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")], @@ -507,7 +476,7 @@ describe("captureViewModel", () => { ], }; - const { nodes } = await captureViewModel(makeCdp(snapshot), 4); + const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured; const ids = nodes.map((n) => n.backendNodeId); // Real page nodes survive. expect(ids).toContain(12); @@ -538,9 +507,7 @@ describe("captureViewModel", () => { strings: S, documents: [ { - scrollOffsetX: 0, - scrollOffsetY: 0, - frameId: "main", + frameId: "root", nodes: { parentIndex: [-1, 0, 1, 0, 3, 4], nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")], @@ -571,12 +538,10 @@ describe("captureViewModel", () => { ], }; - const { excludedBackendNodeIds, frameExcludedBackendNodeIds } = await captureViewModel( - makeCdp(snapshot), - 4, - ); + const { excludedBackendNodeIds } = semanticCapture( + await captureObservationFacts(makeCdp(snapshot), 4), + ).captured; expect(excludedBackendNodeIds).toEqual(new Set([13, 14, 15])); - expect(frameExcludedBackendNodeIds?.get("main")).toEqual(excludedBackendNodeIds); }); it("excludes overlay host matched by tag name only", async () => { @@ -596,6 +561,7 @@ describe("captureViewModel", () => { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1], nodeName: [i("html"), i("body"), i(OVERLAY_HOST_NAME)], @@ -618,7 +584,9 @@ describe("captureViewModel", () => { ], }; - const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4); + const { nodes, excludedBackendNodeIds } = semanticCapture( + await captureObservationFacts(makeCdp(snapshot), 4), + ).captured; expect(nodes.map((n) => n.backendNodeId)).not.toContain(13); expect(excludedBackendNodeIds).toEqual(new Set([13])); }); @@ -641,6 +609,7 @@ describe("captureViewModel", () => { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1], nodeName: [i("html"), i("body"), i("div")], @@ -663,7 +632,9 @@ describe("captureViewModel", () => { ], }; - const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4); + const { nodes, excludedBackendNodeIds } = semanticCapture( + await captureObservationFacts(makeCdp(snapshot), 4), + ).captured; expect(nodes.map((n) => n.backendNodeId)).not.toContain(13); expect(excludedBackendNodeIds).toEqual(new Set([13])); }); @@ -686,6 +657,7 @@ describe("captureViewModel", () => { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1], nodeName: [i("html"), i("body"), i("div")], @@ -708,7 +680,7 @@ describe("captureViewModel", () => { }, ], }; - const { nodes } = await captureViewModel(makeCdp(snapshot), 4); + const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured; expect(nodes.find((n) => n.backendNodeId === 12)?.cursor).toBe("pointer"); expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto"); }); @@ -720,6 +692,7 @@ describe("captureViewModel", () => { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1, 1, 1], nodeName: [i("html"), i("body"), i("button"), i("button"), i("button")], @@ -746,7 +719,7 @@ describe("captureViewModel", () => { ], }; - const { nodes } = await captureViewModel(makeCdp(snapshot), 4); + const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured; expect(nodes.find((node) => node.backendNodeId === 12)?.rendered).toBe(true); expect(nodes.find((node) => node.backendNodeId === 13)?.rendered).toBe(false); @@ -760,66 +733,26 @@ describe("captureViewModel", () => { if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply(); if (method === "Page.getLayoutMetrics") { return { - visualViewport: { clientWidth: 1000 }, - cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 200 }, }; } throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const { nodes } = await captureViewModel(cdp, 4); + const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured; const div = nodes.find((n) => n.backendNodeId === 12); expect(div?.localRect?.y).toBe(-200); expect(div?.rect).toMatchObject({ y: 0, h: 600 }); }); - it("normalizes snapshot-owned scroll before using a stale CSS metrics fallback", async () => { - const snapshot = { - strings: ["html", "button"], - documents: [ - { - frameId: "root", - scrollOffsetX: 40, - scrollOffsetY: 200, - nodes: { - parentIndex: [-1, 0], - nodeName: [0, 1], - backendNodeId: [10, 11], - attributes: [[], []], - }, - layout: { nodeIndex: [1], bounds: [[200, 800, 240, 80]] }, - }, - ], - }; - const cdp: CdpRunner = { - send: vi.fn(async (_tab: number, method: string) => { - if (method === "DOMSnapshot.captureSnapshot") return snapshot; - if (method === "Page.getLayoutMetrics") - return { - visualViewport: { clientWidth: 2000 }, - cssVisualViewport: { clientWidth: 1000 }, - cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 999, pageY: 999 }, - }; - return {}; - }) as CdpRunner["send"], - }; - const captured = await captureViewModel(cdp, 4); - expect(captured.nodes.find((node) => node.backendNodeId === 11)).toMatchObject({ - localRect: { x: 80, y: 300, w: 120, h: 40 }, - rect: { x: 80, y: 300, w: 120, h: 40 }, - rendered: true, - }); - expect(captured.frameGeometryIssues).toEqual([]); - }); - - it("normalizes raw snapshot bounds before viewport clipping", async () => { + it("keeps CSS snapshot bounds independent of legacy metrics ratio", async () => { const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"]; const i = (s: string) => S.indexOf(s); const snapshot = { strings: S, documents: [ { + frameId: "root", nodes: { parentIndex: [-1, 0, 1], nodeName: [i("html"), i("body"), i("div")], @@ -847,8 +780,6 @@ describe("captureViewModel", () => { if (method === "DOMSnapshot.captureSnapshot") return snapshot; if (method === "Page.getLayoutMetrics") { return { - visualViewport: { clientWidth: 2000 }, - cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, layoutViewport: { clientWidth: 2000, clientHeight: 1600 }, }; @@ -856,7 +787,7 @@ describe("captureViewModel", () => { throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const { nodes } = await captureViewModel(cdp, 4); + const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured; expect(nodes.find((n) => n.backendNodeId === 12)?.rect).toEqual({ x: 0, y: 0, @@ -866,8 +797,8 @@ describe("captureViewModel", () => { expect(nodes.find((n) => n.backendNodeId === 12)?.localRect).toEqual({ x: 0, y: 0, - w: 1000, - h: 800, + w: 2000, + h: 1600, }); }); @@ -892,6 +823,7 @@ describe("captureViewModel", () => { strings: S, documents: [ { + frameId: "root", scrollOffsetX: 0, scrollOffsetY: 0, nodes: { @@ -924,15 +856,13 @@ describe("captureViewModel", () => { if (method === "DOMSnapshot.captureSnapshot") return snapshot; if (method === "Page.getLayoutMetrics") { return { - visualViewport: { clientWidth: 1000 }, - cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; } throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const { nodes } = await captureViewModel(cdp, 4); + const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured; const btn = nodes.find((n) => n.backendNodeId === 12); expect(btn?.textContent).toBe("登录"); const link = nodes.find((n) => n.backendNodeId === 14); @@ -942,7 +872,7 @@ describe("captureViewModel", () => { expect(body?.textContent).toBeUndefined(); }); - it("parses iframe sub-documents and returns iframeNodes keyed by iframe backendNodeId", async () => { + it("retains explicit iframe document ownership and projected nodes", async () => { const S = [ "html", "body", @@ -1016,8 +946,6 @@ describe("captureViewModel", () => { if (method === "DOMSnapshot.captureSnapshot") return snapshot; if (method === "Page.getLayoutMetrics") { return { - visualViewport: { clientWidth: 1000 }, - cssVisualViewport: { clientWidth: 1000 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; } @@ -1030,30 +958,21 @@ describe("captureViewModel", () => { throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const { - nodes, - iframeNodes, - frameNodes, - frameOwnerBackendNodeIds, - frameParentIds, - rootFrameId, - } = await captureViewModel(cdp, 4); - // Main frame has 4 nodes; iframe is included - expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe"); - // iframeNodes keyed by the