diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index 182df46..fcd61cc 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -4,9 +4,9 @@ import { CAPTURE_SUPPRESS, type CaptureSuppressMessage } from "@/lib/capture-sup import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "@/lib/overlay-bridge"; import { SessionManager } from "@/session-manager/manager"; import type { CdpRunner } from "@/tools/shared"; -import type { BuildVomSceneOptions, VomFrameDocument } from "../observation"; import { buildFrameVomScene, + buildVomScene, type CdpAxNode, captureVomObservation, handleGetHtml, @@ -17,35 +17,7 @@ import { type ScreenshotDeps, stripDataUrlPrefix, } from "../observation"; -import type { CapturedNode, CapturedSceneInput } from "../vom/facts"; - -interface SceneFixture extends CapturedSceneInput { - documents?: VomFrameDocument[]; -} - -// Only supplies the common root fixture; child documents declare their own ownership. -function buildTestScene( - axNodes: CdpAxNode[], - captured: SceneFixture, - options: BuildVomSceneOptions = {}, -) { - const rootFrameId = captured.rootFrameId ?? "root"; - return buildFrameVomScene( - [ - { - frameId: rootFrameId, - contextScopeId: rootFrameId, - target: { tabId: 7 }, - url: options.pageUrl, - domNodes: captured.nodes, - axNodes, - }, - ...(captured.documents ?? []), - ], - captured, - options, - ); -} +import type { CapturedNode, CapturedViewModel } from "../vom/capture"; function fakeAgentWindow(ids: number[]) { let i = 0; @@ -554,10 +526,10 @@ describe("handleScreenshot overlay suppression", () => { }); // --------------------------------------------------------------------------- -// Semantic scene fixtures +// buildVomScene // --------------------------------------------------------------------------- -describe("buildFrameVomScene", () => { +describe("buildVomScene", () => { it("merges multiple sibling and nested frames without mixing their targets", () => { const node = ( backendNodeId: number, @@ -595,10 +567,25 @@ describe("buildFrameVomScene", () => { node(300, null, "nested", "body"), node(301, 300, "nested", "input", { placeholder: "Nested field" }), ]; - const captured: SceneFixture = { + const captured: CapturedViewModel = { viewport: { width: 1200, height: 800 }, nodes: mainNodes, - + iframeNodes: new Map([ + [10, firstNodes], + [20, secondNodes], + [210, nestedNodes], + ]), + frameNodes: new Map([ + ["main", mainNodes], + ["first", firstNodes], + ["second", secondNodes], + ["nested", nestedNodes], + ]), + frameOwnerBackendNodeIds: new Map([ + ["first", 10], + ["second", 20], + ["nested", 210], + ]), rootFrameId: "main", excludedBackendNodeIds: new Set(), }; @@ -685,6 +672,60 @@ describe("buildFrameVomScene", () => { expect(nestedInput?.parentId).toBe(result.nodes.find((item) => item.backendNodeId === 210)?.id); }); + it("excludes an overlay only in its owning document when target backend ids collide", () => { + const owner = (id: number): CapturedNode => ({ + backendNodeId: id, + parentBackendNodeId: null, + frameId: "main", + tag: "iframe", + attrs: {}, + rect: { x: id * 10, y: 0, w: 200, h: 100 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }); + const owners = [owner(10), owner(20)]; + const result = buildFrameVomScene( + [ + { + frameId: "main", + contextScopeId: "main", + target: { tabId: 7 }, + domNodes: owners, + axNodes: [], + }, + ...["left", "right"].map((frameId, i) => ({ + frameId, + contextScopeId: frameId, + parentFrameId: "main", + ownerBackendNodeId: owners[i].backendNodeId, + target: { tabId: 7, sessionId: frameId }, + excludedBackendNodeIds: new Set(i === 0 ? [7] : []), + domNodes: [], + axNodes: [ + { + nodeId: "button", + frameId, + backendDOMNodeId: 7, + role: { type: "role", value: "button" }, + name: { type: "computedString", value: frameId + " action" }, + }, + ], + })), + ], + { + nodes: owners, + viewport: { width: 800, height: 600 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + }, + ); + const rendered = renderVom(result); + expect(rendered.text).not.toContain("left action"); + expect(rendered.text).toContain("right action"); + expect(rendered.refs.find((ref) => ref.backendNodeId === 7)?.frameId).toBe("right"); + }); + it("does not place a child document at the page root when its frame boundary is unresolved", () => { const childNode: CapturedNode = { backendNodeId: 101, @@ -697,10 +738,11 @@ describe("buildFrameVomScene", () => { position: "static", pointerEvents: "auto", }; - const captured: SceneFixture = { + const captured: CapturedViewModel = { viewport: { width: 800, height: 600 }, nodes: [], - + iframeNodes: new Map(), + frameNodes: new Map([["child", [childNode]]]), rootFrameId: "main", excludedBackendNodeIds: new Set(), }; @@ -762,10 +804,14 @@ describe("buildFrameVomScene", () => { pointerEvents: "auto", }, ]; - const captured: SceneFixture = { + const captured: CapturedViewModel = { viewport: { width: 800, height: 600 }, nodes: mainNodes, - + iframeNodes: new Map(), + frameNodes: new Map([ + ["main", mainNodes], + ["child", []], + ]), rootFrameId: "main", excludedBackendNodeIds: new Set(), }; @@ -841,8 +887,9 @@ describe("buildFrameVomScene", () => { backendDOMNodeId: 200, }, ]; - const captured: SceneFixture = { + const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -868,11 +915,11 @@ describe("buildFrameVomScene", () => { ], }; - const scene = buildTestScene(axNodes, captured); + const scene = buildVomScene(axNodes, captured); expect(scene.viewport).toEqual({ width: 1000, height: 800 }); expect(scene.nodes[0]).toEqual( expect.objectContaining({ - backendNodeId: 100, + id: 100, parentId: null, role: "RootWebArea", name: "Example", @@ -884,14 +931,14 @@ describe("buildFrameVomScene", () => { ); expect(scene.nodes[1]).toEqual( expect.objectContaining({ - backendNodeId: 200, - parentId: scene.nodes.find((node) => node.backendNodeId === 100)!.id, + id: 200, + parentId: 100, role: "button", name: "Submit", tag: "button", rect: { x: 20, y: 20, w: 120, h: 40 }, - domParentId: scene.nodes.find((node) => node.backendNodeId === 100)!.id, - domAncestorIds: [scene.nodes.find((node) => node.backendNodeId === 100)!.id], + domParentId: 100, + domAncestorIds: [100], attrs: {}, }), ); @@ -920,8 +967,9 @@ describe("buildFrameVomScene", () => { backendDOMNodeId: 300, }, ]; - const captured: SceneFixture = { + const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -957,10 +1005,10 @@ describe("buildFrameVomScene", () => { ], }; - const scene = buildTestScene(axNodes, captured, { pageUrl: "https://app.example.test/home" }); + const scene = buildVomScene(axNodes, captured, { pageUrl: "https://app.example.test/home" }); - expect(scene.nodes.find((node) => node.backendNodeId === 200)?.href).toBe("docs.example.org"); - expect(scene.nodes.find((node) => node.backendNodeId === 300)?.href).toBeUndefined(); + expect(scene.nodes.find((node) => node.id === 200)?.href).toBe("docs.example.org"); + expect(scene.nodes.find((node) => node.id === 300)?.href).toBeUndefined(); }); it("uses the nearest backend AX ancestor as parentId", () => { @@ -985,8 +1033,9 @@ describe("buildFrameVomScene", () => { backendDOMNodeId: 30, }, ]; - const captured: SceneFixture = { + const captured: CapturedViewModel = { viewport: { width: 400, height: 300 }, + iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1012,12 +1061,10 @@ describe("buildFrameVomScene", () => { ], }; - const scene = buildTestScene(axNodes, captured); + const scene = buildVomScene(axNodes, captured); - expect(scene.nodes.map((node) => node.backendNodeId)).toEqual([10, 30]); - expect(scene.nodes.find((node) => node.backendNodeId === 30)?.parentId).toBe( - scene.nodes.find((node) => node.backendNodeId === 10)!.id, - ); + expect(scene.nodes.map((node) => node.id)).toEqual([10, 30]); + expect(scene.nodes.find((node) => node.id === 30)?.parentId).toBe(10); }); it("maps iframe sub-document controls to VomNodes", () => { @@ -1035,7 +1082,7 @@ describe("buildFrameVomScene", () => { backendDOMNodeId: 20, }, ]; - const captured: SceneFixture = { + const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, nodes: [ { @@ -1059,15 +1106,10 @@ describe("buildFrameVomScene", () => { pointerEvents: "auto", }, ], - documents: [ - { - frameId: "child", - contextScopeId: "child", - parentFrameId: "root", - ownerBackendNodeId: 20, - target: { tabId: 7 }, - axNodes: [], - domNodes: [ + iframeNodes: new Map([ + [ + 20, + [ { backendNodeId: 101, parentBackendNodeId: null, @@ -1089,25 +1131,26 @@ describe("buildFrameVomScene", () => { pointerEvents: "auto", }, ], - }, - ], + ], + ]), excludedBackendNodeIds: new Set(), }; - const scene = buildTestScene(axNodes, captured); - const iframeControls = scene.nodes.filter((node) => [101, 102].includes(node.backendNodeId!)); + const iframeControls = buildVomScene(axNodes, captured).nodes.filter((node) => + [101, 102].includes(node.id), + ); expect(iframeControls).toEqual([ expect.objectContaining({ - backendNodeId: 101, - parentId: scene.nodes.find((node) => node.backendNodeId === 20)!.id, + id: 101, + parentId: 20, role: "textbox", name: "请输入手机号", sensitive: false, }), expect.objectContaining({ - backendNodeId: 102, - parentId: scene.nodes.find((node) => node.backendNodeId === 20)!.id, + id: 102, + parentId: 20, role: "textbox", name: "密码", sensitive: true, @@ -1128,9 +1171,11 @@ describe("buildFrameVomScene", () => { parentId: "1", role: { type: "role", value: "Iframe" }, backendDOMNodeId: 20, + childIds: ["3"], }, { nodeId: "3", + parentId: "2", role: { type: "role", value: "textbox" }, name: { type: "x", value: "输入密码" }, value: { value: "iframe-secret" }, @@ -1138,7 +1183,7 @@ describe("buildFrameVomScene", () => { properties: [{ name: "inputType", value: { value: "password" } }], }, ]; - const captured: SceneFixture = { + const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, nodes: [ { @@ -1162,15 +1207,10 @@ describe("buildFrameVomScene", () => { pointerEvents: "auto", }, ], - documents: [ - { - frameId: "child", - contextScopeId: "child", - parentFrameId: "root", - ownerBackendNodeId: 20, - target: { tabId: 7 }, - axNodes: [axNodes[2]], - domNodes: [ + iframeNodes: new Map([ + [ + 20, + [ { backendNodeId: 102, parentBackendNodeId: null, @@ -1182,27 +1222,25 @@ describe("buildFrameVomScene", () => { pointerEvents: "auto", }, ], - }, - ], + ], + ]), excludedBackendNodeIds: new Set(), }; - const passwordNodes = buildTestScene(axNodes.slice(0, 2), captured).nodes.filter( + const passwordNodes = buildVomScene(axNodes, captured).nodes.filter( (node) => node.role === "textbox" && node.name === "输入密码", ); expect(passwordNodes).toHaveLength(1); - expect(passwordNodes[0]).toEqual( - expect.objectContaining({ backendNodeId: 202, sensitive: true }), - ); + expect(passwordNodes[0]).toEqual(expect.objectContaining({ id: 202, sensitive: true })); expect(passwordNodes[0].value).toBeUndefined(); - const rendered = renderVom(buildTestScene(axNodes.slice(0, 2), captured)).text; + const rendered = renderVom(buildVomScene(axNodes, captured)).text; expect(rendered).toContain('textbox "输入密码" [filled] ="•••"'); expect(rendered).not.toContain("iframe-secret"); }); it("keeps unnamed iframe controls but skips unnamed iframe links", () => { - const scene = buildTestScene( + const scene = buildVomScene( [ { nodeId: "1", @@ -1241,15 +1279,10 @@ describe("buildFrameVomScene", () => { pointerEvents: "auto", }, ], - documents: [ - { - frameId: "child", - contextScopeId: "child", - parentFrameId: "root", - ownerBackendNodeId: 20, - target: { tabId: 7 }, - axNodes: [], - domNodes: [ + iframeNodes: new Map([ + [ + 20, + [ { backendNodeId: 201, parentBackendNodeId: null, @@ -1271,25 +1304,25 @@ describe("buildFrameVomScene", () => { pointerEvents: "auto", }, ], - }, - ], + ], + ]), excludedBackendNodeIds: new Set(), }, ); - expect(scene.nodes.find((node) => node.backendNodeId === 201)).toEqual( + expect(scene.nodes.find((node) => node.id === 201)).toEqual( expect.objectContaining({ - backendNodeId: 201, - parentId: scene.nodes.find((node) => node.backendNodeId === 20)!.id, + id: 201, + parentId: 20, role: "button", }), ); - expect(scene.nodes.find((node) => node.backendNodeId === 201)).not.toHaveProperty("name"); - expect(scene.nodes.find((node) => node.backendNodeId === 202)).toBeUndefined(); + expect(scene.nodes.find((node) => node.id === 201)).not.toHaveProperty("name"); + expect(scene.nodes.find((node) => node.id === 202)).toBeUndefined(); }); it("preserves captured-only iframe anchors for iframe sub-document controls", () => { - const scene = buildTestScene( + const scene = buildVomScene( [ { nodeId: "1", @@ -1322,15 +1355,10 @@ describe("buildFrameVomScene", () => { pointerEvents: "auto", }, ], - documents: [ - { - frameId: "child", - contextScopeId: "child", - parentFrameId: "root", - ownerBackendNodeId: 20, - target: { tabId: 7 }, - axNodes: [], - domNodes: [ + iframeNodes: new Map([ + [ + 20, + [ { backendNodeId: 201, parentBackendNodeId: null, @@ -1352,16 +1380,16 @@ describe("buildFrameVomScene", () => { pointerEvents: "auto", }, ], - }, - ], + ], + ]), excludedBackendNodeIds: new Set(), }, ); - expect(scene.nodes.find((node) => node.backendNodeId === 20)).toEqual( + expect(scene.nodes.find((node) => node.id === 20)).toEqual( expect.objectContaining({ - backendNodeId: 20, - parentId: scene.nodes.find((node) => node.backendNodeId === 10)!.id, + id: 20, + parentId: 10, role: "Iframe", name: "支付验证", }), @@ -1382,8 +1410,9 @@ describe("buildFrameVomScene", () => { backendDOMNodeId: 10, }, ]; - const captured: SceneFixture = { + const captured: CapturedViewModel = { viewport: { width: 320, height: 240 }, + iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1404,9 +1433,9 @@ describe("buildFrameVomScene", () => { ], }; - expect(buildTestScene(axNodes, captured).nodes[0]).toEqual( + expect(buildVomScene(axNodes, captured).nodes[0]).toEqual( expect.objectContaining({ - backendNodeId: 10, + id: 10, sensitive: true, }), ); @@ -1421,8 +1450,9 @@ describe("buildFrameVomScene", () => { backendDOMNodeId: 10, }, ]; - const captured: SceneFixture = { + const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1439,9 +1469,9 @@ describe("buildFrameVomScene", () => { ], }; - expect(buildTestScene(axNodes, captured).nodes[0]).toEqual( + expect(buildVomScene(axNodes, captured).nodes[0]).toEqual( expect.objectContaining({ - backendNodeId: 10, + id: 10, role: "textbox", name: "+86", placeholder: "输入手机号", @@ -1459,8 +1489,9 @@ describe("buildFrameVomScene", () => { backendDOMNodeId: 10, }, ]; - const captured: SceneFixture = { + const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1477,13 +1508,8 @@ describe("buildFrameVomScene", () => { ], }; - expect(buildTestScene(axNodes, captured).nodes[0]).toEqual( - expect.objectContaining({ - backendNodeId: 10, - role: "textbox", - name: "Email", - value: "a@b.com", - }), + expect(buildVomScene(axNodes, captured).nodes[0]).toEqual( + expect.objectContaining({ id: 10, role: "textbox", name: "Email", value: "a@b.com" }), ); }); @@ -1502,8 +1528,9 @@ describe("buildFrameVomScene", () => { backendDOMNodeId: 20, }, ]; - const captured: SceneFixture = { + const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1541,9 +1568,9 @@ describe("buildFrameVomScene", () => { ], }; - expect( - buildTestScene(axNodes, captured).nodes.find((node) => node.backendNodeId === 20), - ).toEqual(expect.objectContaining({ role: "textbox", name: "验证码" })); + expect(buildVomScene(axNodes, captured).nodes.find((node) => node.id === 20)).toEqual( + expect.objectContaining({ role: "textbox", name: "验证码" }), + ); }); it("uses preceding AX static text as a form control label", () => { @@ -1567,8 +1594,9 @@ describe("buildFrameVomScene", () => { backendDOMNodeId: 20, }, ]; - const captured: SceneFixture = { + const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1585,13 +1613,13 @@ describe("buildFrameVomScene", () => { ], }; - expect( - buildTestScene(axNodes, captured).nodes.find((node) => node.backendNodeId === 20), - ).toEqual(expect.objectContaining({ role: "textbox", name: "验证码", inputState: "empty" })); + expect(buildVomScene(axNodes, captured).nodes.find((node) => node.id === 20)).toEqual( + expect.objectContaining({ role: "textbox", name: "验证码", inputState: "empty" }), + ); }); it("marks runtime default input values without treating them as ordinary filled input", () => { - const scene = buildTestScene( + const scene = buildVomScene( [ { nodeId: "1", @@ -1601,6 +1629,7 @@ describe("buildFrameVomScene", () => { ], { viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1644,8 +1673,9 @@ describe("buildFrameVomScene", () => { backendDOMNodeId: 20, }, ]; - const captured: SceneFixture = { + const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1695,14 +1725,9 @@ describe("buildFrameVomScene", () => { ], }; - const scene = buildTestScene(axNodes, captured); - expect(scene.nodes.find((n) => n.backendNodeId === 20)).toEqual( - expect.objectContaining({ - backendNodeId: 20, - role: "button", - name: "close", - cursor: "pointer", - }), + const scene = buildVomScene(axNodes, captured); + expect(scene.nodes.find((n) => n.id === 20)).toEqual( + expect.objectContaining({ id: 20, role: "button", name: "close", cursor: "pointer" }), ); const rendered = renderVom(scene); expect(rendered.text).toContain('@e1 button "close"'); @@ -1736,8 +1761,9 @@ describe("buildFrameVomScene", () => { backendDOMNodeId: 30, }, ]; - const captured: SceneFixture = { + const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1776,9 +1802,9 @@ describe("buildFrameVomScene", () => { ], }; - const scene = buildTestScene(axNodes, captured); - expect(scene.nodes.find((n) => n.backendNodeId === 20)).toBeUndefined(); - expect(scene.nodes.find((n) => n.backendNodeId === 30)?.role).toBe("link"); + const scene = buildVomScene(axNodes, captured); + expect(scene.nodes.find((n) => n.id === 20)).toBeUndefined(); + expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("link"); }); it("promotes only the outermost clickable in a nested pointer chain", () => { @@ -1803,8 +1829,9 @@ describe("buildFrameVomScene", () => { backendDOMNodeId: 30, }, ]; - const captured: SceneFixture = { + const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1843,15 +1870,11 @@ describe("buildFrameVomScene", () => { ], }; - const scene = buildTestScene(axNodes, captured); - expect(scene.nodes.find((n) => n.backendNodeId === 20)).toEqual( - expect.objectContaining({ - backendNodeId: 20, - role: "button", - attrs: { "aria-label": "收藏" }, - }), + const scene = buildVomScene(axNodes, captured); + expect(scene.nodes.find((n) => n.id === 20)).toEqual( + expect.objectContaining({ id: 20, role: "button", attrs: { "aria-label": "收藏" } }), ); - expect(scene.nodes.find((n) => n.backendNodeId === 30)).toBeUndefined(); + expect(scene.nodes.find((n) => n.id === 30)).toBeUndefined(); const rendered = renderVom(scene); expect(rendered.text).toContain('@e1 button "收藏"'); expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([ @@ -1875,8 +1898,9 @@ describe("buildFrameVomScene", () => { backendDOMNodeId: 20, }, ]; - const captured: SceneFixture = { + const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1924,10 +1948,10 @@ describe("buildFrameVomScene", () => { ], }; - const scene = buildTestScene(axNodes, captured); + const scene = buildVomScene(axNodes, captured); expect(scene.activeScopeBlocks).toEqual([ { - triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id, + triggerId: 20, label: "Reviews (12)", lines: ["Jane - ear cups are small", "Bob - great sound"], }, @@ -1951,10 +1975,11 @@ describe("buildFrameVomScene", () => { backendDOMNodeId: 20, }, ]; - const scene = buildTestScene( + const scene = buildVomScene( axNodes, { viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -1987,11 +2012,7 @@ describe("buildFrameVomScene", () => { ); expect(scene.surfaces).toEqual([ - { - triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id, - triggerAction: "hover", - subItems: ["Shoes", "Bags"], - }, + { triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] }, ]); expect(renderVom(scene).text).toContain('@e1 button "Products" [hover first: Shoes | Bags]'); }); @@ -2012,10 +2033,11 @@ describe("buildFrameVomScene", () => { backendDOMNodeId: 21, }, ]; - const scene = buildTestScene( + const scene = buildVomScene( axNodes, { viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -2076,10 +2098,11 @@ describe("buildFrameVomScene", () => { backendDOMNodeId: 21, }, ]; - const scene = buildTestScene( + const scene = buildVomScene( axNodes, { viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -2124,11 +2147,7 @@ describe("buildFrameVomScene", () => { ); expect(scene.surfaces).toEqual([ - { - triggerId: scene.nodes.find((node) => node.backendNodeId === 21)!.id, - triggerAction: "hover", - subItems: ["My profile"], - }, + { triggerId: 21, triggerAction: "hover", subItems: ["My profile"] }, ]); }); @@ -2148,10 +2167,11 @@ describe("buildFrameVomScene", () => { backendDOMNodeId: 21, }, ]; - const scene = buildTestScene( + const scene = buildVomScene( axNodes, { viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -2202,11 +2222,7 @@ describe("buildFrameVomScene", () => { ); expect(scene.surfaces).toEqual([ - { - triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id, - triggerAction: "hover", - subItems: ["My profile", "Sign out"], - }, + { triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] }, ]); expect(renderVom(scene).text).toContain( '@e1 button "image" [hover first: My profile | Sign out]', @@ -2229,10 +2245,11 @@ describe("buildFrameVomScene", () => { backendDOMNodeId: 42, }, ]; - const scene = buildTestScene( + const scene = buildVomScene( axNodes, { viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -2302,8 +2319,9 @@ describe("buildFrameVomScene", () => { properties: [{ name: "inputType", value: { value: "credit-card" } }], }, ]; - const scene = buildTestScene(axNodes, { + const scene = buildVomScene(axNodes, { viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -2350,7 +2368,7 @@ describe("buildFrameVomScene", () => { ], }); - expect(scene.nodes.find((node) => node.backendNodeId === 20)).toEqual( + expect(scene.nodes.find((node) => node.id === 20)).toEqual( expect.objectContaining({ name: "Products [expanded]", attrs: expect.objectContaining({ @@ -2360,17 +2378,13 @@ describe("buildFrameVomScene", () => { }), ); expect(scene.activeScopeBlocks).toEqual([ - { - triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id, - label: "Products [expanded]", - lines: ["Shoes Bags"], - }, + { triggerId: 20, label: "Products [expanded]", lines: ["Shoes Bags"] }, ]); - expect(scene.nodes.find((node) => node.backendNodeId === 30)?.sensitive).toBe(true); + expect(scene.nodes.find((node) => node.id === 30)?.sensitive).toBe(true); }); it("aggregates AX virtual text into unnamed structural nodes", () => { - const scene = buildTestScene( + const scene = buildVomScene( [ { nodeId: "1", @@ -2394,6 +2408,7 @@ describe("buildFrameVomScene", () => { ], { viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -2420,12 +2435,13 @@ describe("buildFrameVomScene", () => { }, ); - expect(scene.nodes.find((node) => node.backendNodeId === 20)?.name).toBe("Inline only text"); + expect(scene.nodes.find((node) => node.id === 20)?.name).toBe("Inline only text"); }); it("marks captured dialog elements as modal without AX role or aria-modal", () => { - const scene = buildTestScene([], { + const scene = buildVomScene([], { viewport: { width: 640, height: 480 }, + iframeNodes: new Map(), excludedBackendNodeIds: new Set(), nodes: [ { @@ -2443,7 +2459,7 @@ describe("buildFrameVomScene", () => { expect(scene.nodes[0]).toEqual( expect.objectContaining({ - backendNodeId: 50, + id: 50, modal: true, }), ); @@ -2472,8 +2488,9 @@ describe("buildFrameVomScene", () => { backendDOMNodeId: 202, }, ]; - const captured: SceneFixture = { + const captured: CapturedViewModel = { viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), excludedBackendNodeIds: new Set([200, 201, 202]), nodes: [ { @@ -2489,9 +2506,9 @@ describe("buildFrameVomScene", () => { ], }; - const scene = buildTestScene(axNodes, captured); - expect(scene.nodes.find((n) => n.backendNodeId === 202)).toBeUndefined(); - expect(scene.nodes.find((n) => n.backendNodeId === 2)).toEqual( + const scene = buildVomScene(axNodes, captured); + expect(scene.nodes.find((n) => n.id === 202)).toBeUndefined(); + expect(scene.nodes.find((n) => n.id === 2)).toEqual( expect.objectContaining({ role: "button", name: "Real Button" }), ); }); @@ -2519,12 +2536,13 @@ describe("buildFrameVomScene", () => { backendDOMNodeId: 203, }, ]; - const scene = buildTestScene(axNodes, { + const scene = buildVomScene(axNodes, { viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), excludedBackendNodeIds: new Set([200]), nodes: [], }); - expect(scene.nodes.find((n) => n.backendNodeId === 203)).toBeUndefined(); + expect(scene.nodes.find((n) => n.id === 203)).toBeUndefined(); }); }); @@ -2637,7 +2655,6 @@ describe("handleSnapshot", () => { strings, documents: [ { - frameId: "root", nodes: { parentIndex: [-1, 0], nodeName: [i("body"), i("button")], @@ -2733,7 +2750,6 @@ describe("handleSnapshot", () => { strings, documents: [ { - frameId: "root", nodes: { parentIndex: [-1, 0], nodeName: [i("body"), i("button")], @@ -2832,7 +2848,6 @@ describe("handleSnapshot", () => { strings, documents: [ { - frameId: "root", nodes: { parentIndex: [-1, 0], nodeName: [i("body"), i("button")], @@ -3101,7 +3116,6 @@ describe("handleSnapshot", () => { strings: S, documents: [ { - frameId: "root", nodes: { parentIndex: [-1, 0, 1, 2], nodeName: [i("html"), i("body"), i("div"), i("input")], @@ -3159,7 +3173,6 @@ describe("handleSnapshot", () => { strings: S, documents: [ { - frameId: "root", nodes: { parentIndex: [-1, 0, 1, 1, 1, 1], nodeName: [i("html"), i("body"), i("input"), i("input"), i("input"), i("input")], @@ -3311,17 +3324,12 @@ describe("handleSnapshot", () => { if (method === "Runtime.releaseObject") return {}; } if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {}; - if (method === "DOMSnapshot.captureSnapshot") - return { ...snapshot, documents: [snapshot.documents[0]] }; + if (method === "DOMSnapshot.captureSnapshot") return snapshot; if (method === "Accessibility.getFullAXTree") return { nodes: mainAx }; throw new Error(`unexpected root CDP method ${method}`); }); const sendToTarget = vi.fn(async (_target, method: string) => { - if (method === "Page.getLayoutMetrics" && ownerGeometry !== "unavailable") - return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } }; - if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {}; - if (method === "DOMSnapshot.captureSnapshot") - return { ...snapshot, documents: [snapshot.documents[1]] }; + if (method === "Accessibility.enable") return {}; if (method === "Accessibility.getFullAXTree") return { nodes: childAx }; throw new Error(`unexpected child CDP method ${method}`); }); @@ -3353,66 +3361,6 @@ describe("handleSnapshot", () => { }; } - it.each([ - false, - true, - ])("preserves external-link hints without graph metadata (snapshot URL: %s)", async (hasSnapshotUrl) => { - const snapshot = { - strings: ["body", "a", "href", "//outside.test/docs", "https://page.test/start"], - documents: [ - { - frameId: "main", - ...(hasSnapshotUrl ? { documentURL: 4 } : {}), - nodes: { - backendNodeId: [1, 2], - parentIndex: [-1, 0], - nodeName: [0, 1], - attributes: [[], [2, 3]], - }, - layout: { - nodeIndex: [0, 1], - bounds: [ - [0, 0, 1000, 800], - [10, 20, 100, 40], - ], - }, - }, - ], - }; - const ax: CdpAxNode[] = [ - { - nodeId: "root", - backendDOMNodeId: 1, - role: { type: "role", value: "RootWebArea" }, - childIds: ["link"], - }, - { - nodeId: "link", - parentId: "root", - backendDOMNodeId: 2, - role: { type: "role", value: "link" }, - name: { type: "computedString", value: "Docs" }, - }, - ]; - const deps = makeOverlayDeps(ax, snapshot, VP_METRICS); - const cdp: CdpRunner = { - ...deps.cdp, - getFrameGraph: async () => { - throw new Error("graph unavailable"); - }, - }; - const result = await captureVomObservation( - cdp, - 4, - hasSnapshotUrl ? undefined : "https://page.test/start", - ); - expect(result.text).toContain("outside.test"); - expect(result.text).toContain("Docs"); - expect( - deps.send.mock.calls.filter(([, method]) => method === "DOMSnapshot.captureSnapshot"), - ).toHaveLength(1); - }); - it("exposes the frame-aware observation result for recording", async () => { const ax: CdpAxNode[] = [ { @@ -3482,7 +3430,6 @@ describe("handleSnapshot", () => { strings, documents: [ { - frameId: "root", nodes: { parentIndex: [-1, 0, 1, 1, 3], nodeName: [index("html"), index("body"), index("button"), index("div"), index("#text")], @@ -3807,7 +3754,6 @@ describe("handleSnapshot", () => { strings: S, documents: [ { - frameId: "root", scrollOffsetX: 0, scrollOffsetY: 0, nodes: { @@ -3834,7 +3780,6 @@ describe("handleSnapshot", () => { }, // documents[1]: cross-origin login form { - frameId: "child", scrollOffsetX: 0, scrollOffsetY: 0, nodes: { @@ -4013,7 +3958,6 @@ describe("handleSnapshot", () => { strings: S, documents: [ { - frameId: "root", nodes: { parentIndex: [-1, 0, 1, 0, 3, 4, 5], nodeName: [ diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index 00e6c42..61ae7dd 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -1,4 +1,3 @@ -import type { CapturedSceneInput } from "./vom/facts"; // Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`, // and semantic `tool.observe` (design §7). Each handler resolves the target // tab (defaulting to the Agent Window's active tab when omitted) and @@ -36,6 +35,7 @@ import { attachDialogs, markDialogCursor } from "./dialogs"; import { rpcError } from "./errors"; import { resolveNodeGeometry } from "./frame-geometry"; import { screenshotPageRect } from "./geometry/coordinate-types"; +import { cssViewport, GeometryContext } from "./geometry/frame-context"; import { type ChromeTabsApi, enforceToolTargetScope, @@ -49,9 +49,15 @@ import { type ToolEffect, } from "./shared"; import { resolveSnapshotRef } from "./snapshot-ref"; -import { type CapturedNode, type CapturedSurfaceProbe, probeHoverSurfaces } from "./vom/capture"; -import { captureObservationFacts, semanticCapture } from "./vom/capture-coordinator"; -import type { FrameDocument as CapturedFrameDocument } from "./vom/frame-document"; +import { + type CapturedNode, + type CapturedSurfaceProbe, + type CapturedViewModel, + captureViewModel, + collectOverlayExcludedBackendIds, + probeHoverSurfaces, +} from "./vom/capture"; +import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture"; import { withOverlayBypass } from "./vom/hover-perception"; import { probeTooltipNames } from "./vom/name-enrichment"; import { @@ -525,7 +531,7 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A function buildConditionalSurfaces( nodes: VomNode[], - captured: CapturedSceneInput, + captured: CapturedViewModel, probes: CapturedSurfaceProbe[], ): CondSurface[] { if (probes.length === 0) return []; @@ -655,9 +661,163 @@ export interface BuildVomSceneOptions { export type VomFrameDocument = CapturedFrameDocument; +function legacyFrameDocuments( + axNodes: CdpAxNode[], + captured: CapturedViewModel, + pageUrl?: string, + rootTarget: VomFrameDocument["target"] = { tabId: 0 }, +): VomFrameDocument[] { + const rootFrameId = captured.rootFrameId ?? "root"; + const documents: VomFrameDocument[] = [ + { + frameId: rootFrameId, + contextScopeId: rootFrameId, + target: rootTarget, + ...(pageUrl ? { url: pageUrl } : {}), + axNodes: [], + domNodes: captured.nodes.map((node) => ({ ...node, frameId: node.frameId ?? rootFrameId })), + excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(rootFrameId), + }, + ]; + const pending = [...captured.iframeNodes.entries()]; + let progress = true; + let nextSyntheticFrame = 1; + while (pending.length > 0 && progress) { + progress = false; + for (let index = pending.length - 1; index >= 0; index -= 1) { + const [ownerBackendNodeId, domNodes] = pending[index]; + const parent = documents.find((document) => + document.domNodes.some((node) => node.backendNodeId === ownerBackendNodeId), + ); + if (!parent) continue; + const frameId = + domNodes.find((node) => node.frameId)?.frameId ?? `legacy-frame-${nextSyntheticFrame++}`; + documents.push({ + frameId, + parentFrameId: parent.frameId, + ownerBackendNodeId, + contextScopeId: frameId, + target: parent.target, + axNodes: [], + domNodes: domNodes.map((node) => ({ ...node, frameId: node.frameId ?? frameId })), + excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(frameId), + }); + pending.splice(index, 1); + progress = true; + } + } + + const documentByFrameId = new Map(documents.map((document) => [document.frameId, document])); + const backendFrame = new Map(); + const childFrameByOwner = new Map(); + for (const document of documents) { + if (document.ownerBackendNodeId !== undefined) { + childFrameByOwner.set(document.ownerBackendNodeId, document.frameId); + } + for (const node of document.domNodes) backendFrame.set(node.backendNodeId, document.frameId); + } + const axById = new Map(axNodes.map((node) => [node.nodeId, node])); + const ownership = new Map(); + const resolving = new Set(); + const frameForAx = (node: CdpAxNode): string => { + const cached = ownership.get(node.nodeId); + if (cached) return cached; + let frameId: string | undefined; + if (node.frameId && documentByFrameId.has(node.frameId)) frameId = node.frameId; + if (!frameId && typeof node.backendDOMNodeId === "number") { + frameId = backendFrame.get(node.backendDOMNodeId); + } + if (!frameId && node.parentId && !resolving.has(node.nodeId)) { + const parent = axById.get(node.parentId); + if (parent) { + frameId = + typeof parent.backendDOMNodeId === "number" + ? childFrameByOwner.get(parent.backendDOMNodeId) + : undefined; + if (!frameId) { + resolving.add(node.nodeId); + frameId = frameForAx(parent); + resolving.delete(node.nodeId); + } + } + } + frameId ??= rootFrameId; + ownership.set(node.nodeId, frameId); + return frameId; + }; + for (const node of axNodes) frameForAx(node); + for (const node of axNodes) { + const frameId = ownership.get(node.nodeId) ?? rootFrameId; + const document = documentByFrameId.get(frameId) ?? documents[0]; + document.axNodes.push({ + ...node, + frameId, + ...(node.parentId && ownership.get(node.parentId) === frameId + ? { parentId: node.parentId } + : { parentId: undefined }), + ...(node.childIds + ? { childIds: node.childIds.filter((childId) => ownership.get(childId) === frameId) } + : {}), + }); + } + return documents; +} + +function withLegacyBackendIds(scene: VomScene): VomScene { + const used = new Set(); + const idMap = new Map(); + let nextVirtualId = -1; + for (const node of scene.nodes) { + const preferred = node.backendNodeId; + const id = preferred !== undefined && !used.has(preferred) ? preferred : nextVirtualId--; + used.add(id); + idMap.set(node.id, id); + } + return { + ...scene, + nodes: scene.nodes.map((node) => ({ + ...node, + id: idMap.get(node.id) as number, + parentId: node.parentId === null ? null : (idMap.get(node.parentId) ?? null), + ...(node.domParentId !== undefined + ? { domParentId: node.domParentId === null ? null : (idMap.get(node.domParentId) ?? null) } + : {}), + ...(node.domAncestorIds + ? { domAncestorIds: node.domAncestorIds.flatMap((id) => idMap.get(id) ?? []) } + : {}), + })), + ...(scene.surfaces + ? { + surfaces: scene.surfaces.map((surface) => ({ + ...surface, + triggerId: idMap.get(surface.triggerId) ?? surface.triggerId, + })), + } + : {}), + ...(scene.activeScopeBlocks + ? { + activeScopeBlocks: scene.activeScopeBlocks.map((block) => ({ + ...block, + triggerId: idMap.get(block.triggerId) ?? block.triggerId, + })), + } + : {}), + }; +} + +export function buildVomScene( + axNodes: CdpAxNode[], + captured: CapturedViewModel, + options: BuildVomSceneOptions = {}, +): VomScene { + return withLegacyBackendIds( + buildFrameVomScene(legacyFrameDocuments(axNodes, captured, options.pageUrl), captured, options), + ); +} + export function buildFrameVomScene( documents: VomFrameDocument[], - captured: CapturedSceneInput, + captured: CapturedViewModel, options: BuildVomSceneOptions = {}, ): VomScene { const scene = buildSemanticVomScene({ @@ -673,7 +833,7 @@ export function buildFrameVomScene( function attachCapturedSceneAnnotations( scene: VomScene, documents: VomFrameDocument[], - captured: CapturedSceneInput, + captured: CapturedViewModel, surfaceProbes: CapturedSurfaceProbe[], ): VomScene { const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId); @@ -823,6 +983,45 @@ export async function handleGetHtml( } } +function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel { + return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() }; +} + +async function fallbackCapturedViewModel( + cdp: CdpRunner, + tabId: number, + geometry: GeometryContext, + signal?: AbortSignal, +): Promise { + throwIfAborted(signal, "observation"); + let viewport = { width: 0, height: 0 }; + try { + const metrics = await geometry.layoutMetrics({ tabId }); + throwIfAborted(signal, "observation"); + const source = cssViewport(metrics); + viewport = { width: source.width, height: source.height }; + } catch (error) { + if (isAbortError(error)) throw error; + } + const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal); + throwIfAborted(signal, "observation"); + return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds }; +} + +async function captureForVom( + cdp: CdpRunner, + tabId: number, + options: CaptureVomObservationOptions, + geometry: GeometryContext, +): Promise { + try { + return await captureViewModel(cdp, tabId, { signal: options.signal, geometry }); + } catch (error) { + if (isAbortError(error)) throw error; + return fallbackCapturedViewModel(cdp, tabId, geometry, options.signal); + } +} + export interface HoverProbeOutcome { /** Whether any active hover was dispatched during this observation. */ performed: boolean; @@ -855,7 +1054,7 @@ const NO_HOVER_PROBES: HoverProbeOutcome = { async function runHoverProbes( cdp: CdpRunner, tabId: number, - captured: CapturedSceneInput, + captured: CapturedViewModel, documents: VomFrameDocument[], staticSemantics: ReturnType, options: CaptureVomObservationOptions, @@ -894,9 +1093,21 @@ export async function captureVomObservation( throwIfAborted(options.signal, "observation"); await cdp.ensureAttachedToUrl?.(tabId, url); throwIfAborted(options.signal, "observation"); - const facts = await captureObservationFacts(cdp, tabId, options.signal, url); - const { captured, documents: normalizedDocuments } = semanticCapture(facts); + const geometry = new GeometryContext(cdp, tabId, undefined, options.signal); + const captured = await captureForVom(cdp, tabId, options, geometry); throwIfAborted(options.signal, "observation"); + const documents = await captureFrameData( + cdp, + tabId, + captured, + options.signal, + geometry, + ); + throwIfAborted(options.signal, "observation"); + const normalizedDocuments = + documents.length === 1 && captured.iframeNodes.size > 0 + ? legacyFrameDocuments(documents[0].axNodes, captured, url, documents[0].target) + : documents; const semanticGraph = buildSemanticGraph({ documents: normalizedDocuments, viewport: captured.viewport, @@ -926,7 +1137,7 @@ export async function captureVomObservation( ); // Reserve space using the renderer's character-based token estimate. Like VOM // headers, this integrity notice remains visible even under a tiny token budget. - const geometryNotice = facts.issues.some((issue) => issue.stage === "geometry") + const geometryNotice = captured.frameGeometryIssues?.length ? "@warning iframe geometry incomplete: some frame content has no top-level coordinates.\n" : ""; const rendered = renderVom(decoratedScene, { diff --git a/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts b/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts deleted file mode 100644 index bc82868..0000000 --- a/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts +++ /dev/null @@ -1,914 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; -import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph"; -import { cdpTargetKey } from "@/browser-driver/frame-graph"; -import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge"; -import type { CdpRunner } from "../../shared"; -import { captureObservationFacts, semanticCapture } from "../capture-coordinator"; -import { buildSemanticGraph } from "../semantic-graph/build"; -import { REQUESTED_STYLES } from "../snapshot"; - -function fixture( - options: { frames?: CdpFrame[]; fail?: string; omitDocument?: string; overlay?: string } = {}, -) { - const frames = options.frames ?? [ - { frameId: "main", target: { tabId: 4 } }, - { - frameId: "same", - parentFrameId: "main", - ownerBackendNodeId: 3, - target: { tabId: 4 }, - }, - { - frameId: "nested", - parentFrameId: "same", - ownerBackendNodeId: 13, - target: { tabId: 4 }, - }, - { - frameId: "remote", - parentFrameId: "main", - ownerBackendNodeId: 4, - target: { tabId: 4, sessionId: "remote" }, - }, - ]; - const elements = new Map( - frames.map((frame, i) => [frame.frameId, frame.target.sessionId ? 1 : i * 10 + 1]), - ); - const logs: Array<{ target: CdpTarget; method: string; params: Record }> = []; - const send = vi.fn( - async (target: CdpTarget, method: string, params: object = {}): Promise => { - logs.push({ target, method, params: params as Record }); - const args = params as Record; - if (options.fail === `${target.sessionId ?? "main"}:${method}`) - throw new Error("fixture failure"); - let result: unknown = {}; - if (method === "Page.getLayoutMetrics") - result = { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } }; - if (method === "DOMSnapshot.captureSnapshot") { - expect((params as { computedStyles: unknown }).computedStyles).toEqual(REQUESTED_STYLES); - result = { - strings: [ - "#document", - "html", - "button", - OVERLAY_HOST_MARKER_ATTR, - "", - "visible", - "1", - "static", - "auto", - ], - documents: frames - .filter( - (frame) => - cdpTargetKey(frame.target) === cdpTargetKey(target) && - frame.frameId !== options.omitDocument, - ) - .map((frame) => { - const element = elements.get(frame.frameId)!; - return { - frameId: frame.frameId, - nodes: { - backendNodeId: [element - 1, element + 10000, element, element + 1], - nodeName: [0, 1, 1, 2], - nodeType: [9, 10, 1, 1], - parentIndex: [-1, 0, 0, 2], - attributes: [[], [], [], frame.frameId === options.overlay ? [3, 4] : []], - }, - layout: { - nodeIndex: [2, 3], - bounds: [ - [0, 0, 1000, 800], - [10, 20, 100, 40], - ], - styles: [ - [7, 8, 8, 5, 6], - [7, 8, 8, 5, 6], - ], - }, - }; - }), - }; - } - if (method === "Accessibility.getFullAXTree") { - const frameId = String(args.frameId); - result = { - nodes: [ - { - nodeId: `${frameId}-button`, - frameId, - backendDOMNodeId: elements.get(frameId)! + 1, - role: { value: "button" }, - name: { value: frameId }, - }, - ], - }; - } - return result as T; - }, - ); - const cdp: CdpRunner = { - getFrameGraph: async () => ({ rootFrameId: frames[0].frameId, frames }), - send: (tabId, method, params) => - (send as NonNullable)({ tabId }, method, params), - sendToTarget: send as NonNullable, - }; - return { cdp, logs, elements }; -} - -describe("captureObservationFacts", () => { - it("collects each target once and scopes equal backend IDs", async () => { - const { cdp, logs } = fixture(); - const facts = await captureObservationFacts(cdp, 4); - expect(logs.filter((call) => call.method === "DOMSnapshot.captureSnapshot")).toHaveLength(2); - expect(logs.filter((call) => call.method === "Accessibility.enable")).toHaveLength(2); - expect(logs.filter((call) => call.method === "Accessibility.getFullAXTree")).toHaveLength(4); - const main = facts.documents.find((doc) => doc.frame.frameId === "main")!; - const remote = facts.documents.find((doc) => doc.frame.frameId === "remote")!; - expect(main.index.nodes.get(2)).not.toBe(remote.index.nodes.get(2)); - expect(main.index.nodes.get(2)?.parentBackendNodeId).toBe(1); - expect(main.index.nodes.get(10001)?.nodeType).toBe(10); - expect(remote.axNodes[0].frameId).toBe("remote"); - expect(main.index.nodes.get(2)).toBe(main.domNodes.find((node) => node.backendNodeId === 2)); - expect(facts.finishedAt).toBeGreaterThanOrEqual(facts.startedAt); - }); - - it("does not retry failed or missing documents and retains valid AX-only semantics", async () => { - const { cdp, logs } = fixture({ - fail: "remote:DOMSnapshot.captureSnapshot", - omitDocument: "same", - }); - const facts = await captureObservationFacts(cdp, 4); - const remote = facts.documents.find((doc) => doc.frame.frameId === "remote")!; - expect(remote.domNodes).toHaveLength(0); - expect(remote.axNodes).toHaveLength(1); - expect(logs.filter((call) => call.method === "DOMSnapshot.captureSnapshot")).toHaveLength(2); - expect(facts.issues).toContainEqual( - expect.objectContaining({ frameId: "same", stage: "dom", reason: "capture-unavailable" }), - ); - expect(facts.issues).toContainEqual( - expect.objectContaining({ target: { tabId: 4, sessionId: "remote" }, stage: "dom" }), - ); - }); - - it("excludes overlay AX in its own document without excluding equal IDs elsewhere", async () => { - const facts = await captureObservationFacts(fixture({ overlay: "remote" }).cdp, 4); - const { documents } = semanticCapture(facts); - const graph = buildSemanticGraph({ - documents, - viewport: facts.viewport, - rootFrameId: facts.rootFrameId, - }); - expect( - [...graph.nodes.values()].find( - (node) => node.frameId === "remote" && node.backendNodeId === 2, - )?.excluded, - ).toBe(true); - expect( - [...graph.nodes.values()].find((node) => node.frameId === "main" && node.backendNodeId === 2) - ?.excluded, - ).toBe(false); - }); - - it.each([ - "cancel", - "worker-abort", - ] as const)("joins target cleanup and stops claiming targets after %s", async (mode) => { - const frames = Array.from({ length: 5 }, (_, i) => ({ - frameId: `f${i}`, - target: { tabId: 4, ...(i ? { sessionId: `s${i}` } : {}) }, - })); - const { cdp } = fixture({ frames }); - const original = cdp.sendToTarget!; - const pending = new Map void; reject: (error: Error) => void }>(); - const calls: Array<{ target: string; method: string }> = []; - let release: (() => void) | undefined; - let released = false; - const send: NonNullable = async ( - target: CdpTarget, - method: string, - params?: object, - ) => { - calls.push({ target: target.sessionId ?? "main", method }); - if (method === "DOMSnapshot.captureSnapshot" && target.sessionId) { - await new Promise((resolve, reject) => - pending.set(target.sessionId!, { resolve, reject }), - ); - } - if (method === "Runtime.releaseObjectGroup") { - await new Promise((resolve) => { - release = resolve; - }); - released = true; - } - const result = await original(target, method, params); - if (method === "DOMSnapshot.captureSnapshot" && !target.sessionId) - (result as { strings: string[] }).strings[2] = "input"; - return result; - }; - cdp.sendToTarget = send; - cdp.send = (tabId, method, params) => send({ tabId }, method, params); - const controller = new AbortController(); - let settled = false; - const capture = captureObservationFacts(cdp, 4, controller.signal); - const rejected = expect(capture).rejects.toMatchObject({ name: "AbortError" }); - void capture.then( - () => { - settled = true; - }, - () => { - settled = true; - }, - ); - await vi.waitFor(() => { - expect(pending.size).toBe(3); - expect(release).toBeDefined(); - }); - if (mode === "cancel") controller.abort(); - // Cancellation may race with an ordinary transport rejection. Without an - // external signal, an escaping AbortError still terminates the worker pool. - pending - .get("s1")! - .reject( - mode === "cancel" - ? new Error("transport closed") - : new DOMException("target aborted", "AbortError"), - ); - pending.get("s2")!.resolve(); - pending.get("s3")!.resolve(); - await vi.waitFor(() => { - if (mode === "worker-abort") - expect( - calls.some( - (call) => call.target === "s3" && call.method === "Accessibility.getFullAXTree", - ), - ).toBe(true); - else - expect( - calls.some( - (call) => call.target === "s3" && call.method === "DOMSnapshot.captureSnapshot", - ), - ).toBe(true); - }); - // Allow worker rejection and the outer promise chain to run while cleanup - // is deliberately held. The old fail-fast pool settles here. - await new Promise((resolve) => setTimeout(resolve, 0)); - expect(settled).toBe(false); - expect(released).toBe(false); - expect(calls.some((call) => call.target === "s4")).toBe(false); - if (mode === "cancel") - expect( - calls.some( - (call) => call.method === "Accessibility.enable" || call.method === "DOM.getDocument", - ), - ).toBe(false); - release!(); - await rejected; - expect(released).toBe(true); - expect(calls.some((call) => call.target === "s4")).toBe(false); - }); - - it("bounds collection across many targets and stops scheduling after cancellation", async () => { - const frames = Array.from({ length: 12 }, (_, i) => ({ - frameId: `f${i}`, - target: { tabId: 4, ...(i ? { sessionId: `s${i}` } : {}) }, - })); - const { cdp } = fixture({ frames }); - let active = 0, - peak = 0; - const original = cdp.sendToTarget!; - const send: NonNullable = async (target, method, params) => { - active++; - peak = Math.max(peak, active); - try { - await new Promise((resolve) => setTimeout(resolve, 1)); - return await original(target, method, params); - } finally { - active--; - } - }; - cdp.sendToTarget = send; - cdp.send = (tabId, method, params) => send({ tabId }, method, params); - await captureObservationFacts(cdp, 4); - // This fixture has no frame edges, so no concurrent owner requests. - expect(peak).toBeLessThanOrEqual(4); - expect(peak).toBeGreaterThan(1); - const controller = new AbortController(); - controller.abort(); - await expect(captureObservationFacts(cdp, 4, controller.signal)).rejects.toMatchObject({ - name: "AbortError", - }); - expect(active).toBe(0); - }); -}); - -function childSnapshot(frameId: string, backendNodeId: number) { - const strings = [frameId, "body", "button", "static", "auto", "pointer"]; - return { - strings, - documents: [ - { - frameId, - nodes: { - parentIndex: [-1, 0], - nodeName: [1, 2], - backendNodeId: [backendNodeId - 1, backendNodeId], - attributes: [[], []], - }, - layout: { - nodeIndex: [0, 1], - styles: [ - [3, 4, 4], - [3, 4, 5], - ], - bounds: [ - [0, 0, 300, 200], - [10, 20, 100, 40], - ], - paintOrders: [0, 1], - }, - }, - ], - }; -} - -describe("OOPIF capture", () => { - it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => { - const sendToTarget = vi.fn(async (target, method) => { - if (method === "Page.getLayoutMetrics") { - return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } }; - } - if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {}; - if (method === "DOMSnapshot.captureSnapshot") { - return target.sessionId === "left-session" - ? childSnapshot("left", 101) - : childSnapshot("right", 201); - } - if (method === "Accessibility.getFullAXTree") return { nodes: [] }; - throw new Error(`unexpected ${method}`); - }); - const cdp: CdpRunner = { - send: vi.fn(async (_tabId, method, params) => { - if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {}; - if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("main", 1); - if (method === "Accessibility.getFullAXTree") return { nodes: [] }; - if (method === "DOM.getBoxModel") { - const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId; - const x = backendNodeId === 10 ? 50 : 500; - return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } }; - } - if (method === "Page.getLayoutMetrics") { - return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } }; - } - throw new Error(`unexpected root ${method}`); - }) as CdpRunner["send"], - sendToTarget: sendToTarget as unknown as NonNullable, - getFrameGraph: vi.fn(async () => ({ - rootFrameId: "main", - frames: [ - { frameId: "main", target: { tabId: 4 } }, - { - frameId: "left", - parentFrameId: "main", - ownerBackendNodeId: 10, - target: { tabId: 4, sessionId: "left-session" }, - }, - { - frameId: "right", - parentFrameId: "main", - ownerBackendNodeId: 20, - target: { tabId: 4, sessionId: "right-session" }, - }, - ], - })), - }; - - const { documents: trees } = semanticCapture(await captureObservationFacts(cdp, 4)); - - expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]); - expect( - trees - .find((doc) => doc.frameId === "left") - ?.domNodes?.find((node) => node.backendNodeId === 101)?.rect, - ).toEqual({ x: 60, y: 120, w: 100, h: 40 }); - expect( - trees - .find((doc) => doc.frameId === "right") - ?.domNodes?.find((node) => node.backendNodeId === 201)?.rect, - ).toEqual({ x: 510, y: 120, w: 100, h: 40 }); - expect( - new Map( - trees - .filter((doc) => doc.ownerBackendNodeId !== undefined) - .map((doc) => [doc.frameId, doc.ownerBackendNodeId]), - ), - ).toEqual( - new Map([ - ["left", 10], - ["right", 20], - ]), - ); - expect( - new Map( - trees.filter((doc) => doc.parentFrameId).map((doc) => [doc.frameId, doc.parentFrameId]), - ), - ).toEqual( - new Map([ - ["left", "main"], - ["right", "main"], - ]), - ); - }); - - it("retains nested owner failures when merging a captured OOPIF", async () => { - const child = childSnapshot("child", 101); - const nested = childSnapshot("nested", 201); - const document = child.documents[0]; - const snapshot = { - strings: child.strings, - documents: [ - { - ...document, - nodes: { ...document.nodes, contentDocumentIndex: { index: [1], value: [1] } }, - }, - nested.documents[0], - ], - }; - const reply = async (_target: unknown, method: string) => { - if (method === "Page.getLayoutMetrics") - return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200 } }; - if (method === "DOMSnapshot.captureSnapshot") - return typeof _target === "number" ? childSnapshot("main", 11) : snapshot; - if (method === "DOM.getBoxModel") - return { model: { content: [50, 100, 350, 100, 350, 300, 50, 300] } }; - if (method === "DOM.resolveNode") throw new Error("nested owner replaced"); - return {}; - }; - const cdp: CdpRunner = { - send: vi.fn(reply) as CdpRunner["send"], - sendToTarget: vi.fn(reply) as NonNullable, - getFrameGraph: vi.fn(async () => ({ - rootFrameId: "main", - frames: [ - { frameId: "main", target: { tabId: 4 } }, - { - frameId: "nested", - parentFrameId: "child", - ownerBackendNodeId: 101, - target: { tabId: 4, sessionId: "child-session" }, - }, - { - frameId: "child", - parentFrameId: "main", - ownerBackendNodeId: 10, - target: { tabId: 4, sessionId: "child-session" }, - }, - ], - })), - }; - const captured = await captureObservationFacts(cdp, 4); - expect( - captured.issues - .filter((issue) => issue.projectionIssue) - .map((issue) => issue.projectionIssue), - ).toEqual([ - { - status: "unavailable", - source: { target: { tabId: 4, sessionId: "child-session" }, frameId: "nested" }, - ownerBackendNodeId: 101, - }, - ]); - expect( - captured.documents.find((doc) => doc.frame.frameId === "nested")?.domNodes[1], - ).toMatchObject({ - rect: null, - localRect: { x: 10, y: 20, w: 100, h: 40 }, - }); - }); - - it("keeps OOPIF semantics when viewport projection is unavailable", async () => { - const cdp: CdpRunner = { - send: vi.fn(async (_tabId, method) => { - if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {}; - if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("main", 1); - if (method === "Accessibility.getFullAXTree") return { nodes: [] }; - if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable"); - if (method === "Page.getLayoutMetrics") { - return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } }; - } - throw new Error(`unexpected root ${method}`); - }) as CdpRunner["send"], - sendToTarget: vi.fn(async (_target, method) => { - if (method === "Page.getLayoutMetrics") throw new Error("viewport unavailable"); - if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {}; - if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("child", 101); - if (method === "Accessibility.getFullAXTree") { - return { - nodes: [ - { - nodeId: "button", - backendDOMNodeId: 101, - role: { type: "role", value: "button" }, - name: { type: "computedString", value: "Continue" }, - }, - ], - }; - } - throw new Error(`unexpected ${method}`); - }) as unknown as NonNullable, - getFrameGraph: vi.fn(async () => ({ - rootFrameId: "main", - frames: [ - { frameId: "main", target: { tabId: 4 } }, - { - frameId: "child", - parentFrameId: "main", - ownerBackendNodeId: 10, - target: { tabId: 4, sessionId: "child-session" }, - }, - ], - })), - }; - - const { documents } = semanticCapture(await captureObservationFacts(cdp, 4)); - const childDocument = documents.find((document) => document.frameId === "child"); - - expect(childDocument?.axNodes).toEqual([ - expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }), - ]); - expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual( - expect.objectContaining({ rect: null, localRect: { x: 10, y: 20, w: 100, h: 40 } }), - ); - }); -}); - -// Six sibling frames and one nested frame exercise scheduling through the real capture entry. -function siblingCaptureFixture( - beforeReply: (method: string, params: Record) => Promise = async () => {}, -) { - const document = (id: number, owners: number[], childIndexes: number[]) => ({ - frameId: `frame-${id}`, - nodes: { - parentIndex: [-1, ...owners.map(() => 0)], - nodeName: [0, ...owners.map(() => 1)], - backendNodeId: [1000 + id, ...owners], - attributes: [[], ...owners.map(() => [])], - contentDocumentIndex: { index: owners.map((_, i) => i + 1), value: childIndexes }, - }, - layout: { - nodeIndex: [0, ...owners.map((_, i) => i + 1)], - bounds: [[0, 0, 200, 100], ...owners.map(() => [0, 0, 200, 100])], - }, - }); - const snapshot = { - strings: ["body", "iframe"], - documents: [ - document(0, [100, 101, 102, 103, 104, 105], [1, 2, 3, 4, 5, 6]), - document(1, [200], [7]), - ...Array.from({ length: 6 }, (_, i) => document(i + 2, [], [])), - ], - }; - let active = 0; - let peak = 0; - const send = vi.fn(async (_tabId: number, method: string, params: object = {}) => { - const args = params as Record; - active++; - peak = Math.max(peak, active); - try { - await beforeReply(method, args); - if (method === "DOMSnapshot.captureSnapshot") return snapshot; - if (method === "Page.getLayoutMetrics") - return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } }; - if (method === "DOM.getBoxModel") - return { model: { content: [0, 0, 200, 0, 200, 100, 0, 100] } }; - if (method === "DOM.resolveNode") return { object: { objectId: String(args.backendNodeId) } }; - if (method === "Runtime.callFunctionOn") - return { result: { value: { width: 200, height: 100 } } }; - return {}; - } finally { - active--; - } - }); - return { - snapshot, - cdp: { send: send as CdpRunner["send"] }, - send, - peak: () => peak, - active: () => active, - measured: () => - send.mock.calls - .filter(([, method]) => method === "DOM.getBoxModel") - .map(([, , params]) => (params as { backendNodeId: number }).backendNodeId), - }; -} - -describe("snapshot document provenance", () => { - it.each([ - "new-child", - "new-root", - "stale-owner", - ])("retains current snapshot evidence with %s graph data", async (mode) => { - const f = siblingCaptureFixture(); - const cdp: CdpRunner = { - ...f.cdp, - getFrameGraph: async () => ({ - rootFrameId: mode === "new-root" ? "old-root" : "frame-0", - frames: [ - { frameId: mode === "new-root" ? "old-root" : "frame-0", target: { tabId: 4 } }, - ...(mode === "stale-owner" - ? [ - { - frameId: "frame-1", - parentFrameId: "frame-0", - ownerBackendNodeId: 999, - target: { tabId: 4 }, - }, - ] - : []), - ], - }), - }; - const facts = await captureObservationFacts(cdp, 4); - expect(facts.rootFrameId).toBe("frame-0"); - expect(facts.documents.find((doc) => doc.frame.frameId === "frame-1")).toMatchObject({ - frame: { ownerBackendNodeId: 100 }, - domNodes: [ - expect.objectContaining({ rect: { x: 0, y: 0, w: 200, h: 100 } }), - expect.anything(), - ], - }); - expect( - facts.documents.find((doc) => doc.frame.frameId === "frame-7")?.domNodes[0].rect, - ).not.toBeNull(); - expect(f.measured()).not.toContain(999); - expect( - f.send.mock.calls.filter(([, method]) => method === "DOMSnapshot.captureSnapshot"), - ).toHaveLength(1); - expect( - f.send.mock.calls.some( - ([, method, params]) => - method === "Accessibility.getFullAXTree" && - (params as { frameId?: string }).frameId === "frame-7", - ), - ).toBe(true); - }); - - it("uses snapshot URLs before graph URLs and limits the caller URL to the root", async () => { - const f = siblingCaptureFixture(); - f.snapshot.strings.push("https://current.test/page", "https://child.test/page"); - Object.assign(f.snapshot.documents[0], { documentURL: 2 }); - Object.assign(f.snapshot.documents[1], { documentURL: 3 }); - const facts = await captureObservationFacts( - { - ...f.cdp, - getFrameGraph: async () => ({ - rootFrameId: "frame-0", - frames: [{ frameId: "frame-0", target: { tabId: 4 }, url: "https://old.test/" }], - }), - }, - 4, - undefined, - "https://caller.test/", - ); - expect(facts.documents.find((doc) => doc.frame.frameId === "frame-0")?.frame.url).toBe( - "https://current.test/page", - ); - expect(facts.documents.find((doc) => doc.frame.frameId === "frame-1")?.frame.url).toBe( - "https://child.test/page", - ); - expect( - facts.documents.find((doc) => doc.frame.frameId === "frame-2")?.frame.url, - ).toBeUndefined(); - }); - - it("accepts a migrated snapshot claim without merging AX from the old target", async () => { - const f = siblingCaptureFixture(); - const cdp: CdpRunner = { - ...f.cdp, - getFrameGraph: async () => ({ - rootFrameId: "frame-0", - frames: [ - { frameId: "frame-0", target: { tabId: 4 } }, - { - frameId: "frame-1", - parentFrameId: "frame-0", - ownerBackendNodeId: 100, - target: { tabId: 4, sessionId: "old" }, - }, - ], - }), - sendToTarget: async (_target: CdpTarget, method: string): Promise => - (method === "Accessibility.getFullAXTree" - ? { nodes: [{ nodeId: "stale", role: { value: "button" }, name: { value: "stale" } }] } - : {}) as T, - }; - const facts = await captureObservationFacts(cdp, 4); - const current = facts.documents.find((doc) => doc.frame.frameId === "frame-1"); - expect(current?.frame.target).toEqual({ tabId: 4 }); - expect(current?.domNodes.length).toBeGreaterThan(0); - expect(current?.axNodes).toEqual([]); - }); - - it.each([ - 0, 1, - ])("rejects duplicate snapshot frame identity at document %i without overwriting", async (index) => { - const f = siblingCaptureFixture(); - f.snapshot.documents.push(f.snapshot.documents[index]); - if (index === 0) { - await expect(captureObservationFacts(f.cdp, 4)).rejects.toThrow( - "root document ownership is ambiguous", - ); - } else { - const facts = await captureObservationFacts(f.cdp, 4); - expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-1"); - expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-7"); - expect( - facts.documents.find((doc) => doc.frame.frameId === "frame-2")?.domNodes.length, - ).toBeGreaterThan(0); - } - }); - - it("retains the caller URL in the root AX-only fallback", async () => { - const cdp: CdpRunner = { - getFrameGraph: async () => { - throw new Error("no graph"); - }, - send: async (_tabId: number, method: string): Promise => { - if (method === "DOMSnapshot.captureSnapshot") throw new Error("no snapshot"); - return ( - method === "Accessibility.getFullAXTree" - ? { nodes: [{ nodeId: "root", role: { value: "RootWebArea" } }] } - : {} - ) as T; - }, - }; - const facts = await captureObservationFacts(cdp, 4, undefined, "https://page.test/"); - expect(facts.documents[0].frame.url).toBe("https://page.test/"); - expect(facts.documents[0].axNodes).toHaveLength(1); - }); - - it("retains disconnected document data without inventing top-level geometry", async () => { - const f = siblingCaptureFixture(); - f.snapshot.documents[0].nodes.contentDocumentIndex = { index: [], value: [] }; - const facts = await captureObservationFacts(f.cdp, 4); - expect( - facts.documents.find((doc) => doc.frame.frameId === "frame-1")?.domNodes[0], - ).toMatchObject({ rect: null, localRect: { x: 0, y: 0, w: 200, h: 100 } }); - expect(f.measured()).toEqual([]); - expect(facts.issues).toContainEqual( - expect.objectContaining({ frameId: "frame-1", stage: "ownership" }), - ); - }); - - it.each([ - false, - true, - ])("isolates competing target claims independent of batch order %s", async (reverse) => { - const f = siblingCaptureFixture(); - const remote = { - frameId: "frame-1", - parentFrameId: "frame-0", - ownerBackendNodeId: 100, - target: { tabId: 4, sessionId: "remote" }, - }; - const main = { frameId: "frame-0", target: { tabId: 4 } }; - const cdp: CdpRunner = { - ...f.cdp, - getFrameGraph: async () => ({ - rootFrameId: "frame-0", - frames: reverse ? [remote, main] : [main, remote], - }), - sendToTarget: async (_target: CdpTarget, method: string): Promise => - (method === "DOMSnapshot.captureSnapshot" - ? { ...f.snapshot, documents: [f.snapshot.documents[1]] } - : method === "Page.getLayoutMetrics" - ? { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } } - : {}) as T, - }; - const facts = await captureObservationFacts(cdp, 4); - expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-1"); - expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-7"); - expect( - facts.documents.find((doc) => doc.frame.frameId === "frame-2")?.domNodes[0].rect, - ).not.toBeNull(); - expect(facts.issues).toContainEqual( - expect.objectContaining({ frameId: "frame-1", stage: "ownership" }), - ); - }); -}); - -describe("sibling frame measurement scheduling", () => { - it("bounds concurrent reads, fills free slots and preserves breadth-first output despite reordered replies", async () => { - const pending = new Map void>(); - const fixture = siblingCaptureFixture(async (method, params) => { - if (method === "DOM.getBoxModel" && Number(params.backendNodeId) < 200) - await new Promise((resolve) => pending.set(Number(params.backendNodeId), resolve)); - }); - const capture = captureObservationFacts(fixture.cdp, 4); - await vi.waitFor(() => expect(pending.size).toBe(4)); - expect(fixture.measured()).toEqual([100, 101, 102, 103]); - pending.get(103)!(); - await vi.waitFor(() => expect(pending.has(104)).toBe(true)); - pending.get(104)!(); - await vi.waitFor(() => expect(pending.has(105)).toBe(true)); - expect(fixture.measured()).not.toContain(200); - for (const id of [105, 102, 101, 100]) pending.get(id)!(); - const captured = await capture; - expect( - captured.issues - .filter((issue) => issue.stage === "geometry") - .map((issue) => issue.projectionIssue), - ).toEqual([]); - expect(fixture.peak()).toBe(4); - expect(fixture.active()).toBe(0); - expect(fixture.measured()).toEqual([100, 101, 102, 103, 104, 105, 200]); - expect(captured.documents.map((doc) => doc.frame.frameId)).toEqual( - Array.from({ length: 8 }, (_, i) => `frame-${i}`), - ); - expect( - captured.documents.find((doc) => doc.frame.frameId === "frame-7")?.domNodes[0].rect, - ).toEqual({ x: 0, y: 0, w: 200, h: 100 }); - for (const method of [ - "DOM.getBoxModel", - "DOM.resolveNode", - "Runtime.callFunctionOn", - "Runtime.releaseObject", - ]) - expect(fixture.send.mock.calls.filter(([, name]) => name === method)).toHaveLength(7); - }); - - it("does not measure descendants of a failed owner and retains sibling geometry and local nodes", async () => { - const fixture = siblingCaptureFixture(async (method, params) => { - if (method === "DOM.resolveNode" && params.backendNodeId === 100) - throw new Error("owner replaced"); - }); - const captured = await captureObservationFacts(fixture.cdp, 4); - expect( - captured.issues - .filter((issue) => issue.stage === "geometry") - .map((issue) => issue.projectionIssue), - ).toEqual([ - { - status: "unavailable", - source: { target: { tabId: 4 }, frameId: "frame-1" }, - ownerBackendNodeId: 100, - }, - { - status: "blocked", - source: { target: { tabId: 4 }, frameId: "frame-7" }, - cause: captured.issues.find((issue) => issue.frameId === "frame-1")?.projectionIssue, - }, - ]); - expect(fixture.measured()).not.toContain(200); - expect( - captured.documents.find((doc) => doc.frame.frameId === "frame-7")?.domNodes[0], - ).toMatchObject({ - tag: "body", - rect: null, - localRect: { x: 0, y: 0, w: 200, h: 100 }, - }); - expect( - captured.documents.find((doc) => doc.frame.frameId === "frame-2")?.domNodes[0].rect, - ).toEqual({ x: 0, y: 0, w: 200, h: 100 }); - expect(captured.documents.map((doc) => doc.frame.frameId)).toEqual( - Array.from({ length: 8 }, (_, i) => `frame-${i}`), - ); - }); - - it("stops scheduling on cancellation and waits for resolved objects to be released", async () => { - const controller = new AbortController(); - const resolutions = new Map void>(); - const releases = new Map void>(); - const fixture = siblingCaptureFixture(async (method, params) => { - if (method === "DOM.resolveNode") - await new Promise((resolve) => - resolutions.set(String(params.backendNodeId), resolve), - ); - if (method === "Runtime.releaseObject") - await new Promise((resolve) => releases.set(String(params.objectId), resolve)); - }); - let settled = false; - const capture = captureObservationFacts(fixture.cdp, 4, controller.signal); - const rejected = expect(capture).rejects.toMatchObject({ name: "AbortError" }); - void capture.then( - () => { - settled = true; - }, - () => { - settled = true; - }, - ); - await vi.waitFor(() => expect(resolutions.size).toBe(4)); - controller.abort(); - for (const resolve of resolutions.values()) resolve(); - await vi.waitFor(() => expect(releases.size).toBe(4)); - expect(settled).toBe(false); - expect(fixture.measured()).toEqual([100, 101, 102, 103]); - for (const release of releases.values()) release(); - await rejected; - expect(fixture.active()).toBe(0); - expect(fixture.send.mock.calls.some(([, method]) => method === "Runtime.callFunctionOn")).toBe( - false, - ); - }); -}); diff --git a/apps/extension/src/tools/vom/__tests__/capture.test.ts b/apps/extension/src/tools/vom/__tests__/capture.test.ts index 413512a..a1b5dfb 100644 --- a/apps/extension/src/tools/vom/__tests__/capture.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, vi } from "vitest"; import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge"; -import { collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture"; -import { captureObservationFacts, semanticCapture } from "../capture-coordinator"; +import type { CdpRunner } from "../../shared"; +import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture"; // Minimal but format-accurate captureSnapshot reply: a body with one // fixed full-screen overlay div carrying a password input. @@ -27,7 +27,6 @@ function fakeSnapshotReply() { strings: S, documents: [ { - frameId: "root", nodes: { parentIndex: [-1, 0, 1, 2], nodeType: [1, 1, 1, 1], @@ -75,7 +74,6 @@ function hoverTriggerSnapshotReply() { strings: S, documents: [ { - frameId: "root", nodes: { parentIndex: [-1, 0, 1], nodeType: [1, 1, 1], @@ -120,7 +118,6 @@ function twoHoverTriggerSnapshotReply() { strings: S, documents: [ { - frameId: "root", nodes: { parentIndex: [-1, 0, 1, 1], nodeType: [1, 1, 1, 1], @@ -174,7 +171,6 @@ function nestedHoverTriggerSnapshotReply() { strings: S, documents: [ { - frameId: "root", nodes: { parentIndex: [-1, 0, 1, 2, 3], nodeType: [1, 1, 1, 1, 1], @@ -248,11 +244,13 @@ function makeCdp(snapshot: unknown) { }; } -describe("snapshot capture and hover", () => { +describe("captureViewModel", () => { it("parses nodes, attrs, rects, paint order and styles", async () => { - const { nodes, viewport } = semanticCapture( - await captureObservationFacts(makeCdp(fakeSnapshotReply()), 4), - ).captured; + const { nodes, viewport, iframeNodes } = await captureViewModel( + makeCdp(fakeSnapshotReply()), + 4, + ); + expect(iframeNodes.size).toBe(0); expect(viewport).toEqual({ width: 1000, height: 800 }); const div = nodes.find((n) => n.backendNodeId === 12); expect(div).toMatchObject({ tag: "div", position: "fixed", parentBackendNodeId: 11 }); @@ -264,11 +262,35 @@ describe("snapshot capture and hover", () => { it("never hovers the page while capturing", async () => { const cdp = makeCdp(fakeSnapshotReply()); - semanticCapture(await captureObservationFacts(cdp, 4)).captured; + await captureViewModel(cdp, 4); expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything()); }); + it("batch-enriches form controls without per-node object resolution", async () => { + const cdp = makeCdp(fakeSnapshotReply()); + const result = await captureViewModel(cdp, 4); + const input = result.nodes.find((node) => node.backendNodeId === 13); + + expect(input).toMatchObject({ + formState: "filled", + formPlaceholder: "secret", + attrs: { type: "password" }, + }); + expect(input?.formValue).toBeUndefined(); + expect(input?.formDefaultValue).toBeUndefined(); + expect(cdp.send).toHaveBeenCalledWith( + 4, + "Runtime.evaluate", + expect.objectContaining({ + serializationOptions: expect.objectContaining({ serialization: "deep" }), + }), + ); + expect(cdp.send).toHaveBeenCalledWith(4, "Runtime.releaseObjectGroup", expect.anything()); + expect(cdp.send).not.toHaveBeenCalledWith(4, "DOM.resolveNode", expect.anything()); + expect(cdp.send).not.toHaveBeenCalledWith(4, "Runtime.callFunctionOn", expect.anything()); + }); + it("runs hover surface probes when explicitly enabled", async () => { let hoverStateCalls = 0; const cdp = { @@ -308,7 +330,7 @@ describe("snapshot capture and hover", () => { }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured; + const captured = await captureViewModel(cdp, 4); const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes); expect(surfaceProbes).toEqual([ @@ -368,7 +390,7 @@ describe("snapshot capture and hover", () => { }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured; + const captured = await captureViewModel(cdp, 4); const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes); expect(surfaceProbes).toEqual([ @@ -413,7 +435,7 @@ describe("snapshot capture and hover", () => { }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured; + const captured = await captureViewModel(cdp, 4); await probeHoverSurfaces(cdp, 4, captured.nodes); expect(hoverMoves).toBe(1); @@ -445,7 +467,6 @@ describe("snapshot capture and hover", () => { strings: S, documents: [ { - frameId: "root", nodes: { parentIndex: [-1, 0, 1, 0, 3, 4], nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")], @@ -476,7 +497,7 @@ describe("snapshot capture and hover", () => { ], }; - const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured; + const { nodes } = await captureViewModel(makeCdp(snapshot), 4); const ids = nodes.map((n) => n.backendNodeId); // Real page nodes survive. expect(ids).toContain(12); @@ -507,7 +528,7 @@ describe("snapshot capture and hover", () => { strings: S, documents: [ { - frameId: "root", + frameId: "main", nodes: { parentIndex: [-1, 0, 1, 0, 3, 4], nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")], @@ -538,10 +559,12 @@ describe("snapshot capture and hover", () => { ], }; - const { excludedBackendNodeIds } = semanticCapture( - await captureObservationFacts(makeCdp(snapshot), 4), - ).captured; + const { excludedBackendNodeIds, frameExcludedBackendNodeIds } = await captureViewModel( + makeCdp(snapshot), + 4, + ); expect(excludedBackendNodeIds).toEqual(new Set([13, 14, 15])); + expect(frameExcludedBackendNodeIds?.get("main")).toEqual(excludedBackendNodeIds); }); it("excludes overlay host matched by tag name only", async () => { @@ -561,7 +584,6 @@ describe("snapshot capture and hover", () => { strings: S, documents: [ { - frameId: "root", nodes: { parentIndex: [-1, 0, 1], nodeName: [i("html"), i("body"), i(OVERLAY_HOST_NAME)], @@ -584,9 +606,7 @@ describe("snapshot capture and hover", () => { ], }; - const { nodes, excludedBackendNodeIds } = semanticCapture( - await captureObservationFacts(makeCdp(snapshot), 4), - ).captured; + const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4); expect(nodes.map((n) => n.backendNodeId)).not.toContain(13); expect(excludedBackendNodeIds).toEqual(new Set([13])); }); @@ -609,7 +629,6 @@ describe("snapshot capture and hover", () => { strings: S, documents: [ { - frameId: "root", nodes: { parentIndex: [-1, 0, 1], nodeName: [i("html"), i("body"), i("div")], @@ -632,9 +651,7 @@ describe("snapshot capture and hover", () => { ], }; - const { nodes, excludedBackendNodeIds } = semanticCapture( - await captureObservationFacts(makeCdp(snapshot), 4), - ).captured; + const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4); expect(nodes.map((n) => n.backendNodeId)).not.toContain(13); expect(excludedBackendNodeIds).toEqual(new Set([13])); }); @@ -657,7 +674,6 @@ describe("snapshot capture and hover", () => { strings: S, documents: [ { - frameId: "root", nodes: { parentIndex: [-1, 0, 1], nodeName: [i("html"), i("body"), i("div")], @@ -680,7 +696,7 @@ describe("snapshot capture and hover", () => { }, ], }; - const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured; + const { nodes } = await captureViewModel(makeCdp(snapshot), 4); expect(nodes.find((n) => n.backendNodeId === 12)?.cursor).toBe("pointer"); expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto"); }); @@ -692,7 +708,6 @@ describe("snapshot capture and hover", () => { strings: S, documents: [ { - frameId: "root", nodes: { parentIndex: [-1, 0, 1, 1, 1], nodeName: [i("html"), i("body"), i("button"), i("button"), i("button")], @@ -719,7 +734,7 @@ describe("snapshot capture and hover", () => { ], }; - const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured; + const { nodes } = await captureViewModel(makeCdp(snapshot), 4); expect(nodes.find((node) => node.backendNodeId === 12)?.rendered).toBe(true); expect(nodes.find((node) => node.backendNodeId === 13)?.rendered).toBe(false); @@ -739,7 +754,7 @@ describe("snapshot capture and hover", () => { throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured; + const { nodes } = await captureViewModel(cdp, 4); const div = nodes.find((n) => n.backendNodeId === 12); expect(div?.localRect?.y).toBe(-200); expect(div?.rect).toMatchObject({ y: 0, h: 600 }); @@ -752,7 +767,6 @@ describe("snapshot capture and hover", () => { strings: S, documents: [ { - frameId: "root", nodes: { parentIndex: [-1, 0, 1], nodeName: [i("html"), i("body"), i("div")], @@ -787,7 +801,7 @@ describe("snapshot capture and hover", () => { throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured; + const { nodes } = await captureViewModel(cdp, 4); expect(nodes.find((n) => n.backendNodeId === 12)?.rect).toEqual({ x: 0, y: 0, @@ -823,7 +837,6 @@ describe("snapshot capture and hover", () => { strings: S, documents: [ { - frameId: "root", scrollOffsetX: 0, scrollOffsetY: 0, nodes: { @@ -862,7 +875,7 @@ describe("snapshot capture and hover", () => { throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured; + const { nodes } = await captureViewModel(cdp, 4); const btn = nodes.find((n) => n.backendNodeId === 12); expect(btn?.textContent).toBe("登录"); const link = nodes.find((n) => n.backendNodeId === 14); @@ -872,7 +885,7 @@ describe("snapshot capture and hover", () => { expect(body?.textContent).toBeUndefined(); }); - it("retains explicit iframe document ownership and projected nodes", async () => { + it("parses iframe sub-documents and returns iframeNodes keyed by iframe backendNodeId", async () => { const S = [ "html", "body", @@ -958,21 +971,30 @@ describe("snapshot capture and hover", () => { throw new Error(method); }) as unknown as (tabId: number, method: string, params?: object) => Promise, }; - const facts = await captureObservationFacts(cdp, 4); - const root = facts.documents.find((doc) => doc.frame.frameId === "main-frame")!; - const child = facts.documents.find((doc) => doc.frame.frameId === "child-frame")!; - expect(facts.documents).toHaveLength(2); - expect(root.domNodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe"); - const subNodes = child.domNodes; + const { + nodes, + iframeNodes, + frameNodes, + frameOwnerBackendNodeIds, + frameParentIds, + rootFrameId, + } = await captureViewModel(cdp, 4); + // Main frame has 4 nodes; iframe is included + expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe"); + // iframeNodes keyed by the