diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index 76d989c..fcd61cc 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -672,6 +672,60 @@ describe("buildVomScene", () => { expect(nestedInput?.parentId).toBe(result.nodes.find((item) => item.backendNodeId === 210)?.id); }); + it("excludes an overlay only in its owning document when target backend ids collide", () => { + const owner = (id: number): CapturedNode => ({ + backendNodeId: id, + parentBackendNodeId: null, + frameId: "main", + tag: "iframe", + attrs: {}, + rect: { x: id * 10, y: 0, w: 200, h: 100 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }); + const owners = [owner(10), owner(20)]; + const result = buildFrameVomScene( + [ + { + frameId: "main", + contextScopeId: "main", + target: { tabId: 7 }, + domNodes: owners, + axNodes: [], + }, + ...["left", "right"].map((frameId, i) => ({ + frameId, + contextScopeId: frameId, + parentFrameId: "main", + ownerBackendNodeId: owners[i].backendNodeId, + target: { tabId: 7, sessionId: frameId }, + excludedBackendNodeIds: new Set(i === 0 ? [7] : []), + domNodes: [], + axNodes: [ + { + nodeId: "button", + frameId, + backendDOMNodeId: 7, + role: { type: "role", value: "button" }, + name: { type: "computedString", value: frameId + " action" }, + }, + ], + })), + ], + { + nodes: owners, + viewport: { width: 800, height: 600 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + }, + ); + const rendered = renderVom(result); + expect(rendered.text).not.toContain("left action"); + expect(rendered.text).toContain("right action"); + expect(rendered.refs.find((ref) => ref.backendNodeId === 7)?.frameId).toBe("right"); + }); + it("does not place a child document at the page root when its frame boundary is unresolved", () => { const childNode: CapturedNode = { backendNodeId: 101, diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index 6f3cae2..ea1f0a1 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -676,6 +676,7 @@ function legacyFrameDocuments( ...(pageUrl ? { url: pageUrl } : {}), axNodes: [], domNodes: captured.nodes.map((node) => ({ ...node, frameId: node.frameId ?? rootFrameId })), + excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(rootFrameId), }, ]; const pending = [...captured.iframeNodes.entries()]; @@ -699,6 +700,7 @@ function legacyFrameDocuments( target: parent.target, axNodes: [], domNodes: domNodes.map((node) => ({ ...node, frameId: node.frameId ?? frameId })), + excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(frameId), }); pending.splice(index, 1); progress = true; diff --git a/apps/extension/src/tools/vom/__tests__/capture.test.ts b/apps/extension/src/tools/vom/__tests__/capture.test.ts index b70ca2d..1748dad 100644 --- a/apps/extension/src/tools/vom/__tests__/capture.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture.test.ts @@ -538,6 +538,9 @@ describe("captureViewModel", () => { strings: S, documents: [ { + scrollOffsetX: 0, + scrollOffsetY: 0, + frameId: "main", nodes: { parentIndex: [-1, 0, 1, 0, 3, 4], nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")], @@ -568,8 +571,12 @@ describe("captureViewModel", () => { ], }; - const { excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4); + const { excludedBackendNodeIds, frameExcludedBackendNodeIds } = await captureViewModel( + makeCdp(snapshot), + 4, + ); expect(excludedBackendNodeIds).toEqual(new Set([13, 14, 15])); + expect(frameExcludedBackendNodeIds?.get("main")).toEqual(excludedBackendNodeIds); }); it("excludes overlay host matched by tag name only", async () => { @@ -1168,6 +1175,8 @@ describe("captureViewModel", () => { }, }, { + scrollOffsetX: 0, + scrollOffsetY: 0, nodes: { parentIndex: [-1, 0, 1], nodeName: [i("html"), i("body"), i("iframe")], @@ -1175,8 +1184,6 @@ describe("captureViewModel", () => { attributes: [[], [], []], contentDocumentIndex: { index: [2], value: [2] }, }, - scrollOffsetX: 0, - scrollOffsetY: 0, layout: { nodeIndex: [1, 2], styles: [ @@ -1191,14 +1198,14 @@ describe("captureViewModel", () => { }, }, { + scrollOffsetX: 0, + scrollOffsetY: 0, nodes: { parentIndex: [-1, 0], nodeName: [i("body"), i("input")], backendNodeId: [30, 31], attributes: [[], [i("type"), i("text")]], }, - scrollOffsetX: 0, - scrollOffsetY: 0, layout: { nodeIndex: [1], styles: [[i("static"), i("auto")]], diff --git a/apps/extension/src/tools/vom/__tests__/facts.test.ts b/apps/extension/src/tools/vom/__tests__/facts.test.ts new file mode 100644 index 0000000..2991049 --- /dev/null +++ b/apps/extension/src/tools/vom/__tests__/facts.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, it, vi } from "vitest"; +import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge"; +import { createCaptureCheckpoint } from "../capture-abort"; +import { buildDocumentIndex, type DecodedNode } from "../facts"; +import { decodeDocument, REQUESTED_STYLES } from "../snapshot"; + +function node(id: number, parent: number | null): DecodedNode { + return { + backendNodeId: id, + parentBackendNodeId: parent, + tag: "div", + attrs: {}, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }; +} + +describe("document facts", () => { + it.each([ + 1_000, 10_000, 100_000, + ])("indexes deep and wide inputs of %i without copying ancestors", async (count) => { + for (const wide of [false, true]) { + let reads = 0; + const input = Array.from({ length: count }, (_, i) => { + const item = node(i, i ? (wide ? 0 : i - 1) : null); + if (i === 0) item.attrs[OVERLAY_HOST_MARKER_ATTR] = ""; + Object.defineProperty(item, "parentBackendNodeId", { + get: () => { + reads++; + return i ? (wide ? 0 : i - 1) : null; + }, + }); + return item; + }).reverse(); // parent need not precede child + const index = await buildDocumentIndex(input); + expect(index.nodes.size).toBe(count); + expect(index.excludedBackendNodeIds.size).toBe(count); + expect(reads).toBeLessThanOrEqual(count * 8); + expect(index.nodes.get(count - 1)).toBe(input[0]); + } + }); + + it("handles cycles and orphans and propagates overlay through shadow roots", async () => { + const host = { ...node(1, null), attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } }; + const input = [ + node(3, 2), + { ...node(2, 1), tag: "#document-fragment" }, + host, + node(4, 99), + node(5, 6), + node(6, 5), + node(7, 6), + ]; + const index = await buildDocumentIndex(input); + expect([...index.excludedBackendNodeIds].sort()).toEqual([1, 2, 3]); + expect(index.nodes.size).toBe(input.length); + }); + + it("does not propagate a descendant overlay backwards into a cycle", async () => { + const overlay = { ...node(3, 1), attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } }; + const index = await buildDocumentIndex([overlay, node(1, 2), node(2, 1)]); + expect([...index.excludedBackendNodeIds]).toEqual([3]); + }); + + it("decodes current observation bounds, styles and attributes without projecting coordinates", async () => { + const strings = ["div", "aria-hidden", "true", "inert", "", "visible", "0", "static", "auto"]; + const styles = REQUESTED_STYLES.map((style) => + style === "visibility" ? 5 : style === "opacity" ? 6 : style === "position" ? 7 : 8, + ); + const { nodes } = await decodeDocument( + { + nodes: { backendNodeId: [1], parentIndex: [-1], nodeName: [0], attributes: [[1, 2, 3, 4]] }, + layout: { nodeIndex: [0], bounds: [[10, 20, 120, 40]], styles: [styles] }, + }, + strings, + ); + expect(nodes[0].attrs).toEqual({ "aria-hidden": "true", inert: "" }); + expect(nodes[0].layout).toEqual({ + boundsSpace: "snapshot-document-layout", + bounds: [10, 20, 120, 40], + styles: { + position: "static", + "pointer-events": "auto", + cursor: "auto", + visibility: "visible", + opacity: "0", + }, + }); + expect(nodes[0]).not.toHaveProperty("rect"); + expect(nodes[0]).not.toHaveProperty("excluded"); + }); + + it("keeps short blocks synchronous and checks cancellation after yielding", async () => { + const clock = vi.spyOn(performance, "now").mockReturnValue(0); + try { + const controller = new AbortController(); + const checkpoint = createCaptureCheckpoint(controller.signal); + for (let i = 0; i < 20; i++) expect(checkpoint()).toBeUndefined(); + clock.mockReturnValue(9); + const pending = checkpoint(); + expect(pending).toBeInstanceOf(Promise); + controller.abort(); + await expect(pending).rejects.toMatchObject({ name: "AbortError" }); + } finally { + clock.mockRestore(); + } + }); + + it("checks cancellation at bounded indexing blocks", async () => { + const controller = new AbortController(); + const input = Array.from({ length: 100_000 }, (_, i) => node(i, i ? i - 1 : null)); + let reads = 0; + for (const item of input) + Object.defineProperty(item, "attrs", { + get: () => { + reads++; + if (reads === 100) controller.abort(); + return {}; + }, + }); + await expect(buildDocumentIndex(input, controller.signal)).rejects.toMatchObject({ + name: "AbortError", + }); + expect(reads).toBeLessThanOrEqual(356); + }); +}); diff --git a/apps/extension/src/tools/vom/__tests__/form-state.test.ts b/apps/extension/src/tools/vom/__tests__/form-state.test.ts index bcf6118..9dd9b70 100644 --- a/apps/extension/src/tools/vom/__tests__/form-state.test.ts +++ b/apps/extension/src/tools/vom/__tests__/form-state.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from "vitest"; import { type CdpRunner, cdpRunnerForTarget } from "../../shared"; -import { captureViewModel } from "../capture"; +import { enrichFormControlStates } from "../form-capture"; +import { decodeDocument } from "../snapshot"; interface Control { id: number; @@ -77,21 +78,32 @@ function reply(entries: Array<[number, unknown]>) { }; } -function fakeCdp(groups: Control[][], evaluate: (params: Record) => unknown) { +async function fakeCdp( + groups: Control[][], + evaluate: (params: Record) => unknown, +) { const snap = snapshot(groups); const send = vi.fn(async (_tabId: number, method: string, params?: object) => { - if (method === "DOMSnapshot.captureSnapshot") return snap; if (method === "Runtime.evaluate") return evaluate(params as Record); return {}; }); - return { send, cdp: { send: send as CdpRunner["send"] } }; + const nodes = await decodeControls(snap); + return { send, cdp: { send: send as CdpRunner["send"] }, nodes }; +} + +async function decodeControls(snap: ReturnType) { + return Promise.all( + snap.documents.map(async (doc) => + (await decodeDocument(doc, snap.strings)).nodes.map((node) => ({ ...node, rect: null })), + ), + ); } describe("form state identity", () => { it("matches reordered controls by identity and preserves controls missing from the batch", async () => { // The snapshot includes a shadow input. The runtime query skips it, // encounters a new input, and sees the remaining inputs in a new order. - const { cdp } = fakeCdp( + const { cdp, nodes } = await fakeCdp( [ [ { id: 10, value: "" }, @@ -106,21 +118,21 @@ describe("form state identity", () => { [11, state("alice@example.com")], ]), ); - const captured = await captureViewModel(cdp, 7); - expect(captured.nodes.find((n) => n.backendNodeId === 10)).toMatchObject({ + await enrichFormControlStates(cdp, 7, nodes); + expect(nodes[0].find((n) => n.backendNodeId === 10)).toMatchObject({ formValue: "", formState: "empty", formPlaceholder: "snapshot placeholder", }); - expect(captured.nodes.find((n) => n.backendNodeId === 11)).toMatchObject({ + expect(nodes[0].find((n) => n.backendNodeId === 11)).toMatchObject({ formValue: "alice@example.com", }); - expect(captured.nodes.find((n) => n.backendNodeId === 12)).toMatchObject({ formValue: "上海" }); - expect(captured.nodes.some((n) => n.backendNodeId === 99)).toBe(false); + expect(nodes[0].find((n) => n.backendNodeId === 12)).toMatchObject({ formValue: "上海" }); + expect(nodes[0].some((n) => n.backendNodeId === 99)).toBe(false); }); it("does not shift state between frames when one frame is absent from the runtime batch", async () => { - const { cdp } = fakeCdp( + const { cdp, nodes } = await fakeCdp( [ [], [{ id: 11, value: "inaccessible frame" }], @@ -133,8 +145,8 @@ describe("form state identity", () => { [21, state("second frame")], ]), ); - const captured = await captureViewModel(cdp, 7); - const controls = [...captured.iframeNodes.values()].flat(); + await enrichFormControlStates(cdp, 7, nodes); + const controls = nodes.slice(1).flat(); expect(controls.find((n) => n.backendNodeId === 11)?.formValue).toBe("inaccessible frame"); expect(controls.find((n) => n.backendNodeId === 21)?.formValue).toBe("second frame"); expect(controls.find((n) => n.backendNodeId === 31)?.formValue).toBe("third frame"); @@ -142,7 +154,6 @@ describe("form state identity", () => { it("keeps identical backend ids in different OOPIF targets separate", async () => { const sendToTarget = vi.fn(async (target, method) => { - if (method === "DOMSnapshot.captureSnapshot") return snapshot([[{ id: 11, value: "old" }]]); if (method === "Runtime.evaluate") return reply([[11, state(target.sessionId)]]); return {}; }); @@ -151,8 +162,9 @@ describe("form state identity", () => { }); const cdp = { send, sendToTarget } as unknown as CdpRunner; for (const sessionId of ["left-frame", "right-frame"]) { - const captured = await captureViewModel(cdpRunnerForTarget(cdp, { tabId: 7, sessionId }), 7); - expect(captured.nodes.find((n) => n.backendNodeId === 11)?.formValue).toBe(sessionId); + const nodes = await decodeControls(snapshot([[{ id: 11, value: "old" }]])); + await enrichFormControlStates(cdpRunnerForTarget(cdp, { tabId: 7, sessionId }), 7, nodes); + expect(nodes[0].find((n) => n.backendNodeId === 11)?.formValue).toBe(sessionId); expect(sendToTarget).toHaveBeenCalledWith( { tabId: 7, sessionId }, "Runtime.releaseObjectGroup", @@ -168,7 +180,7 @@ describe("form state identity", () => { "malformed", "invalid-state", ])("preserves snapshot data when enrichment is %s", async (mode) => { - const { cdp, send } = fakeCdp([[{ id: 11, value: "snapshot value" }]], () => { + const { cdp, send, nodes } = await fakeCdp([[{ id: 11, value: "snapshot value" }]], () => { if (mode === "unsupported") throw new Error("serialization unsupported"); if (mode === "legacy") return { result: { value: { controls: [state("wrong value")] } } }; const result = reply([ @@ -178,8 +190,8 @@ describe("form state identity", () => { result.result.deepSerializedValue.value[0].value[1].value = "not json"; return result; }); - const captured = await captureViewModel(cdp, 7); - expect(captured.nodes.find((n) => n.backendNodeId === 11)).toMatchObject({ + expect(await enrichFormControlStates(cdp, 7, nodes)).toBe(mode !== "unsupported"); + expect(nodes[0].find((n) => n.backendNodeId === 11)).toMatchObject({ formValue: "snapshot value", formPlaceholder: "snapshot placeholder", }); @@ -188,18 +200,34 @@ describe("form state identity", () => { it("releases the batch's remote objects when capture is cancelled", async () => { const controller = new AbortController(); - const { cdp, send } = fakeCdp([[{ id: 11, value: "old" }]], () => { + const { cdp, send, nodes } = await fakeCdp([[{ id: 11, value: "old" }]], () => { controller.abort(); return reply([[11, state("new")]]); }); - await expect(captureViewModel(cdp, 7, { signal: controller.signal })).rejects.toMatchObject({ + await expect(enrichFormControlStates(cdp, 7, nodes, controller.signal)).rejects.toMatchObject({ name: "AbortError", }); expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything()); }); + it("does not report success when cancelled during object cleanup", async () => { + const controller = new AbortController(); + const nodes = await decodeControls(snapshot([[{ id: 11, value: "old" }]])); + const cdp: CdpRunner = { + send: vi.fn(async (_tabId, method) => { + if (method === "Runtime.evaluate") return reply([[11, state("new")]]); + if (method === "Runtime.releaseObjectGroup") controller.abort(); + return {}; + }) as CdpRunner["send"], + }; + await expect(enrichFormControlStates(cdp, 7, nodes, controller.signal)).rejects.toMatchObject({ + name: "AbortError", + }); + expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything()); + }); + it("skips entries without node identity while still applying valid entries", async () => { - const { cdp } = fakeCdp( + const { cdp, nodes } = await fakeCdp( [ [ { id: 11, value: "snapshot value" }, @@ -216,20 +244,20 @@ describe("form state identity", () => { return result; }, ); - const captured = await captureViewModel(cdp, 7); - expect(captured.nodes.find((n) => n.backendNodeId === 11)?.formValue).toBe("snapshot value"); - expect(captured.nodes.find((n) => n.backendNodeId === 12)?.formValue).toBe("live value"); + await enrichFormControlStates(cdp, 7, nodes); + expect(nodes[0].find((n) => n.backendNodeId === 11)?.formValue).toBe("snapshot value"); + expect(nodes[0].find((n) => n.backendNodeId === 12)?.formValue).toBe("live value"); }); it.each([ "password", "text", ])("redacts a sensitive %s control after identity matching", async (type) => { - const { cdp } = fakeCdp([[{ id: 11, value: "secret", type }]], () => + const { cdp, nodes } = await fakeCdp([[{ id: 11, value: "secret", type }]], () => reply([[11, { ...state("secret"), sensitive: true }]]), ); - const captured = await captureViewModel(cdp, 7); - const control = captured.nodes.find((n) => n.backendNodeId === 11); + await enrichFormControlStates(cdp, 7, nodes); + const control = nodes[0].find((n) => n.backendNodeId === 11); expect(control?.formState).toBe("filled"); expect(control?.formValue).toBeUndefined(); expect(control?.formDefaultValue).toBeUndefined(); @@ -246,7 +274,7 @@ describe("form state identity", () => { defaultValue: "", placeholder: "", })); - const { cdp, send } = fakeCdp([controls], (params) => { + const { cdp, send, nodes } = await fakeCdp([controls], (params) => { const doc = { querySelectorAll: (selector: string) => selector === "input,textarea,select" ? elements : [], @@ -257,9 +285,9 @@ describe("form state identity", () => { expect(entries).toHaveLength(250); return reply(entries.map(([element, json]) => [element.backendNodeId, JSON.parse(json)])); }); - const captured = await captureViewModel(cdp, 7); - expect(captured.nodes.find((n) => n.backendNodeId === 250)?.formValue).toBe("live 250"); - expect(captured.nodes.find((n) => n.backendNodeId === 251)?.formValue).toBe("old"); + await enrichFormControlStates(cdp, 7, nodes); + expect(nodes[0].find((n) => n.backendNodeId === 250)?.formValue).toBe("live 250"); + expect(nodes[0].find((n) => n.backendNodeId === 251)?.formValue).toBe("old"); expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1); expect( send.mock.calls.filter(([, method]) => method === "Runtime.releaseObjectGroup"), diff --git a/apps/extension/src/tools/vom/__tests__/frame-capture.test.ts b/apps/extension/src/tools/vom/__tests__/frame-capture.test.ts index 1e7d5b5..52951df 100644 --- a/apps/extension/src/tools/vom/__tests__/frame-capture.test.ts +++ b/apps/extension/src/tools/vom/__tests__/frame-capture.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from "vitest"; +import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge"; import type { CdpRunner } from "../../shared"; import type { CapturedNode, CapturedViewModel } from "../capture"; import { captureFrameData } from "../frame-capture"; @@ -32,7 +33,7 @@ function childSnapshot(frameId: string, backendNodeId: number) { parentIndex: [-1, 0], nodeName: [1, 2], backendNodeId: [backendNodeId - 1, backendNodeId], - attributes: [[], []], + attributes: [[], []] as number[][], }, layout: { nodeIndex: [0, 1], @@ -140,9 +141,15 @@ describe("captureFrameData", () => { } if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {}; if (method === "DOMSnapshot.captureSnapshot") { - return target.sessionId === "left-session" - ? childSnapshot("left", 101) - : childSnapshot("right", 201); + if (target.sessionId !== "left-session") return childSnapshot("right", 201); + const snapshot = childSnapshot("left", 101); + const marker = snapshot.strings.push(OVERLAY_HOST_MARKER_ATTR) - 1; + const nodes = snapshot.documents[0].nodes; + nodes.backendNodeId.push(201); // Same id as a real button in the other target. + nodes.parentIndex.push(0); + nodes.nodeName.push(2); + nodes.attributes.push([marker, -1]); + return snapshot; } if (method === "Accessibility.getFullAXTree") return { nodes: [] }; throw new Error(`unexpected ${method}`); @@ -188,6 +195,10 @@ describe("captureFrameData", () => { const trees = await captureFrameData(cdp, 4, captured); + expect(trees.find((tree) => tree.frameId === "left")?.excludedBackendNodeIds).toEqual( + new Set([201]), + ); + expect(trees.find((tree) => tree.frameId === "right")?.excludedBackendNodeIds?.size).toBe(0); expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]); expect( captured.frameNodes?.get("left")?.find((node) => node.backendNodeId === 101)?.rect, @@ -274,10 +285,5 @@ describe("captureFrameData", () => { expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual( expect.objectContaining({ rect: null, localRect: null, rendered: true }), ); - expect(captured.frameGeometryIssues).toContainEqual({ - status: "unavailable", - source: { target: { tabId: 4, sessionId: "child-session" }, frameId: "child" }, - reason: "snapshot-coordinates-unavailable", - }); }); }); diff --git a/apps/extension/src/tools/vom/__tests__/frame-document.test.ts b/apps/extension/src/tools/vom/__tests__/frame-document.test.ts index 5ffe551..462d5c3 100644 --- a/apps/extension/src/tools/vom/__tests__/frame-document.test.ts +++ b/apps/extension/src/tools/vom/__tests__/frame-document.test.ts @@ -29,7 +29,7 @@ function captured(frameNodes: Map): CapturedViewModel { } describe("buildFrameDocuments", () => { - it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", () => { + it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", async () => { const graph: CdpFrameGraph = { rootFrameId: "main", frames: [ @@ -54,7 +54,7 @@ describe("buildFrameDocuments", () => { }, ]; - const documents = buildFrameDocuments( + const documents = await buildFrameDocuments( graph, batches, captured( @@ -72,7 +72,7 @@ describe("buildFrameDocuments", () => { expect(childNodes.find((node) => node.nodeId === "virtual")?.frameId).toBe("child"); }); - it("deduplicates overlapping same-target AX results using the strongest ownership", () => { + it("deduplicates overlapping same-target AX results using the strongest ownership", async () => { const graph: CdpFrameGraph = { rootFrameId: "main", frames: [ @@ -92,7 +92,7 @@ describe("buildFrameDocuments", () => { }, ]; - const documents = buildFrameDocuments( + const documents = await buildFrameDocuments( graph, batches, captured( @@ -107,7 +107,7 @@ describe("buildFrameDocuments", () => { expect(documents.find((document) => document.frameId === "child")?.axNodes).toHaveLength(1); }); - it("uses target-scoped backend ownership when OOPIF backend ids collide", () => { + it("uses target-scoped backend ownership when OOPIF backend ids collide", async () => { const graph: CdpFrameGraph = { rootFrameId: "main", frames: [ @@ -129,7 +129,7 @@ describe("buildFrameDocuments", () => { nodes: [{ nodeId: `${frame.frameId}-node`, backendDOMNodeId: 7 }], })); - const documents = buildFrameDocuments( + const documents = await buildFrameDocuments( graph, batches, captured( @@ -149,7 +149,7 @@ describe("buildFrameDocuments", () => { ); }); - it("uses captured frame parent identity instead of guessing from backend ids", () => { + it("uses captured frame parent identity instead of guessing from backend ids", async () => { const frameNodes = new Map([ ["main", [domNode("main", 1)]], ["left", [domNode("left", 7)]], @@ -164,11 +164,25 @@ describe("buildFrameDocuments", () => { nodes: [], })); - const documents = buildFrameDocuments(null, frames, capture); + const documents = await buildFrameDocuments(null, frames, capture); expect(documents.find((document) => document.frameId === "child")).toMatchObject({ parentFrameId: "left", ownerBackendNodeId: 7, }); }); + it("resolves a reverse-ordered deep AX parent chain without recursion", async () => { + const frame = { frameId: "main", target: { tabId: 4 } }; + const nodes: FrameOwnedAxNode[] = Array.from({ length: 10000 }, (_, i) => ({ + nodeId: String(i), + ...(i ? { parentId: String(i - 1) } : { frameId: "main" }), + })).reverse(); + const documents = await buildFrameDocuments( + { rootFrameId: "main", frames: [frame] }, + [{ frame, nodes }], + captured(new Map()), + ); + expect(documents[0].axNodes).toHaveLength(10000); + expect(documents[0].axNodes[0].frameId).toBe("main"); + }); }); diff --git a/apps/extension/src/tools/vom/capture-abort.ts b/apps/extension/src/tools/vom/capture-abort.ts new file mode 100644 index 0000000..7ca65bf --- /dev/null +++ b/apps/extension/src/tools/vom/capture-abort.ts @@ -0,0 +1,40 @@ +function captureAbortError(): Error { + const error = new Error("observation aborted"); + error.name = "AbortError"; + return error; +} + +export function isAbortError(error: unknown): boolean { + return ( + typeof error === "object" && + error !== null && + (error as { name?: string }).name === "AbortError" + ); +} + +export function throwIfAborted(signal: AbortSignal | undefined): void { + if (signal?.aborted) throw captureAbortError(); +} + +/** Check at bounded work blocks; yield only after a processing slice is spent. + * Each caller owns its deadline, so unrelated captures do not share state. */ +export function createCaptureCheckpoint(signal?: AbortSignal): () => Promise | undefined { + throwIfAborted(signal); + let deadline = performance.now() + 8; + return () => { + throwIfAborted(signal); + if (performance.now() < deadline) return; + return new Promise((resolve) => { + const channel = new MessageChannel(); + channel.port1.onmessage = () => { + channel.port1.close(); + channel.port2.close(); + resolve(); + }; + channel.port2.postMessage(null); + }).then(() => { + throwIfAborted(signal); + deadline = performance.now() + 8; + }); + }; +} diff --git a/apps/extension/src/tools/vom/capture-types.ts b/apps/extension/src/tools/vom/capture-types.ts new file mode 100644 index 0000000..dffe0dd --- /dev/null +++ b/apps/extension/src/tools/vom/capture-types.ts @@ -0,0 +1,37 @@ +import type { Rect } from "@browser-skill/vom"; + +export interface CapturedNode { + backendNodeId: number; + parentBackendNodeId: number | null; + frameId?: string; + /** Owning iframe backend node id; `null` for the top-level document. */ + ownerFrameBackendNodeId?: number | null; + tag: string; + attrs: Record; + /** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */ + rect: Rect | null; + /** Frame-local viewport-relative CSS px before top-level projection. */ + localRect?: Rect | null; + paintOrder: number; + position: string; + pointerEvents: string; + /** + * computed `cursor`. `cursor: pointer` is the strongest CDP-free signal + * that a non-semantic element (a `
`/`` with a click handler) + * is actually an interactive control — used by the adapter to surface + * custom buttons/checkboxes the AX tree drops as `generic`. Optional like + * `textContent`: the live parser always sets it, hand-built fixtures may not. + */ + cursor?: string; + /** + * Whether the live DOM snapshot provides a painted, non-hidden box for this + * node. Semantic resolution uses this only for DOM fallback nodes; AX-backed + * nodes remain authoritative even when they are outside the viewport. + */ + rendered?: boolean; + textContent?: string; + formState?: "empty" | "filled" | "default"; + formValue?: string; + formDefaultValue?: string; + formPlaceholder?: string; +} diff --git a/apps/extension/src/tools/vom/capture.ts b/apps/extension/src/tools/vom/capture.ts index 0c894d9..4b61c5e 100644 --- a/apps/extension/src/tools/vom/capture.ts +++ b/apps/extension/src/tools/vom/capture.ts @@ -1,20 +1,14 @@ -// CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics -// → CapturedNode[] + Viewport. This is the ONLY VOM module that touches -// raw CDP. The captureSnapshot reply is columnar (parallel arrays + a -// shared string table); requested computed styles are decoded through -// STYLE_COL so capture and visibility policy share one explicit contract. +// Coordinate DOMSnapshot acquisition, frame geometry and hover probes. +// Snapshot decoding and form enrichment live in their own modules. -import type { Rect, Viewport } from "@browser-skill/vom"; +import type { Viewport } from "@browser-skill/vom"; import type { CdpTarget } from "@/browser-driver/frame-graph"; import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy"; -import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge"; +import { OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge"; import { type FrameProjectionIssue, type FrameProjectionState, - projectSnapshotRect, - type SnapshotCoordinates, snapshotCoordinates, - snapshotViewportRect, } from "../geometry/coordinate-types"; import { GeometryContext, @@ -22,48 +16,20 @@ import { snapshotLayoutScale, } from "../geometry/frame-context"; import type { CdpRunner } from "../shared"; +import { isAbortError, throwIfAborted } from "./capture-abort"; +import type { CapturedNode } from "./capture-types"; +import { enrichFormControlStates } from "./form-capture"; import { clearHover, ProbeBudget, waitForHover } from "./hover-perception"; +import { type FrameContext, normalizeDocument } from "./normalize"; +import { + REQUESTED_STYLES, + type SnapshotDocument, + type SnapshotReply, + snapshotFrameId, + sparseIndexMap, +} from "./snapshot"; -const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const; -const STYLE_COL = Object.fromEntries( - REQUESTED_STYLES.map((name, index) => [name, index]), -) as Record<(typeof REQUESTED_STYLES)[number], number>; - -export interface CapturedNode { - backendNodeId: number; - parentBackendNodeId: number | null; - frameId?: string; - /** Owning iframe backend node id; `null` for the top-level document. */ - ownerFrameBackendNodeId?: number | null; - tag: string; - attrs: Record; - /** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */ - rect: Rect | null; - /** Frame-local viewport-relative CSS px before top-level projection. */ - localRect?: Rect | null; - paintOrder: number; - position: string; - pointerEvents: string; - /** - * computed `cursor`. `cursor: pointer` is the strongest CDP-free signal - * that a non-semantic element (a `
`/`` with a click handler) - * is actually an interactive control — used by the adapter to surface - * custom buttons/checkboxes the AX tree drops as `generic`. Optional like - * `textContent`: the live parser always sets it, hand-built fixtures may not. - */ - cursor?: string; - /** - * Whether the live DOM snapshot provides a painted, non-hidden box for this - * node. Semantic resolution uses this only for DOM fallback nodes; AX-backed - * nodes remain authoritative even when they are outside the viewport. - */ - rendered?: boolean; - textContent?: string; - formState?: "empty" | "filled" | "default"; - formValue?: string; - formDefaultValue?: string; - formPlaceholder?: string; -} +export type { CapturedNode } from "./capture-types"; export type CapturedIframeNodes = Map; @@ -80,6 +46,7 @@ export interface CapturedViewModel { viewport: Viewport; iframeNodes: CapturedIframeNodes; frameNodes?: Map; + frameExcludedBackendNodeIds?: Map>; frameOwnerBackendNodeIds?: Map; /** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */ frameParentIds?: Map; @@ -96,244 +63,9 @@ export interface CaptureViewModelOptions { target?: CdpTarget; } -function captureAbortError(): Error { - const error = new Error("observation aborted"); - error.name = "AbortError"; - return error; -} - -function isAbortError(error: unknown): boolean { - return ( - typeof error === "object" && - error !== null && - (error as { name?: string }).name === "AbortError" - ); -} - -function throwIfAborted(signal: AbortSignal | undefined): void { - if (signal?.aborted) throw captureAbortError(); -} - -/** Sparse array format Chrome uses for infrequently-set per-node fields. */ -interface SparseArray { - index: number[]; - value: number[]; -} - -interface RareBooleanData { - index: number[]; -} - -interface SnapshotDocument { - frameId?: string | number; - scrollOffsetX?: number; - scrollOffsetY?: number; - nodes?: { - parentIndex?: number[]; - nodeName?: number[]; - backendNodeId?: number[]; - attributes?: number[][]; - /** - * Per-node text value (index into strings), set for `#text` / CDATA nodes. - * Element nodes carry -1. Same length as `backendNodeId`. - */ - nodeValue?: number[]; - /** Maps node array index → index into `documents[]` for frame content. */ - contentDocumentIndex?: SparseArray; - inputValue?: SparseArray; - textValue?: SparseArray; - inputChecked?: RareBooleanData; - optionSelected?: RareBooleanData; - }; - layout?: { - nodeIndex?: number[]; - styles?: number[][]; - bounds?: number[][]; - paintOrders?: number[]; - }; -} - -function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined { - if (typeof document.frameId === "string") return document.frameId || undefined; - if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined; - return undefined; -} - -interface SnapshotReply { - strings?: string[]; - documents?: SnapshotDocument[]; -} - -function isFormControlTag(tag: string): boolean { - return tag === "input" || tag === "textarea" || tag === "select"; -} - -function isSensitiveFormControl(node: Pick): boolean { - return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password"; -} - -function snapshotFormState( - value: string | undefined, - defaultValue: string, - hasDefaultValue: boolean, - sensitive: boolean, -): CapturedNode["formState"] { - if (sensitive) { - if (value === undefined && !hasDefaultValue) return undefined; - return (value ?? defaultValue) === "" ? "empty" : "filled"; - } - if (value === undefined) return undefined; - if (value === "") return "empty"; - return value === defaultValue ? "default" : "filled"; -} - -const MAX_FORM_ENRICH_CONTROLS = 250; - -interface CapturedFormState { - value?: string; - defaultValue?: string; - placeholder?: string; - state?: "empty" | "filled" | "default"; - sensitive?: boolean; -} - -interface DeepSerializedValue { - type: string; - value?: unknown; -} - -function formStateBatchExpression(maxControls: number): string { - return `(() => { - const maxControls = ${JSON.stringify(maxControls)}; - let remaining = maxControls; - const controlSelector = "input,textarea,select"; - const controlState = (el) => { - const tag = el.tagName.toLowerCase(); - const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag; - const sensitive = type === "password" || type === "credit-card"; - const rawValue = typeof el.value === "string" ? el.value : ""; - const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : ""; - const placeholder = typeof el.placeholder === "string" ? el.placeholder : ""; - const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled"; - return { - state, - sensitive, - placeholder, - ...(sensitive ? {} : { value: rawValue, defaultValue }), - }; - }; - const controls = []; - const collect = (doc) => { - for (const el of Array.from(doc.querySelectorAll(controlSelector))) { - if (remaining <= 0) break; - // Deep serialization supplies the node's backend id. Keep the - // state as JSON so decoding needs no general-purpose V8 deserializer. - controls.push([el, JSON.stringify(controlState(el))]); - remaining -= 1; - } - for (const frame of Array.from(doc.querySelectorAll("iframe"))) { - if (remaining <= 0) break; - let childDoc = null; - try { childDoc = frame.contentDocument; } catch { childDoc = null; } - if (childDoc) collect(childDoc); - } - }; - collect(document); - return controls; - })()`; -} - -function formStatesByBackendId(result: RuntimeEvaluateReply): Map { - const states = new Map(); - const serialized = result.result?.deepSerializedValue; - if (serialized?.type !== "array" || !Array.isArray(serialized.value)) return states; - for (const entry of serialized.value as DeepSerializedValue[]) { - if (entry?.type !== "array" || !Array.isArray(entry.value)) continue; - const [element, json] = entry.value as DeepSerializedValue[]; - if (element?.type !== "node" || json?.type !== "string" || typeof json.value !== "string") { - continue; - } - const backendNodeId = (element.value as { backendNodeId?: number } | undefined)?.backendNodeId; - if (typeof backendNodeId !== "number" || !Number.isSafeInteger(backendNodeId)) continue; - try { - const state = JSON.parse(json.value) as CapturedFormState | null; - if ( - !state || - !["empty", "filled", "default"].includes(state.state ?? "") || - typeof state.sensitive !== "boolean" || - typeof state.placeholder !== "string" || - (!state.sensitive && - (typeof state.value !== "string" || typeof state.defaultValue !== "string")) - ) { - continue; - } - states.set(backendNodeId, state); - } catch { - // A malformed entry must not overwrite the snapshot's own state. - } - } - return states; -} - -function applyFormStates(nodes: CapturedNode[], states: Map): void { - for (const node of nodes) { - if (!isFormControlTag(node.tag)) continue; - const state = states.get(node.backendNodeId); - if (!state) continue; - node.formState = state.state; - node.formPlaceholder = state.placeholder ?? ""; - if (state.sensitive || isSensitiveFormControl(node)) { - delete node.formValue; - delete node.formDefaultValue; - delete node.attrs.value; - } else { - node.formDefaultValue = state.defaultValue ?? ""; - if (state.value !== undefined) node.formValue = state.value; - } - } -} - -async function enrichFormControlStates( - cdp: CdpRunner, - tabId: number, - frameNodeGroups: CapturedNode[][], - signal?: AbortSignal, -): Promise { - const hasControls = frameNodeGroups.some((nodes) => - nodes.some((node) => isFormControlTag(node.tag)), - ); - if (!hasControls) return; - const objectGroup = `bsk-vom-forms-${crypto.randomUUID()}`; - try { - throwIfAborted(signal); - const result = await cdp.send(tabId, "Runtime.evaluate", { - expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS), - objectGroup, - serializationOptions: { - serialization: "deep", - maxDepth: 3, - additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" }, - }, - }); - throwIfAborted(signal); - // Backend ids are scoped to this capture's CDP target. OOPIF targets - // use their own captureViewModel call and never share this lookup. - const states = formStatesByBackendId(result); - for (const nodes of frameNodeGroups) { - applyFormStates(nodes, states); - } - } catch (error) { - if (isAbortError(error)) throw error; - // Best-effort enrichment. DOMSnapshot/AX data still carries the nodes. - } finally { - await cdp.send(tabId, "Runtime.releaseObjectGroup", { objectGroup }).catch(() => undefined); - } -} - interface RuntimeEvaluateReply { result?: { value?: unknown; - deepSerializedValue?: DeepSerializedValue; }; } @@ -352,36 +84,6 @@ interface CdpDomNode { shadowRoots?: CdpDomNode[]; } -interface ParseDocumentResult { - nodes: CapturedNode[]; - excludedBackendNodeIds: Set; -} - -interface FrameContext { - frameId?: string; - ownerFrameBackendNodeId: number | null; - projection: FrameProjectionState; - target: CdpTarget; - coordinates: SnapshotCoordinates | null; - layoutUnitsPerCssPixel: number | null; -} - -function str(strings: string[], idx: number | undefined): string { - if (idx === undefined || idx < 0) return ""; - return strings[idx] ?? ""; -} - -function sparseIndexMap(sparse: SparseArray | undefined): Map { - const out = new Map(); - if (!sparse?.index || !sparse.value) return out; - for (let i = 0; i < sparse.index.length; i++) { - const nodeIndex = sparse.index[i]; - const docIndex = sparse.value[i]; - if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex); - } - return out; -} - function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set): void { if (!node) return; if (typeof node.backendNodeId === "number") { @@ -791,198 +493,12 @@ export async function collectOverlayExcludedBackendIds( return excluded; } -/** - * Parse one DOMSnapshot document into CapturedNode[]. - * - * @param doc - the raw DOMSnapshot document object - * @param strings - the shared string table for the whole snapshot - * @param context - frame coordinate context; output rects are top viewport-relative - */ -function parseDocumentNodes( - doc: SnapshotDocument, - strings: string[], - context: FrameContext, -): ParseDocumentResult { - const dn = doc.nodes; - const dl = doc.layout; - if (!dn?.backendNodeId) { - return { nodes: [], excludedBackendNodeIds: new Set() }; - } - - const count = dn.backendNodeId.length; - const layoutByNode = new Map(); - (dl?.nodeIndex ?? []).forEach((nodeIdx, layoutIdx) => layoutByNode.set(nodeIdx, layoutIdx)); - - // The agent's own overlay (WXT shadow host, marked with OVERLAY_HOST_MARKER_ATTR) - // is injected into the page and its open shadow root - // is inlined by DOMSnapshot as descendants of the host. Its fixed - // full-viewport click-blocker would otherwise dominate occlusion detection - // and hide the real page, so mark the host + its whole subtree excluded. - // parentIndex always references an earlier node, so one forward pass suffices. - const excluded = new Array(count).fill(false); - const excludedBackendNodeIds = new Set(); - for (let n = 0; n < count; n++) { - const parentIdx = dn.parentIndex?.[n] ?? -1; - const inherited = parentIdx >= 0 && excluded[parentIdx]; - let isOverlayHost = inherited; - if (!isOverlayHost) { - const pairs = dn.attributes?.[n] ?? []; - const attrNames: string[] = []; - for (let a = 0; a + 1 < pairs.length; a += 2) { - attrNames.push(str(strings, pairs[a])); - } - isOverlayHost = isOverlayHostNode(str(strings, dn.nodeName?.[n]), attrNames); - } - if (!isOverlayHost) continue; - excluded[n] = true; - const backendNodeId = dn.backendNodeId[n]; - if (backendNodeId !== undefined) { - excludedBackendNodeIds.add(backendNodeId); - } - } - - const posCol = STYLE_COL.position; - const peCol = STYLE_COL["pointer-events"]; - const cursorCol = STYLE_COL.cursor; - const visibilityCol = STYLE_COL.visibility; - const opacityCol = STYLE_COL.opacity; - const inputValues = sparseIndexMap(dn.inputValue); - const textValues = sparseIndexMap(dn.textValue); - const checkedInputs = new Set(dn.inputChecked?.index ?? []); - const selectedOptions = new Set(dn.optionSelected?.index ?? []); - - // Collect visible text from #text child nodes so element CapturedNodes - // carry a textContent value usable as a button/link label fallback. - // nodeValue is a parallel array: string index for text nodes, -1 otherwise. - const nodeTextContent = new Map(); - if (dn.nodeValue) { - for (let n = 0; n < count; n++) { - const nvIdx = dn.nodeValue[n] ?? -1; - if (nvIdx < 0) continue; - const text = str(strings, nvIdx).trim(); - if (!text) continue; - const parentIdx = dn.parentIndex?.[n] ?? -1; - if (parentIdx >= 0) { - const existing = nodeTextContent.get(parentIdx); - nodeTextContent.set(parentIdx, existing ? `${existing} ${text}` : text); - } - } - } - - // Build CapturedNodes. - const nodes: CapturedNode[] = []; - for (let n = 0; n < count; n++) { - if (excluded[n]) continue; - const backendNodeId = dn.backendNodeId[n]; - const parentIdx = dn.parentIndex?.[n] ?? -1; - const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null; - const tag = str(strings, dn.nodeName?.[n]).toLowerCase(); - - const attrs: Record = {}; - const pairs = dn.attributes?.[n] ?? []; - for (let a = 0; a + 1 < pairs.length; a += 2) { - attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]); - } - - let rect: Rect | null = null; - let localRect: Rect | null = null; - let paintOrder = 0; - let position = "static"; - let pointerEvents = "auto"; - let cursor = "auto"; - let rendered = false; - const li = layoutByNode.get(n); - if (li !== undefined) { - const b = dl?.bounds?.[li]; - if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) { - const input = snapshotViewportRect( - b, - { target: context.target, frameId: context.frameId }, - context.coordinates, - ); - if (input) { - const { x, y, width: w, height: h } = input.rect; - localRect = { x, y, w, h }; - } - const bounds = - input && context.projection.status === "available" - ? projectSnapshotRect(input, context.projection.projection) - : null; - rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null; - } - paintOrder = dl?.paintOrders?.[li] ?? 0; - const styleRow = dl?.styles?.[li] ?? []; - position = str(strings, styleRow[posCol]) || "static"; - pointerEvents = str(strings, styleRow[peCol]) || "auto"; - cursor = str(strings, styleRow[cursorCol]) || "auto"; - const visibility = str(strings, styleRow[visibilityCol]) || "visible"; - const opacity = str(strings, styleRow[opacityCol]) || "1"; - rendered = - !!b && - b.length >= 4 && - b.slice(0, 4).every(Number.isFinite) && - b[2] > 0 && - b[3] > 0 && - visibility !== "hidden" && - visibility !== "collapse" && - (Number.parseFloat(opacity) || 0) > 0; - } - - // Skip non-element nodes (#text, #cdata-section, etc.) — they carry no - // geometry and are never queried by callers. - if (tag.startsWith("#")) continue; - - const textContent = nodeTextContent.get(n); - - const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n); - const rawFormValue = - rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined; - const sensitive = isSensitiveFormControl({ tag, attrs }); - const formDefaultValue = attrs.value ?? ""; - const formValue = sensitive ? undefined : rawFormValue; - const formState = snapshotFormState( - rawFormValue, - formDefaultValue, - Object.prototype.hasOwnProperty.call(attrs, "value"), - sensitive, - ); - if (sensitive) delete attrs.value; - - nodes.push({ - backendNodeId, - parentBackendNodeId, - ...(context.frameId ? { frameId: context.frameId } : {}), - ownerFrameBackendNodeId: context.ownerFrameBackendNodeId, - tag, - attrs, - rect, - localRect, - paintOrder, - position, - pointerEvents, - cursor, - rendered, - textContent, - ...(formValue !== undefined ? { formValue } : {}), - ...(tag === "input" || tag === "textarea" - ? { - formPlaceholder: attrs.placeholder ?? "", - ...(!sensitive ? { formDefaultValue } : {}), - ...(formState ? { formState } : {}), - } - : {}), - ...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}), - ...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}), - }); - } - return { nodes, excludedBackendNodeIds }; -} - interface ParseFrameDocumentsResult { frameGeometryIssues: FrameProjectionIssue[]; iframeNodes: CapturedIframeNodes; frameOwnerBackendNodeIds: Map; frameParentIds: Map; + frameExcludedBackendNodeIds: Map>; excludedBackendNodeIds: Set; } @@ -999,6 +515,7 @@ async function parseChildFrameDocuments( const iframeNodes: CapturedIframeNodes = new Map(); const frameOwnerBackendNodeIds = new Map(); const frameParentIds = new Map(); + const frameExcludedBackendNodeIds = new Map>(); const excludedBackendNodeIds = new Set(); const parentDoc = documents[parentDocIndex]; const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex); @@ -1007,6 +524,7 @@ async function parseChildFrameDocuments( iframeNodes, frameOwnerBackendNodeIds, frameParentIds, + frameExcludedBackendNodeIds, excludedBackendNodeIds, frameGeometryIssues, }; @@ -1096,13 +614,14 @@ async function parseChildFrameDocuments( }; const nextVisited = new Set(visited); nextVisited.add(childDocIndex); - const parsed = parseDocumentNodes(childDoc, strings, childContext); + const parsed = await normalizeDocument(childDoc, strings, childContext, signal); iframeNodes.set(iframeBackendId, parsed.nodes); if (childContext.frameId) { frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId); + frameExcludedBackendNodeIds.set(childContext.frameId, parsed.index.excludedBackendNodeIds); if (parentContext.frameId) frameParentIds.set(childContext.frameId, parentContext.frameId); } - for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id); + for (const id of parsed.index.excludedBackendNodeIds) excludedBackendNodeIds.add(id); const nested = await parseChildFrameDocuments( documents, @@ -1123,6 +642,8 @@ async function parseChildFrameDocuments( for (const [frameId, parentFrameId] of nested.frameParentIds) { frameParentIds.set(frameId, parentFrameId); } + for (const [frameId, excluded] of nested.frameExcludedBackendNodeIds) + frameExcludedBackendNodeIds.set(frameId, excluded); for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id); } @@ -1130,6 +651,7 @@ async function parseChildFrameDocuments( iframeNodes, frameOwnerBackendNodeIds, frameParentIds, + frameExcludedBackendNodeIds, excludedBackendNodeIds, frameGeometryIssues, }; @@ -1201,9 +723,9 @@ export async function captureViewModel( coordinates, layoutUnitsPerCssPixel, }; - const mainParsed = parseDocumentNodes(doc0, strings, topContext); + const mainParsed = await normalizeDocument(doc0, strings, topContext, options.signal); const nodes = mainParsed.nodes; - const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds); + const excludedBackendNodeIds = new Set(mainParsed.index.excludedBackendNodeIds); const ownerIds = new Set(); for (const document of documents) { @@ -1222,6 +744,11 @@ export async function captureViewModel( options.signal, ); const iframeNodes = frameParsed.iframeNodes; + if (topContext.frameId) + frameParsed.frameExcludedBackendNodeIds.set( + topContext.frameId, + mainParsed.index.excludedBackendNodeIds, + ); for (const id of frameParsed.excludedBackendNodeIds) { excludedBackendNodeIds.add(id); } @@ -1251,6 +778,7 @@ export async function captureViewModel( ...frameParsed.frameGeometryIssues, ], frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds, + frameExcludedBackendNodeIds: frameParsed.frameExcludedBackendNodeIds, frameParentIds: frameParsed.frameParentIds, ...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}), excludedBackendNodeIds, diff --git a/apps/extension/src/tools/vom/facts.ts b/apps/extension/src/tools/vom/facts.ts new file mode 100644 index 0000000..e257d01 --- /dev/null +++ b/apps/extension/src/tools/vom/facts.ts @@ -0,0 +1,94 @@ +import { isOverlayHostNode } from "@/lib/overlay-bridge"; +import { createCaptureCheckpoint } from "./capture-abort"; +import type { CapturedNode } from "./capture-types"; + +/** Bounds retain their raw snapshot document layout units until normalization. */ +export interface SnapshotLayout { + readonly boundsSpace: "snapshot-document-layout"; + bounds?: number[]; + styles: Readonly>; +} + +export interface DecodedNode extends Omit { + nodeType?: number; + layout?: SnapshotLayout; +} + +export interface NodeFacts extends CapturedNode { + nodeType?: number; + layout?: SnapshotLayout; +} + +export interface DocumentIndex { + readonly nodes: ReadonlyMap; + readonly excludedBackendNodeIds: ReadonlySet; +} + +export interface DecodedDocument { + nodes: DecodedNode[]; +} + +/** All snapshot nodes participate, including document and shadow roots. This + * preserves overlay propagation when the semantic adapter omits those nodes. */ +export async function buildDocumentIndex( + input: readonly T[], + signal?: AbortSignal, +): Promise> { + const checkpoint = createCaptureCheckpoint(signal); + const nodes = new Map(); + const overlayByNode = new Map(); + const excludedBackendNodeIds = new Set(); + for (let i = 0; i < input.length; i++) { + if (i % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } + const node = input[i]; + nodes.set(node.backendNodeId, node); + } + let work = 0; + for (const node of input) { + if (work++ % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } + if (overlayByNode.has(node.backendNodeId)) continue; + const path: DecodedNode[] = []; + const visiting = new Set(); + let current: DecodedNode | undefined = node; + let overlay = false; + while (current && !overlayByNode.has(current.backendNodeId)) { + if (work++ % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } + if (visiting.has(current.backendNodeId)) { + // Only an overlay inside the cycle may mark the cycle and its descendants. + overlay = path + .slice(path.findIndex((item) => item.backendNodeId === current!.backendNodeId)) + .some((item) => isOverlayHostNode(item.tag, Object.keys(item.attrs))); + break; + } + visiting.add(current.backendNodeId); + path.push(current); + if (current.parentBackendNodeId === null) { + current = undefined; + break; + } + current = nodes.get(current.parentBackendNodeId); + } + if (current && overlayByNode.has(current.backendNodeId)) + overlay = overlayByNode.get(current.backendNodeId)!; + for (let i = path.length - 1; i >= 0; i--) { + if (work++ % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } + const item = path[i]; + overlay = overlay || isOverlayHostNode(item.tag, Object.keys(item.attrs)); + overlayByNode.set(item.backendNodeId, overlay); + if (overlay) excludedBackendNodeIds.add(item.backendNodeId); + } + } + return { nodes, excludedBackendNodeIds }; +} diff --git a/apps/extension/src/tools/vom/form-capture.ts b/apps/extension/src/tools/vom/form-capture.ts new file mode 100644 index 0000000..c4c390c --- /dev/null +++ b/apps/extension/src/tools/vom/form-capture.ts @@ -0,0 +1,155 @@ +import type { CdpRunner } from "../shared"; +import { isAbortError, throwIfAborted } from "./capture-abort"; +import type { CapturedNode } from "./capture-types"; + +interface RuntimeEvaluateReply { + result?: { deepSerializedValue?: DeepSerializedValue }; +} +const MAX_FORM_ENRICH_CONTROLS = 250; + +interface CapturedFormState { + value?: string; + defaultValue?: string; + placeholder?: string; + state?: "empty" | "filled" | "default"; + sensitive?: boolean; +} + +interface DeepSerializedValue { + type: string; + value?: unknown; +} + +function formStateBatchExpression(maxControls: number): string { + return `(() => { + const maxControls = ${JSON.stringify(maxControls)}; + let remaining = maxControls; + const controlSelector = "input,textarea,select"; + const controlState = (el) => { + const tag = el.tagName.toLowerCase(); + const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag; + const sensitive = type === "password" || type === "credit-card"; + const rawValue = typeof el.value === "string" ? el.value : ""; + const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : ""; + const placeholder = typeof el.placeholder === "string" ? el.placeholder : ""; + const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled"; + return { + state, + sensitive, + placeholder, + ...(sensitive ? {} : { value: rawValue, defaultValue }), + }; + }; + const controls = []; + const collect = (doc) => { + for (const el of Array.from(doc.querySelectorAll(controlSelector))) { + if (remaining <= 0) break; + // Deep serialization supplies the node's backend id. Keep the + // state as JSON so decoding needs no general-purpose V8 deserializer. + controls.push([el, JSON.stringify(controlState(el))]); + remaining -= 1; + } + for (const frame of Array.from(doc.querySelectorAll("iframe"))) { + if (remaining <= 0) break; + let childDoc = null; + try { childDoc = frame.contentDocument; } catch { childDoc = null; } + if (childDoc) collect(childDoc); + } + }; + collect(document); + return controls; + })()`; +} + +function formStatesByBackendId(result: RuntimeEvaluateReply): Map { + const states = new Map(); + const serialized = result.result?.deepSerializedValue; + if (serialized?.type !== "array" || !Array.isArray(serialized.value)) return states; + for (const entry of serialized.value as DeepSerializedValue[]) { + if (entry?.type !== "array" || !Array.isArray(entry.value)) continue; + const [element, json] = entry.value as DeepSerializedValue[]; + if (element?.type !== "node" || json?.type !== "string" || typeof json.value !== "string") { + continue; + } + const backendNodeId = (element.value as { backendNodeId?: number } | undefined)?.backendNodeId; + if (typeof backendNodeId !== "number" || !Number.isSafeInteger(backendNodeId)) continue; + try { + const state = JSON.parse(json.value) as CapturedFormState | null; + if ( + !state || + !["empty", "filled", "default"].includes(state.state ?? "") || + typeof state.sensitive !== "boolean" || + typeof state.placeholder !== "string" || + (!state.sensitive && + (typeof state.value !== "string" || typeof state.defaultValue !== "string")) + ) { + continue; + } + states.set(backendNodeId, state); + } catch { + // A malformed entry must not overwrite the snapshot's own state. + } + } + return states; +} + +function applyFormStates(nodes: CapturedNode[], states: Map): void { + for (const node of nodes) { + if (!["input", "textarea", "select"].includes(node.tag)) continue; + const state = states.get(node.backendNodeId); + if (!state) continue; + node.formState = state.state; + node.formPlaceholder = state.placeholder ?? ""; + if ( + state.sensitive || + (node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password") + ) { + delete node.formValue; + delete node.formDefaultValue; + delete node.attrs.value; + } else { + node.formDefaultValue = state.defaultValue ?? ""; + if (state.value !== undefined) node.formValue = state.value; + } + } +} + +export async function enrichFormControlStates( + cdp: CdpRunner, + tabId: number, + frameNodeGroups: CapturedNode[][], + signal?: AbortSignal, +): Promise { + const hasControls = frameNodeGroups.some((nodes) => + nodes.some((node) => ["input", "textarea", "select"].includes(node.tag)), + ); + if (!hasControls) return true; + const objectGroup = `bsk-vom-forms-${crypto.randomUUID()}`; + try { + throwIfAborted(signal); + const result = await cdp.send(tabId, "Runtime.evaluate", { + expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS), + objectGroup, + serializationOptions: { + serialization: "deep", + maxDepth: 3, + additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" }, + }, + }); + throwIfAborted(signal); + // Backend ids are scoped to this capture's CDP target. OOPIF targets + // have separate enrichment calls and never share this lookup. + const states = formStatesByBackendId(result); + for (const nodes of frameNodeGroups) { + applyFormStates(nodes, states); + } + return true; + } catch (error) { + if (isAbortError(error)) throw error; + // Best-effort enrichment. DOMSnapshot/AX data still carries the nodes. + return false; + } finally { + await cdp.send(tabId, "Runtime.releaseObjectGroup", { objectGroup }).catch(() => undefined); + throwIfAborted(signal); + } +} diff --git a/apps/extension/src/tools/vom/frame-capture.ts b/apps/extension/src/tools/vom/frame-capture.ts index 374945a..d097a97 100644 --- a/apps/extension/src/tools/vom/frame-capture.ts +++ b/apps/extension/src/tools/vom/frame-capture.ts @@ -56,6 +56,11 @@ async function captureMissingFrameDocuments( geometry, target: frame.target, }); + if (child.frameExcludedBackendNodeIds?.size) { + captured.frameExcludedBackendNodeIds ??= new Map(); + for (const [id, excluded] of child.frameExcludedBackendNodeIds) + captured.frameExcludedBackendNodeIds.set(id, excluded); + } if (child.frameGeometryIssues?.length) { (captured.frameGeometryIssues ??= []).push(...child.frameGeometryIssues); } @@ -174,5 +179,5 @@ export async function captureFrameData( if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, geometry, signal); throwIfAborted(signal); const batches = await captureAxTrees(cdp, tabId, frames, captured, signal); - return buildFrameDocuments(graph, batches, captured); + return buildFrameDocuments(graph, batches, captured, signal); } diff --git a/apps/extension/src/tools/vom/frame-document.ts b/apps/extension/src/tools/vom/frame-document.ts index 16e99a7..1020d5b 100644 --- a/apps/extension/src/tools/vom/frame-document.ts +++ b/apps/extension/src/tools/vom/frame-document.ts @@ -4,7 +4,8 @@ import { type CdpTarget, cdpTargetKey, } from "@/browser-driver/frame-graph"; -import type { CapturedNode, CapturedViewModel } from "./capture"; +import { createCaptureCheckpoint } from "./capture-abort"; +import type { CapturedNode } from "./capture-types"; export interface FrameOwnedAxNode { nodeId: string; @@ -23,6 +24,17 @@ export interface FrameDocument extends CdpFrame { contextScopeId: string; axNodes: T[]; domNodes: CapturedNode[]; + excludedBackendNodeIds?: ReadonlySet; +} + +/** Explicit snapshot document data used to join DOM and AX ownership. */ +export interface FrameDomInput { + nodes: CapturedNode[]; + rootFrameId?: string; + frameNodes?: ReadonlyMap; + frameOwnerBackendNodeIds?: ReadonlyMap; + frameParentIds?: ReadonlyMap; + frameExcludedBackendNodeIds?: ReadonlyMap>; } interface Ownership { @@ -47,7 +59,7 @@ function targetBackendKey(target: CdpTarget, backendNodeId: number): string { function frameList( graph: CdpFrameGraph | null, batches: FrameAxBatch[], - captured: CapturedViewModel, + captured: FrameDomInput, ): CdpFrame[] { const frames = new Map(); for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame); @@ -70,11 +82,14 @@ function frameList( }); } -export function buildFrameDocuments( +export async function buildFrameDocuments( graph: CdpFrameGraph | null, batches: FrameAxBatch[], - captured: CapturedViewModel, -): FrameDocument[] { + captured: FrameDomInput, + signal?: AbortSignal, +): Promise[]> { + const checkpoint = createCaptureCheckpoint(signal); + let work = 0; const frames = frameList(graph, batches, captured); const frameById = new Map(frames.map((frame) => [frame.frameId, frame])); const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId; @@ -83,54 +98,68 @@ export function buildFrameDocuments( const backendOwner = new Map(); for (const frame of frames) { for (const node of domNodesForFrame(frame.frameId)) { + if (work++ % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId); } } const candidates = new Map>(); for (const batch of batches) { - const nodeById = new Map(batch.nodes.map((node) => [node.nodeId, node])); - const ownershipByNodeId = new Map(); - const resolving = new Set(); - const resolveOwnership = (node: T): Ownership => { - const cached = ownershipByNodeId.get(node.nodeId); - if (cached) return cached; - if (node.frameId && frameById.has(node.frameId)) { - const ownership = { frameId: node.frameId, strength: 4 }; - ownershipByNodeId.set(node.nodeId, ownership); - return ownership; - } - if (typeof node.backendDOMNodeId === "number") { - const frameId = backendOwner.get( - targetBackendKey(batch.frame.target, node.backendDOMNodeId), - ); - if (frameId) { - const ownership = { frameId, strength: 3 }; - ownershipByNodeId.set(node.nodeId, ownership); - return ownership; - } - } - if (node.parentId && !resolving.has(node.nodeId)) { - const parent = nodeById.get(node.parentId); - if (parent) { - resolving.add(node.nodeId); - const inherited = resolveOwnership(parent); - resolving.delete(node.nodeId); - const ownership = { - frameId: inherited.frameId, - strength: Math.min(2, inherited.strength), - }; - ownershipByNodeId.set(node.nodeId, ownership); - return ownership; - } - } - const ownership = { frameId: batch.frame.frameId, strength: 1 }; - ownershipByNodeId.set(node.nodeId, ownership); - return ownership; - }; - + const nodeById = new Map(); for (const node of batch.nodes) { - const ownership = resolveOwnership(node); + if (work++ % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } + nodeById.set(node.nodeId, node); + } + const ownershipByNodeId = new Map(); + for (const node of batch.nodes) { + const path: T[] = []; + const visiting = new Set(); + let current: T | undefined = node; + let ownership: Ownership = { frameId: batch.frame.frameId, strength: 1 }; + while (current) { + if (work++ % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } + const cached = ownershipByNodeId.get(current.nodeId); + if (cached) { + ownership = cached; + break; + } + if (visiting.has(current.nodeId)) break; + visiting.add(current.nodeId); + if (current.frameId && frameById.has(current.frameId)) { + ownership = { frameId: current.frameId, strength: 4 }; + ownershipByNodeId.set(current.nodeId, ownership); + break; + } + const frameId = + typeof current.backendDOMNodeId === "number" + ? backendOwner.get(targetBackendKey(batch.frame.target, current.backendDOMNodeId)) + : undefined; + if (frameId) { + ownership = { frameId, strength: 3 }; + ownershipByNodeId.set(current.nodeId, ownership); + break; + } + path.push(current); + current = current.parentId ? nodeById.get(current.parentId) : undefined; + } + for (let i = path.length - 1; i >= 0; i--) { + if (work++ % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } + ownership = { frameId: ownership.frameId, strength: Math.min(2, ownership.strength) }; + ownershipByNodeId.set(path[i].nodeId, ownership); + } + if (!ownershipByNodeId.has(node.nodeId)) ownershipByNodeId.set(node.nodeId, ownership); const key = targetNodeKey(batch.frame.target, node.nodeId); const existing = candidates.get(key); if (!existing || ownership.strength > existing.ownership.strength) { @@ -144,6 +173,10 @@ export function buildFrameDocuments( ); const axNodesByFrame = new Map(); for (const candidate of candidates.values()) { + if (work++ % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } const { node, target, ownership } = candidate; const parentFrameId = node.parentId ? ownershipByTargetNode.get(targetNodeKey(target, node.parentId)) @@ -169,5 +202,6 @@ export function buildFrameDocuments( contextScopeId: frame.frameId, axNodes: axNodesByFrame.get(frame.frameId) ?? [], domNodes: domNodesForFrame(frame.frameId), + excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(frame.frameId), })); } diff --git a/apps/extension/src/tools/vom/normalize.ts b/apps/extension/src/tools/vom/normalize.ts new file mode 100644 index 0000000..66bbc4d --- /dev/null +++ b/apps/extension/src/tools/vom/normalize.ts @@ -0,0 +1,78 @@ +import type { CdpTarget } from "@/browser-driver/frame-graph"; +import { + type FrameProjectionState, + projectSnapshotRect, + type SnapshotCoordinates, + snapshotViewportRect, +} from "../geometry/coordinate-types"; +import { createCaptureCheckpoint } from "./capture-abort"; +import { buildDocumentIndex, type DocumentIndex, type NodeFacts } from "./facts"; +import { decodeDocument, type SnapshotDocument } from "./snapshot"; + +export interface FrameContext { + frameId?: string; + ownerFrameBackendNodeId: number | null; + projection: FrameProjectionState; + target: CdpTarget; + coordinates: SnapshotCoordinates | null; + layoutUnitsPerCssPixel: number | null; +} + +export interface NormalizedDocument { + nodes: NodeFacts[]; + index: DocumentIndex; +} + +/** Interpret one document using a supplied projection. No live reads or frame scheduling. */ +export async function normalizeDocument( + doc: SnapshotDocument, + strings: string[], + context: FrameContext, + signal?: AbortSignal, +): Promise { + const decoded = await decodeDocument(doc, strings, signal); + const checkpoint = createCaptureCheckpoint(signal); + const nodes: NodeFacts[] = []; + for (let i = 0; i < decoded.nodes.length; i++) { + if (i % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } + const node = decoded.nodes[i]; + const bounds = node.layout?.bounds ?? []; + const input = snapshotViewportRect( + bounds, + { target: context.target, frameId: context.frameId }, + context.coordinates, + ); + const local = input?.rect; + const rect = + input && context.projection.status === "available" + ? projectSnapshotRect(input, context.projection.projection) + : null; + const visibility = node.layout?.styles.visibility || "visible"; + const opacity = node.layout?.styles.opacity || "1"; + nodes.push({ + ...node, + ...(context.frameId ? { frameId: context.frameId } : {}), + ownerFrameBackendNodeId: context.ownerFrameBackendNodeId, + localRect: local ? { x: local.x, y: local.y, w: local.width, h: local.height } : null, + rect: rect ? { x: rect.x, y: rect.y, w: rect.width, h: rect.height } : null, + rendered: + bounds.length >= 4 && + bounds.slice(0, 4).every(Number.isFinite) && + bounds[2] > 0 && + bounds[3] > 0 && + visibility !== "hidden" && + visibility !== "collapse" && + (Number.parseFloat(opacity) || 0) > 0, + }); + } + const index = await buildDocumentIndex(nodes, signal); + return { + nodes: nodes.filter( + (node) => !node.tag.startsWith("#") && !index.excludedBackendNodeIds.has(node.backendNodeId), + ), + index, + }; +} diff --git a/apps/extension/src/tools/vom/record-safe-observation.ts b/apps/extension/src/tools/vom/record-safe-observation.ts index 13c4f5d..d87d49f 100644 --- a/apps/extension/src/tools/vom/record-safe-observation.ts +++ b/apps/extension/src/tools/vom/record-safe-observation.ts @@ -2,7 +2,10 @@ import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom"; import type { CdpTarget } from "@/browser-driver/frame-graph"; import { readRecordingDocumentIdentity } from "@/shared/recording-document-identity"; import type { CapturedSurfaceProbe } from "./capture"; -import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture"; +import type { + FrameDocument as CapturedFrameDocument, + FrameOwnedAxNode as FrameAxNode, +} from "./frame-document"; /** Frame identity needed to resolve an `@eN` that lives in an iframe. */ export interface CaptureVomFrame { diff --git a/apps/extension/src/tools/vom/semantic-graph/build.ts b/apps/extension/src/tools/vom/semantic-graph/build.ts index c1530bd..d40c608 100644 --- a/apps/extension/src/tools/vom/semantic-graph/build.ts +++ b/apps/extension/src/tools/vom/semantic-graph/build.ts @@ -61,8 +61,9 @@ export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGrap dom, sourceOrder: index, excluded: - document.frameId === rootFrameId && - input.excludedBackendNodeIds?.has(dom.backendNodeId) === true, + document.excludedBackendNodeIds?.has(dom.backendNodeId) === true || + (document.frameId === rootFrameId && + input.excludedBackendNodeIds?.has(dom.backendNodeId) === true), }; nodes.set(id, node); frameNodeIds.push(id); @@ -90,9 +91,10 @@ export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGrap ax, sourceOrder: document.domNodes.length + index, excluded: - document.frameId === rootFrameId && typeof ax.backendDOMNodeId === "number" && - input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true, + (document.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true || + (document.frameId === rootFrameId && + input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true)), }); frameNodeIds.push(id); } diff --git a/apps/extension/src/tools/vom/snapshot.ts b/apps/extension/src/tools/vom/snapshot.ts new file mode 100644 index 0000000..bac1b41 --- /dev/null +++ b/apps/extension/src/tools/vom/snapshot.ts @@ -0,0 +1,224 @@ +import { createCaptureCheckpoint } from "./capture-abort"; +import type { CapturedNode } from "./capture-types"; +import type { DecodedDocument, DecodedNode } from "./facts"; +export const REQUESTED_STYLES = [ + "position", + "pointer-events", + "cursor", + "visibility", + "opacity", +] as const; +const STYLE_COL = Object.fromEntries( + REQUESTED_STYLES.map((name, index) => [name, index]), +) as Record<(typeof REQUESTED_STYLES)[number], number>; +/** Sparse array format Chrome uses for infrequently-set per-node fields. */ +interface SparseArray { + index: number[]; + value: number[]; +} + +interface RareBooleanData { + index: number[]; +} + +export interface SnapshotDocument { + frameId?: string | number; + scrollOffsetX?: number; + scrollOffsetY?: number; + nodes?: { + parentIndex?: number[]; + nodeName?: number[]; + nodeType?: number[]; + backendNodeId?: number[]; + attributes?: number[][]; + /** + * Per-node text value (index into strings), set for `#text` / CDATA nodes. + * Element nodes carry -1. Same length as `backendNodeId`. + */ + nodeValue?: number[]; + /** Maps node array index → index into `documents[]` for frame content. */ + contentDocumentIndex?: SparseArray; + inputValue?: SparseArray; + textValue?: SparseArray; + inputChecked?: RareBooleanData; + optionSelected?: RareBooleanData; + }; + layout?: { + nodeIndex?: number[]; + styles?: number[][]; + bounds?: number[][]; + paintOrders?: number[]; + }; +} + +export function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined { + if (typeof document.frameId === "string") return document.frameId || undefined; + if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined; + return undefined; +} + +export interface SnapshotReply { + strings?: string[]; + documents?: SnapshotDocument[]; +} + +function isSensitiveFormControl(node: Pick): boolean { + return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password"; +} + +function snapshotFormState( + value: string | undefined, + defaultValue: string, + hasDefaultValue: boolean, + sensitive: boolean, +): CapturedNode["formState"] { + if (sensitive) { + if (value === undefined && !hasDefaultValue) return undefined; + return (value ?? defaultValue) === "" ? "empty" : "filled"; + } + if (value === undefined) return undefined; + if (value === "") return "empty"; + return value === defaultValue ? "default" : "filled"; +} + +function str(strings: string[], idx: number | undefined): string { + if (idx === undefined || idx < 0) return ""; + return strings[idx] ?? ""; +} + +export function sparseIndexMap(sparse: SparseArray | undefined): Map { + const out = new Map(); + if (!sparse?.index || !sparse.value) return out; + for (let i = 0; i < sparse.index.length; i++) { + const nodeIndex = sparse.index[i]; + const docIndex = sparse.value[i]; + if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex); + } + return out; +} + +export async function decodeDocument( + doc: SnapshotDocument, + strings: string[], + signal?: AbortSignal, +): Promise { + const checkpoint = createCaptureCheckpoint(signal); + const dn = doc.nodes; + const dl = doc.layout; + if (!dn?.backendNodeId) { + return { nodes: [] }; + } + + const count = dn.backendNodeId.length; + const layoutByNode = new Map(); + for (let i = 0; i < (dl?.nodeIndex?.length ?? 0); i++) { + if (i % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } + layoutByNode.set(dl!.nodeIndex![i], i); + } + + const inputValues = sparseIndexMap(dn.inputValue); + const textValues = sparseIndexMap(dn.textValue); + const checkedInputs = new Set(dn.inputChecked?.index ?? []); + const selectedOptions = new Set(dn.optionSelected?.index ?? []); + + // Collect visible text from #text child nodes so element CapturedNodes + // carry a textContent value usable as a button/link label fallback. + // nodeValue is a parallel array: string index for text nodes, -1 otherwise. + const nodeTextContent = new Map(); + if (dn.nodeValue) { + for (let n = 0; n < count; n++) { + if (n % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } + const nvIdx = dn.nodeValue[n] ?? -1; + if (nvIdx < 0) continue; + const text = str(strings, nvIdx).trim(); + if (!text) continue; + const parentIdx = dn.parentIndex?.[n] ?? -1; + if (parentIdx >= 0) { + const existing = nodeTextContent.get(parentIdx); + if (existing) existing.push(text); + else nodeTextContent.set(parentIdx, [text]); + } + } + } + + // Decode fields without assigning a coordinate projection or semantic policy. + const nodes: DecodedNode[] = []; + for (let n = 0; n < count; n++) { + if (n % 256 === 0) { + const pending = checkpoint(); + if (pending) await pending; + } + const backendNodeId = dn.backendNodeId[n]; + const parentIdx = dn.parentIndex?.[n] ?? -1; + const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null; + const tag = str(strings, dn.nodeName?.[n]).toLowerCase(); + + const attrs: Record = {}; + const pairs = dn.attributes?.[n] ?? []; + for (let a = 0; a + 1 < pairs.length; a += 2) { + attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]); + } + + const li = layoutByNode.get(n); + const styleRow = li === undefined ? [] : (dl?.styles?.[li] ?? []); + const styles: Record = {}; + for (const name of REQUESTED_STYLES) styles[name] = str(strings, styleRow[STYLE_COL[name]]); + const layout = + li === undefined + ? undefined + : { + boundsSpace: "snapshot-document-layout" as const, + bounds: dl?.bounds?.[li], + styles, + }; + + const textContent = nodeTextContent.get(n)?.join(" "); + + const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n); + const rawFormValue = + rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined; + const sensitive = isSensitiveFormControl({ tag, attrs }); + const formDefaultValue = attrs.value ?? ""; + const formValue = sensitive ? undefined : rawFormValue; + const formState = snapshotFormState( + rawFormValue, + formDefaultValue, + Object.prototype.hasOwnProperty.call(attrs, "value"), + sensitive, + ); + if (sensitive) delete attrs.value; + + nodes.push({ + backendNodeId, + nodeType: dn.nodeType?.[n], + parentBackendNodeId, + + tag, + attrs, + layout, + paintOrder: li === undefined ? 0 : (dl?.paintOrders?.[li] ?? 0), + position: styles.position || "static", + pointerEvents: styles["pointer-events"] || "auto", + cursor: styles.cursor || "auto", + + textContent, + ...(formValue !== undefined ? { formValue } : {}), + ...(tag === "input" || tag === "textarea" + ? { + formPlaceholder: attrs.placeholder ?? "", + ...(!sensitive ? { formDefaultValue } : {}), + ...(formState ? { formState } : {}), + } + : {}), + ...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}), + ...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}), + }); + } + return { nodes }; +}