From bcb9beb4f885e12ce87012320b3322d55087b5de Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Tue, 8 Sep 2026 20:42:05 +0800 Subject: [PATCH] refactor(observation): defer visual facts until visual discovery --- .../vom/__tests__/capture-coordinator.test.ts | 2 +- .../src/tools/vom/__tests__/facts.test.ts | 74 ++++--------------- apps/extension/src/tools/vom/facts.ts | 59 ++++----------- apps/extension/src/tools/vom/snapshot.ts | 17 ----- 4 files changed, 31 insertions(+), 121 deletions(-) diff --git a/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts b/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts index 624a2ed..981019d 100644 --- a/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts @@ -126,7 +126,7 @@ describe("captureObservationFacts", () => { const main = facts.documents.find((doc) => doc.frame.frameId === "main")!; const remote = facts.documents.find((doc) => doc.frame.frameId === "remote")!; expect(main.index.nodes.get(2)).not.toBe(remote.index.nodes.get(2)); - expect(main.index.children.get(1)).toEqual([2]); + expect(main.index.nodes.get(2)?.parentBackendNodeId).toBe(1); expect(main.index.nodes.get(10001)?.nodeType).toBe(10); expect(remote.axNodes[0].frameId).toBe("remote"); expect(main.index.nodes.get(2)).toBe(main.domNodes.find((node) => node.backendNodeId === 2)); diff --git a/apps/extension/src/tools/vom/__tests__/facts.test.ts b/apps/extension/src/tools/vom/__tests__/facts.test.ts index 1ab108c..09247db 100644 --- a/apps/extension/src/tools/vom/__tests__/facts.test.ts +++ b/apps/extension/src/tools/vom/__tests__/facts.test.ts @@ -23,6 +23,7 @@ describe("document facts", () => { let reads = 0; const input = Array.from({ length: count }, (_, i) => { const item = node(i, i ? (wide ? 0 : i - 1) : null); + if (i === 0) item.attrs[OVERLAY_HOST_MARKER_ATTR] = ""; Object.defineProperty(item, "parentBackendNodeId", { get: () => { reads++; @@ -33,14 +34,13 @@ describe("document facts", () => { }).reverse(); // parent need not precede child const index = await buildDocumentIndex(input); expect(index.nodes.size).toBe(count); - expect(index.ancestry.get(count - 1)).toEqual({ complete: true, overlay: false }); - expect(index.children.get(0)?.length).toBe(wide ? count - 1 : 1); + expect(index.excludedBackendNodeIds.size).toBe(count); expect(reads).toBeLessThanOrEqual(count * 8); expect(index.nodes.get(count - 1)).toBe(input[0]); } }); - it("marks cycles and orphans incomplete and propagates overlay through shadow roots", async () => { + it("handles cycles and orphans and propagates overlay through shadow roots", async () => { const host = { ...node(1, null), attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } }; const input = [ node(3, 2), @@ -53,7 +53,7 @@ describe("document facts", () => { ]; const index = await buildDocumentIndex(input); expect([...index.excludedBackendNodeIds].sort()).toEqual([1, 2, 3]); - for (const id of [4, 5, 6, 7]) expect(index.ancestry.get(id)?.complete).toBe(false); + expect(index.nodes.size).toBe(input.length); }); it("does not propagate a descendant overlay backwards into a cycle", async () => { @@ -62,74 +62,28 @@ describe("document facts", () => { expect([...index.excludedBackendNodeIds]).toEqual([3]); }); - it("does not treat a missing snapshot parent as a complete root", async () => { - const { nodes } = await decodeDocument( - { nodes: { backendNodeId: [1], nodeName: [0], parentIndex: [99] } }, - ["div"], - ); - const index = await buildDocumentIndex(nodes); - expect(index.ancestry.get(1)?.complete).toBe(false); - }); - - it("retains raw box units and independent style/interaction evidence", async () => { - const strings = [ - "html", - "canvas", - "aria-hidden", - "true", - "inert", - "", - "hidden", - "visible", - "0", - "auto", - "scroll", - "matrix(2,0,0,2,0,0)", - ]; + it("decodes current observation bounds, styles and attributes without projecting coordinates", async () => { + const strings = ["div", "aria-hidden", "true", "inert", "", "visible", "0", "static", "auto"]; const styles = REQUESTED_STYLES.map((style) => - style === "visibility" - ? 7 - : style === "opacity" - ? 8 - : style === "overflow-x" - ? 10 - : style === "transform" - ? 11 - : 9, + style === "visibility" ? 5 : style === "opacity" ? 6 : style === "position" ? 7 : 8, ); const { nodes } = await decodeDocument( { - nodes: { - backendNodeId: [1, 2], - parentIndex: [-1, 0], - nodeName: [0, 1], - attributes: [[2, 3, 4, 5], []], - }, - layout: { - nodeIndex: [0, 1], - bounds: [ - [0, 0, 200, 100], - [10, 20, 120, 40], - ], - clientRects: [ - [5, 5, 90, 40], - [0, 0, 120, 40], - ], - styles: [styles, styles], - }, + nodes: { backendNodeId: [1], parentIndex: [-1], nodeName: [0], attributes: [[1, 2, 3, 4]] }, + layout: { nodeIndex: [0], bounds: [[10, 20, 120, 40]], styles: [styles] }, }, strings, ); expect(nodes[0].attrs).toEqual({ "aria-hidden": "true", inert: "" }); - expect(nodes[0].layout).toMatchObject({ + expect(nodes[0].layout).toEqual({ boundsSpace: "snapshot-document-css", - clientSpace: "unscaled-client-offset-css", - clientRect: [5, 5, 90, 40], + bounds: [10, 20, 120, 40], styles: { + position: "static", + "pointer-events": "auto", + cursor: "auto", visibility: "visible", opacity: "0", - "overflow-x": "scroll", - transform: "matrix(2,0,0,2,0,0)", }, }); expect(nodes[0]).not.toHaveProperty("rect"); diff --git a/apps/extension/src/tools/vom/facts.ts b/apps/extension/src/tools/vom/facts.ts index 8674125..f973271 100644 --- a/apps/extension/src/tools/vom/facts.ts +++ b/apps/extension/src/tools/vom/facts.ts @@ -56,29 +56,18 @@ export interface CapturedSceneInput { excludedBackendNodeIds: ReadonlySet; } -/** Snapshot client/offset rectangles retain their protocol units. In particular, - * transformed bounds cannot be combined with unscaled client offsets. */ +/** Bounds retain their snapshot document CSS units until normalization. */ export interface SnapshotLayout { readonly boundsSpace: "snapshot-document-css"; - readonly clientSpace: "unscaled-client-offset-css"; bounds?: number[]; - clientRect?: number[]; - offsetRect?: number[]; - scrollRect?: number[]; styles: Readonly>; } export interface DecodedNode extends Omit { nodeType?: number; - parentMissing?: boolean; layout?: SnapshotLayout; } -export interface AncestorState { - complete: boolean; - overlay: boolean; -} - export interface NodeFacts extends CapturedNode { nodeType?: number; layout?: SnapshotLayout; @@ -86,8 +75,6 @@ export interface NodeFacts extends CapturedNode { export interface DocumentIndex { readonly nodes: ReadonlyMap; - readonly children: ReadonlyMap; - readonly ancestry: ReadonlyMap; readonly excludedBackendNodeIds: ReadonlySet; } @@ -144,67 +131,53 @@ export async function captureCheckpoint(signal?: AbortSignal): Promise { } /** All snapshot nodes participate, including document and shadow roots. This - * preserves ancestry evidence even when the semantic adapter omits those nodes. */ + * preserves overlay propagation when the semantic adapter omits those nodes. */ export async function buildDocumentIndex( input: readonly T[], signal?: AbortSignal, ): Promise> { const nodes = new Map(); - const children = new Map(); - const ancestry = new Map(); + const overlayByNode = new Map(); const excludedBackendNodeIds = new Set(); for (let i = 0; i < input.length; i++) { if (i % 256 === 0) await captureCheckpoint(signal); const node = input[i]; nodes.set(node.backendNodeId, node); - if (node.parentBackendNodeId !== null) { - const siblings = children.get(node.parentBackendNodeId); - if (siblings) siblings.push(node.backendNodeId); - else children.set(node.parentBackendNodeId, [node.backendNodeId]); - } } let work = 0; for (const node of input) { if (work++ % 256 === 0) await captureCheckpoint(signal); - if (ancestry.has(node.backendNodeId)) continue; + if (overlayByNode.has(node.backendNodeId)) continue; const path: DecodedNode[] = []; const visiting = new Set(); let current: DecodedNode | undefined = node; - let state: AncestorState = { complete: true, overlay: false }; - while (current && !ancestry.has(current.backendNodeId)) { + let overlay = false; + while (current && !overlayByNode.has(current.backendNodeId)) { if (work++ % 256 === 0) await captureCheckpoint(signal); if (visiting.has(current.backendNodeId)) { - // No member of a malformed cycle supplies complete ancestry evidence. - state = { - complete: false, - overlay: path - .slice(path.findIndex((item) => item.backendNodeId === current!.backendNodeId)) - .some((item) => isOverlayHostNode(item.tag, Object.keys(item.attrs))), - }; + // Only an overlay inside the cycle may mark the cycle and its descendants. + overlay = path + .slice(path.findIndex((item) => item.backendNodeId === current!.backendNodeId)) + .some((item) => isOverlayHostNode(item.tag, Object.keys(item.attrs))); break; } visiting.add(current.backendNodeId); path.push(current); - if (current.parentMissing) state = { complete: false, overlay: false }; if (current.parentBackendNodeId === null) { current = undefined; break; } current = nodes.get(current.parentBackendNodeId); - if (!current) state = { complete: false, overlay: false }; } - if (current && ancestry.has(current.backendNodeId)) - state = ancestry.get(current.backendNodeId)!; + if (current && overlayByNode.has(current.backendNodeId)) + overlay = overlayByNode.get(current.backendNodeId)!; for (let i = path.length - 1; i >= 0; i--) { if (work++ % 256 === 0) await captureCheckpoint(signal); const item = path[i]; - state = { - complete: state.complete, - overlay: state.overlay || isOverlayHostNode(item.tag, Object.keys(item.attrs)), - }; - ancestry.set(item.backendNodeId, state); - if (state.overlay) excludedBackendNodeIds.add(item.backendNodeId); + overlay = overlay || isOverlayHostNode(item.tag, Object.keys(item.attrs)); + overlayByNode.set(item.backendNodeId, overlay); + if (overlay) excludedBackendNodeIds.add(item.backendNodeId); } } - return { nodes, children, ancestry, excludedBackendNodeIds }; + return { nodes, excludedBackendNodeIds }; } diff --git a/apps/extension/src/tools/vom/snapshot.ts b/apps/extension/src/tools/vom/snapshot.ts index 70dfe45..42fc5d4 100644 --- a/apps/extension/src/tools/vom/snapshot.ts +++ b/apps/extension/src/tools/vom/snapshot.ts @@ -6,13 +6,6 @@ export const REQUESTED_STYLES = [ "cursor", "visibility", "opacity", - "display", - "overflow-x", - "overflow-y", - "transform", - "zoom", - "clip-path", - "mask-image", ] as const; const STYLE_COL = Object.fromEntries( REQUESTED_STYLES.map((name, index) => [name, index]), @@ -54,9 +47,6 @@ export interface SnapshotDocument { styles?: number[][]; bounds?: number[][]; paintOrders?: number[]; - clientRects?: number[][]; - offsetRects?: number[][]; - scrollRects?: number[][]; }; } @@ -173,11 +163,7 @@ export async function decodeDocument( ? undefined : { boundsSpace: "snapshot-document-css" as const, - clientSpace: "unscaled-client-offset-css" as const, bounds: dl?.bounds?.[li], - clientRect: dl?.clientRects?.[li], - offsetRect: dl?.offsetRects?.[li], - scrollRect: dl?.scrollRects?.[li], styles, }; @@ -201,9 +187,6 @@ export async function decodeDocument( backendNodeId, nodeType: dn.nodeType?.[n], parentBackendNodeId, - ...(parentIdx >= 0 && dn.backendNodeId[parentIdx] === undefined - ? { parentMissing: true } - : {}), tag, attrs,