refactor(observation): defer visual facts until visual discovery

apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts

@@ -126,7 +126,7 @@ describe("captureObservationFacts", () => {
     const main = facts.documents.find((doc) => doc.frame.frameId === "main")!;
     const remote = facts.documents.find((doc) => doc.frame.frameId === "remote")!;
     expect(main.index.nodes.get(2)).not.toBe(remote.index.nodes.get(2));
-    expect(main.index.children.get(1)).toEqual([2]);
+    expect(main.index.nodes.get(2)?.parentBackendNodeId).toBe(1);
     expect(main.index.nodes.get(10001)?.nodeType).toBe(10);
     expect(remote.axNodes[0].frameId).toBe("remote");
     expect(main.index.nodes.get(2)).toBe(main.domNodes.find((node) => node.backendNodeId === 2));

apps/extension/src/tools/vom/__tests__/facts.test.ts

@@ -23,6 +23,7 @@ describe("document facts", () => {
       let reads = 0;
       const input = Array.from({ length: count }, (_, i) => {
         const item = node(i, i ? (wide ? 0 : i - 1) : null);
+        if (i === 0) item.attrs[OVERLAY_HOST_MARKER_ATTR] = "";
         Object.defineProperty(item, "parentBackendNodeId", {
           get: () => {
             reads++;
@@ -33,14 +34,13 @@ describe("document facts", () => {
       }).reverse(); // parent need not precede child
       const index = await buildDocumentIndex(input);
       expect(index.nodes.size).toBe(count);
-      expect(index.ancestry.get(count - 1)).toEqual({ complete: true, overlay: false });
+      expect(index.excludedBackendNodeIds.size).toBe(count);
-      expect(index.children.get(0)?.length).toBe(wide ? count - 1 : 1);
       expect(reads).toBeLessThanOrEqual(count * 8);
       expect(index.nodes.get(count - 1)).toBe(input[0]);
     }
   });
 
-  it("marks cycles and orphans incomplete and propagates overlay through shadow roots", async () => {
+  it("handles cycles and orphans and propagates overlay through shadow roots", async () => {
     const host = { ...node(1, null), attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } };
     const input = [
       node(3, 2),
@@ -53,7 +53,7 @@ describe("document facts", () => {
     ];
     const index = await buildDocumentIndex(input);
     expect([...index.excludedBackendNodeIds].sort()).toEqual([1, 2, 3]);
-    for (const id of [4, 5, 6, 7]) expect(index.ancestry.get(id)?.complete).toBe(false);
+    expect(index.nodes.size).toBe(input.length);
   });
 
   it("does not propagate a descendant overlay backwards into a cycle", async () => {
@@ -62,74 +62,28 @@ describe("document facts", () => {
     expect([...index.excludedBackendNodeIds]).toEqual([3]);
   });
 
-  it("does not treat a missing snapshot parent as a complete root", async () => {
+  it("decodes current observation bounds, styles and attributes without projecting coordinates", async () => {
-    const { nodes } = await decodeDocument(
+    const strings = ["div", "aria-hidden", "true", "inert", "", "visible", "0", "static", "auto"];
-      { nodes: { backendNodeId: [1], nodeName: [0], parentIndex: [99] } },
-      ["div"],
-    );
-    const index = await buildDocumentIndex(nodes);
-    expect(index.ancestry.get(1)?.complete).toBe(false);
-  });
-
-  it("retains raw box units and independent style/interaction evidence", async () => {
-    const strings = [
-      "html",
-      "canvas",
-      "aria-hidden",
-      "true",
-      "inert",
-      "",
-      "hidden",
-      "visible",
-      "0",
-      "auto",
-      "scroll",
-      "matrix(2,0,0,2,0,0)",
-    ];
     const styles = REQUESTED_STYLES.map((style) =>
-      style === "visibility"
+      style === "visibility" ? 5 : style === "opacity" ? 6 : style === "position" ? 7 : 8,
-        ? 7
-        : style === "opacity"
-          ? 8
-          : style === "overflow-x"
-            ? 10
-            : style === "transform"
-              ? 11
-              : 9,
     );
     const { nodes } = await decodeDocument(
       {
-        nodes: {
+        nodes: { backendNodeId: [1], parentIndex: [-1], nodeName: [0], attributes: [[1, 2, 3, 4]] },
-          backendNodeId: [1, 2],
+        layout: { nodeIndex: [0], bounds: [[10, 20, 120, 40]], styles: [styles] },
-          parentIndex: [-1, 0],
-          nodeName: [0, 1],
-          attributes: [[2, 3, 4, 5], []],
-        },
-        layout: {
-          nodeIndex: [0, 1],
-          bounds: [
-            [0, 0, 200, 100],
-            [10, 20, 120, 40],
-          ],
-          clientRects: [
-            [5, 5, 90, 40],
-            [0, 0, 120, 40],
-          ],
-          styles: [styles, styles],
-        },
       },
       strings,
     );
     expect(nodes[0].attrs).toEqual({ "aria-hidden": "true", inert: "" });
-    expect(nodes[0].layout).toMatchObject({
+    expect(nodes[0].layout).toEqual({
       boundsSpace: "snapshot-document-css",
-      clientSpace: "unscaled-client-offset-css",
+      bounds: [10, 20, 120, 40],
-      clientRect: [5, 5, 90, 40],
       styles: {
+        position: "static",
+        "pointer-events": "auto",
+        cursor: "auto",
         visibility: "visible",
         opacity: "0",
-        "overflow-x": "scroll",
-        transform: "matrix(2,0,0,2,0,0)",
       },
     });
     expect(nodes[0]).not.toHaveProperty("rect");

apps/extension/src/tools/vom/facts.ts

@@ -56,29 +56,18 @@ export interface CapturedSceneInput {
   excludedBackendNodeIds: ReadonlySet<number>;
 }
 
-/** Snapshot client/offset rectangles retain their protocol units. In particular,
+/** Bounds retain their snapshot document CSS units until normalization. */
- * transformed bounds cannot be combined with unscaled client offsets. */
 export interface SnapshotLayout {
   readonly boundsSpace: "snapshot-document-css";
-  readonly clientSpace: "unscaled-client-offset-css";
   bounds?: number[];
-  clientRect?: number[];
-  offsetRect?: number[];
-  scrollRect?: number[];
   styles: Readonly<Record<string, string>>;
 }
 
 export interface DecodedNode extends Omit<CapturedNode, "rect" | "localRect" | "rendered"> {
   nodeType?: number;
-  parentMissing?: boolean;
   layout?: SnapshotLayout;
 }
 
-export interface AncestorState {
-  complete: boolean;
-  overlay: boolean;
-}
-
 export interface NodeFacts extends CapturedNode {
   nodeType?: number;
   layout?: SnapshotLayout;
@@ -86,8 +75,6 @@ export interface NodeFacts extends CapturedNode {
 
 export interface DocumentIndex<T extends DecodedNode = NodeFacts> {
   readonly nodes: ReadonlyMap<number, T>;
-  readonly children: ReadonlyMap<number, readonly number[]>;
-  readonly ancestry: ReadonlyMap<number, AncestorState>;
   readonly excludedBackendNodeIds: ReadonlySet<number>;
 }
 
@@ -144,67 +131,53 @@ export async function captureCheckpoint(signal?: AbortSignal): Promise<void> {
 }
 
 /** All snapshot nodes participate, including document and shadow roots. This
- * preserves ancestry evidence even when the semantic adapter omits those nodes. */
+ * preserves overlay propagation when the semantic adapter omits those nodes. */
 export async function buildDocumentIndex<T extends DecodedNode>(
   input: readonly T[],
   signal?: AbortSignal,
 ): Promise<DocumentIndex<T>> {
   const nodes = new Map<number, T>();
-  const children = new Map<number, number[]>();
+  const overlayByNode = new Map<number, boolean>();
-  const ancestry = new Map<number, AncestorState>();
   const excludedBackendNodeIds = new Set<number>();
   for (let i = 0; i < input.length; i++) {
     if (i % 256 === 0) await captureCheckpoint(signal);
     const node = input[i];
     nodes.set(node.backendNodeId, node);
-    if (node.parentBackendNodeId !== null) {
-      const siblings = children.get(node.parentBackendNodeId);
-      if (siblings) siblings.push(node.backendNodeId);
-      else children.set(node.parentBackendNodeId, [node.backendNodeId]);
-    }
   }
   let work = 0;
   for (const node of input) {
     if (work++ % 256 === 0) await captureCheckpoint(signal);
-    if (ancestry.has(node.backendNodeId)) continue;
+    if (overlayByNode.has(node.backendNodeId)) continue;
     const path: DecodedNode[] = [];
     const visiting = new Set<number>();
     let current: DecodedNode | undefined = node;
-    let state: AncestorState = { complete: true, overlay: false };
+    let overlay = false;
-    while (current && !ancestry.has(current.backendNodeId)) {
+    while (current && !overlayByNode.has(current.backendNodeId)) {
       if (work++ % 256 === 0) await captureCheckpoint(signal);
       if (visiting.has(current.backendNodeId)) {
-        // No member of a malformed cycle supplies complete ancestry evidence.
+        // Only an overlay inside the cycle may mark the cycle and its descendants.
-        state = {
+        overlay = path
-          complete: false,
+          .slice(path.findIndex((item) => item.backendNodeId === current!.backendNodeId))
-          overlay: path
+          .some((item) => isOverlayHostNode(item.tag, Object.keys(item.attrs)));
-            .slice(path.findIndex((item) => item.backendNodeId === current!.backendNodeId))
-            .some((item) => isOverlayHostNode(item.tag, Object.keys(item.attrs))),
-        };
         break;
       }
       visiting.add(current.backendNodeId);
       path.push(current);
-      if (current.parentMissing) state = { complete: false, overlay: false };
       if (current.parentBackendNodeId === null) {
         current = undefined;
         break;
       }
       current = nodes.get(current.parentBackendNodeId);
-      if (!current) state = { complete: false, overlay: false };
     }
-    if (current && ancestry.has(current.backendNodeId))
+    if (current && overlayByNode.has(current.backendNodeId))
-      state = ancestry.get(current.backendNodeId)!;
+      overlay = overlayByNode.get(current.backendNodeId)!;
     for (let i = path.length - 1; i >= 0; i--) {
       if (work++ % 256 === 0) await captureCheckpoint(signal);
       const item = path[i];
-      state = {
+      overlay = overlay || isOverlayHostNode(item.tag, Object.keys(item.attrs));
-        complete: state.complete,
+      overlayByNode.set(item.backendNodeId, overlay);
-        overlay: state.overlay || isOverlayHostNode(item.tag, Object.keys(item.attrs)),
+      if (overlay) excludedBackendNodeIds.add(item.backendNodeId);
-      };
-      ancestry.set(item.backendNodeId, state);
-      if (state.overlay) excludedBackendNodeIds.add(item.backendNodeId);
     }
   }
-  return { nodes, children, ancestry, excludedBackendNodeIds };
+  return { nodes, excludedBackendNodeIds };
 }

apps/extension/src/tools/vom/snapshot.ts

@@ -6,13 +6,6 @@ export const REQUESTED_STYLES = [
   "cursor",
   "visibility",
   "opacity",
-  "display",
-  "overflow-x",
-  "overflow-y",
-  "transform",
-  "zoom",
-  "clip-path",
-  "mask-image",
 ] as const;
 const STYLE_COL = Object.fromEntries(
   REQUESTED_STYLES.map((name, index) => [name, index]),
@@ -54,9 +47,6 @@ export interface SnapshotDocument {
     styles?: number[][];
     bounds?: number[][];
     paintOrders?: number[];
-    clientRects?: number[][];
-    offsetRects?: number[][];
-    scrollRects?: number[][];
   };
 }
 
@@ -173,11 +163,7 @@ export async function decodeDocument(
         ? undefined
         : {
             boundsSpace: "snapshot-document-css" as const,
-            clientSpace: "unscaled-client-offset-css" as const,
             bounds: dl?.bounds?.[li],
-            clientRect: dl?.clientRects?.[li],
-            offsetRect: dl?.offsetRects?.[li],
-            scrollRect: dl?.scrollRects?.[li],
             styles,
           };
 
@@ -201,9 +187,6 @@ export async function decodeDocument(
       backendNodeId,
       nodeType: dn.nodeType?.[n],
       parentBackendNodeId,
-      ...(parentIdx >= 0 && dn.backendNodeId[parentIdx] === undefined
-        ? { parentMissing: true }
-        : {}),
 
       tag,
       attrs,