fix(geometry): unify frame projection and screenshot coordinates

2026-09-10 13:01:23 +08:00 · 2026-09-09 13:20:21 +08:00
parent 90b0ff301b
commit 1485addfd5
11 changed files with 899 additions and 305 deletions
@@ -754,7 +754,7 @@ describe("captureViewModel", () => {
    expect(div?.rect).toMatchObject({ y: 0, h: 600 });
  });

-  it("normalizes device-pixel bounds by devicePixelRatio", async () => {
+  it("keeps CSS snapshot bounds independent of legacy metrics ratio", async () => {
    const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
    const i = (s: string) => S.indexOf(s);
    const snapshot = {
@@ -802,6 +802,12 @@ describe("captureViewModel", () => {
      w: 1000,
      h: 800,
    });
+    expect(nodes.find((n) => n.backendNodeId === 12)?.localRect).toEqual({
+      x: 0,
+      y: 0,
+      w: 2000,
+      h: 1600,
+    });
  });

  it("extracts textContent from #text child nodes via nodeValue", async () => {
@@ -950,6 +956,12 @@ describe("captureViewModel", () => {
            cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
          };
        }
+        if (method === "DOM.getBoxModel")
+          return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } };
+        if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
+        if (method === "Runtime.callFunctionOn")
+          return { result: { value: { width: 800, height: 400 } } };
+        if (method === "Runtime.releaseObject") return {};
        throw new Error(method);
      }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
    };
@@ -979,7 +991,7 @@ describe("captureViewModel", () => {
    expect(frameParentIds?.get("child-frame")).toBe("main-frame");
  });

-  it("captures iframe documents without requiring owner-frame geometry", async () => {
+  it("projects iframe documents independently of owner snapshot geometry", async () => {
    const S = [
      "html",
      "body",
@@ -1040,6 +1052,12 @@ describe("captureViewModel", () => {
            cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
          };
        }
+        if (method === "DOM.getBoxModel")
+          return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } };
+        if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
+        if (method === "Runtime.callFunctionOn")
+          return { result: { value: { width: 800, height: 400 } } };
+        if (method === "Runtime.releaseObject") return {};
        throw new Error(method);
      }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
    };
@@ -1048,7 +1066,7 @@ describe("captureViewModel", () => {
    const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);

    expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
-    expect(input?.rect).toBeNull();
+    expect(input?.rect).toEqual({ x: 120, y: 320, w: 120, h: 60 });
  });

  it("recursively normalizes nested iframe sub-documents", async () => {
@@ -1127,16 +1145,43 @@ describe("captureViewModel", () => {
      ],
    };
    const cdp = {
-      send: vi.fn(async (_t: number, method: string) => {
-        if (method === "DOMSnapshot.enable") return {};
-        if (method === "DOMSnapshot.captureSnapshot") return snapshot;
-        if (method === "Page.getLayoutMetrics") {
-          return {
-            cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
-          };
-        }
-        throw new Error(method);
-      }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
+      send: vi.fn(
+        async (
+          _t: number,
+          method: string,
+          params?: { backendNodeId?: number; objectId?: string },
+        ) => {
+          if (method === "DOMSnapshot.enable") return {};
+          if (method === "DOMSnapshot.captureSnapshot") return snapshot;
+          if (method === "Page.getLayoutMetrics") {
+            return {
+              cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
+            };
+          }
+          if (method === "DOM.getBoxModel")
+            return {
+              model: {
+                content:
+                  params?.backendNodeId === 13
+                    ? [10, 20, 310, 20, 310, 220, 10, 220]
+                    : [15, 26, 115, 26, 115, 106, 15, 106],
+              },
+            };
+          if (method === "DOM.resolveNode")
+            return { object: { objectId: String(params?.backendNodeId) } };
+          if (method === "Runtime.callFunctionOn")
+            return {
+              result: {
+                value:
+                  params?.objectId === "13"
+                    ? { width: 300, height: 200 }
+                    : { width: 100, height: 80 },
+              },
+            };
+          if (method === "Runtime.releaseObject") return {};
+          throw new Error(method);
+        },
+      ) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
    };

    const { iframeNodes } = await captureViewModel(cdp, 4);
@@ -1150,7 +1195,7 @@ describe("captureViewModel", () => {
    expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
  });

-  it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => {
+  it("subtracts CSS scroll once without dividing snapshot bounds by a metrics ratio", async () => {
    const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
    const i = (s: string) => S.indexOf(s);
    const snapshot = {
@@ -1192,7 +1237,7 @@ describe("captureViewModel", () => {
      }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
    };
    const { nodes } = await captureViewModel(cdp, 4);
-    expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 / 2 - 100);
+    expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 - 100);
  });

  it("collectOverlayExcludedBackendIds walks the pierced overlay host subtree", async () => {
@@ -5,9 +5,15 @@
 // STYLE_COL so capture and visibility policy share one explicit contract.

 import type { Rect, Viewport } from "@browser-skill/vom";
+import type { CdpTarget } from "@/browser-driver/frame-graph";
 import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
 import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
-import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
+import {
+  projectSnapshotRect,
+  type SnapshotProjection,
+  snapshotViewportRect,
+} from "../geometry/coordinate-types";
+import { cssViewport, GeometryContext, type LayoutMetrics } from "../geometry/frame-context";
 import type { CdpRunner } from "../shared";
 import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";

@@ -77,6 +83,8 @@ export interface CapturedViewModel {

 export interface CaptureViewModelOptions {
  signal?: AbortSignal;
+  geometry?: GeometryContext;
+  target?: CdpTarget;
 }

 function captureAbortError(): Error {
@@ -147,16 +155,6 @@ interface SnapshotReply {
  documents?: SnapshotDocument[];
 }

-interface LayoutMetricsReply {
-  cssLayoutViewport?: {
-    clientWidth?: number;
-    clientHeight?: number;
-    pageX?: number;
-    pageY?: number;
-  };
-  layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
-}
-
 function isFormControlTag(tag: string): boolean {
  return tag === "input" || tag === "textarea" || tag === "select";
 }
@@ -353,7 +351,8 @@ interface ParseDocumentResult {
 interface FrameContext {
  frameId?: string;
  ownerFrameBackendNodeId: number | null;
-  projection: GeometryProjection | null;
+  projection: SnapshotProjection | null;
+  target: CdpTarget;
  scrollX: number;
  scrollY: number;
 }
@@ -363,15 +362,6 @@ function str(strings: string[], idx: number | undefined): string {
  return strings[idx] ?? "";
 }

-function devicePixelRatio(metrics: LayoutMetricsReply): number {
-  const layoutW = metrics.layoutViewport?.clientWidth ?? 0;
-  const cssW = metrics.cssLayoutViewport?.clientWidth ?? 0;
-  if (!layoutW || !cssW) return 1;
-  const dpr = layoutW / cssW;
-  if (!Number.isFinite(dpr) || dpr <= 0) return 1;
-  return dpr >= 1 ? dpr : 1;
-}
-
 function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
  const out = new Map<number, number>();
  if (!sparse?.index || !sparse.value) return out;
@@ -797,13 +787,11 @@ export async function collectOverlayExcludedBackendIds(
 *
 * @param doc   - the raw DOMSnapshot document object
 * @param strings - the shared string table for the whole snapshot
- * @param dpr   - device-pixel-ratio from Page.getLayoutMetrics
 * @param context - frame coordinate context; output rects are top viewport-relative
 */
 function parseDocumentNodes(
  doc: SnapshotDocument,
  strings: string[],
-  dpr: number,
  context: FrameContext,
 ): ParseDocumentResult {
  const dn = doc.nodes;
@@ -898,15 +886,17 @@ function parseDocumentNodes(
    if (li !== undefined) {
      const b = dl?.bounds?.[li];
      if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
-        localRect = {
-          x: b[0] / dpr - context.scrollX,
-          y: b[1] / dpr - context.scrollY,
-          w: b[2] / dpr,
-          h: b[3] / dpr,
-        };
-        const bounds = context.projection
-          ? projectRectToViewport(localRect, context.projection)
-          : null;
+        const input = snapshotViewportRect(
+          b,
+          { target: context.target, frameId: context.frameId },
+          { x: context.scrollX, y: context.scrollY },
+        );
+        if (input) {
+          const { x, y, width: w, height: h } = input.rect;
+          localRect = { x, y, w, h };
+        }
+        const bounds =
+          input && context.projection ? projectSnapshotRect(input, context.projection) : null;
        rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
      }
      paintOrder = dl?.paintOrders?.[li] ?? 0;
@@ -980,15 +970,14 @@ interface ParseFrameDocumentsResult {
  excludedBackendNodeIds: Set<number>;
 }

-function parseChildFrameDocuments(
+async function parseChildFrameDocuments(
  documents: SnapshotDocument[],
  strings: string[],
-  dpr: number,
  parentDocIndex: number,
-  parentNodes: CapturedNode[],
  parentContext: FrameContext,
+  geometry: GeometryContext,
  visited = new Set<number>(),
-): ParseFrameDocumentsResult {
+): Promise<ParseFrameDocumentsResult> {
  const iframeNodes: CapturedIframeNodes = new Map();
  const frameOwnerBackendNodeIds = new Map<string, number>();
  const frameParentIds = new Map<string, string>();
@@ -1000,7 +989,6 @@ function parseChildFrameDocuments(
  }

  const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
-  const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));

  for (const [nodeArrayIdx, childDocIndex] of cdi) {
    if (visited.has(childDocIndex)) continue;
@@ -1008,14 +996,31 @@ function parseChildFrameDocuments(
    if (!childDoc) continue;
    const iframeBackendId = parentBackendIds[nodeArrayIdx];
    if (iframeBackendId === undefined) continue;
-    const iframeNode = parentNodeByBackendId.get(iframeBackendId);
-    const projection =
-      parentContext.projection && iframeNode?.localRect
-        ? childFrameProjection(parentContext.projection, iframeNode.localRect)
-        : null;
+    let projection: SnapshotProjection | null = null;
+    const source = { target: parentContext.target, frameId: snapshotFrameId(childDoc, strings) };
+    const parentProjection = parentContext.projection?.geometry;
+    if (parentProjection) {
+      // Each owner quad is target-relative; do not apply the parent's transform twice.
+      const edge = parentProjection.edges[0];
+      const clips = edge
+        ? [edge.destinationQuad, ...(edge.destinationClips ?? [])]
+        : parentProjection.sourceClips;
+      try {
+        projection = await geometry.snapshotProjection(
+          source,
+          iframeBackendId,
+          clips,
+          parentProjection.topViewport,
+        );
+      } catch (error) {
+        if (isAbortError(error)) throw error;
+        console.debug("[bsk capture] frame owner geometry unavailable", error);
+      }
+    }

    const childContext: FrameContext = {
-      frameId: snapshotFrameId(childDoc, strings),
+      frameId: source.frameId,
+      target: source.target,
      ownerFrameBackendNodeId: iframeBackendId,
      projection,
      scrollX: childDoc.scrollOffsetX ?? 0,
@@ -1023,7 +1028,7 @@ function parseChildFrameDocuments(
    };
    const nextVisited = new Set(visited);
    nextVisited.add(childDocIndex);
-    const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext);
+    const parsed = parseDocumentNodes(childDoc, strings, childContext);
    iframeNodes.set(iframeBackendId, parsed.nodes);
    if (childContext.frameId) {
      frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId);
@@ -1031,13 +1036,12 @@ function parseChildFrameDocuments(
    }
    for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);

-    const nested = parseChildFrameDocuments(
+    const nested = await parseChildFrameDocuments(
      documents,
      strings,
-      dpr,
      childDocIndex,
-      parsed.nodes,
      childContext,
+      geometry,
      nextVisited,
    );
    for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
@@ -1061,22 +1065,20 @@ export async function captureViewModel(
  options: CaptureViewModelOptions = {},
 ): Promise<CapturedViewModel> {
  throwIfAborted(options.signal);
-  let metrics: LayoutMetricsReply = {};
+  const target = options.target ?? { tabId };
+  const geometry = options.geometry ?? new GeometryContext(cdp, tabId, undefined, options.signal);
+  let metrics: LayoutMetrics = {};
  try {
-    metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
+    metrics = await geometry.layoutMetrics(target);
  } catch (error) {
    if (isAbortError(error)) throw error;
    console.debug("[bsk capture] layout metrics unavailable", error);
  }
  throwIfAborted(options.signal);
-  const dpr = devicePixelRatio(metrics);
-  const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
-  const viewport: Viewport = {
-    width: vpSrc.clientWidth ?? 0,
-    height: vpSrc.clientHeight ?? 0,
-  };
-  const scrollX = vpSrc.pageX ?? 0;
-  const scrollY = vpSrc.pageY ?? 0;
+  const measured = cssViewport(metrics);
+  const viewport: Viewport = { width: measured.width, height: measured.height };
+  const scrollX = measured.scrollX;
+  const scrollY = measured.scrollY;

  await cdp.send(tabId, "DOMSnapshot.enable", {});
  throwIfAborted(options.signal);
@@ -1102,19 +1104,19 @@ export async function captureViewModel(
  const topContext: FrameContext = {
    frameId: snapshotFrameId(doc0, strings),
    ownerFrameBackendNodeId: null,
+    target,
    projection: {
-      sourceClips: [],
-      edges: [],
-      topViewport: viewport,
+      source: { target, frameId: snapshotFrameId(doc0, strings) },
+      geometry: { sourceClips: [], edges: [], topViewport: viewport },
    },
    scrollX,
    scrollY,
  };
-  const mainParsed = parseDocumentNodes(doc0, strings, dpr, topContext);
+  const mainParsed = parseDocumentNodes(doc0, strings, topContext);
  const nodes = mainParsed.nodes;
  const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);

-  const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext);
+  const frameParsed = await parseChildFrameDocuments(documents, strings, 0, topContext, geometry);
  const iframeNodes = frameParsed.iframeNodes;
  for (const id of frameParsed.excludedBackendNodeIds) {
    excludedBackendNodeIds.add(id);
@@ -1,6 +1,6 @@
 import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
-import { resolveFrameProjection } from "../frame-geometry";
 import { type GeometryProjection, projectRectToViewport } from "../geometry";
+import { GeometryContext } from "../geometry/frame-context";
 import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
 import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
 import {
@@ -21,16 +21,6 @@ function isAbortError(err: unknown): boolean {
  return err instanceof Error && err.name === "AbortError";
 }

-async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
-  if (!cdp.getFrameGraph) return null;
-  try {
-    return await cdp.getFrameGraph(tabId);
-  } catch (err) {
-    console.debug("[bsk observation] frame graph capture failed", err);
-    return null;
-  }
-}
-
 function transformFrameNodes(
  nodes: CapturedNode[],
  projection: GeometryProjection | null,
@@ -51,6 +41,7 @@ async function captureMissingFrameDocuments(
  tabId: number,
  graph: CdpFrameGraph,
  captured: CapturedViewModel,
+  geometry: GeometryContext,
  signal?: AbortSignal,
 ): Promise<void> {
  if (!captured.frameNodes) captured.frameNodes = new Map();
@@ -62,10 +53,12 @@ async function captureMissingFrameDocuments(
    try {
      const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
        signal,
+        geometry,
+        target: frame.target,
      });
      let projection: GeometryProjection | null = null;
      try {
-        projection = await resolveFrameProjection(cdp, graph, frame.frameId);
+        projection = await geometry.targetProjection(frame.frameId);
      } catch (err) {
        if (isAbortError(err)) throw err;
        console.debug("[bsk observation] child frame geometry projection failed", {
@@ -171,10 +164,11 @@ export async function captureFrameData<T extends FrameAxNode>(
  tabId: number,
  captured: CapturedViewModel,
  signal?: AbortSignal,
+  geometry = new GeometryContext(cdp, tabId, undefined, signal),
 ): Promise<CapturedFrameDocument<T>[]> {
-  const graph = await discoverFrameGraph(cdp, tabId);
+  const graph = await geometry.graph();
  const frames = graph?.frames ?? [];
-  if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal);
+  if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, geometry, signal);
  throwIfAborted(signal);
  const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
  return buildFrameDocuments(graph, batches, captured);