fix(vom): iframe oopif geometry

apps/extension/src/tools/__tests__/element-geometry.test.ts

@@ -1,25 +1,28 @@
 import { describe, expect, it } from "vitest";
-import { nodeBoundingRect, quadBoundingRect } from "../element-geometry";
+import { nodeContentRegion } from "../element-geometry";
 import type { CdpRunner } from "../shared";
 
-describe("quadBoundingRect", () => {
-  it("computes axis-aligned bounds from an 8-double quad", () => {
-    expect(quadBoundingRect([10, 20, 110, 20, 110, 60, 10, 60])).toEqual({
-      x: 10,
-      y: 20,
-      width: 100,
-      height: 40,
-    });
+describe("nodeContentRegion", () => {
+  it("rejects invalid content quads before using the box model fallback", async () => {
+    const send = async (_tabId: number, method: string) => {
+      if (method === "DOM.getContentQuads") return { quads: [[0, 0, 0, 0, 0, 0, 0, 0]] };
+      if (method === "DOM.getBoxModel") {
+        return { model: { content: [5, 10, 25, 10, 25, 30, 5, 30] } };
+      }
+      throw new Error(`unexpected CDP call ${method}`);
+    };
+
+    await expect(nodeContentRegion({ send: send as CdpRunner["send"] }, 7, 555)).resolves.toEqual([
+      [
+        { x: 5, y: 10 },
+        { x: 25, y: 10 },
+        { x: 25, y: 30 },
+        { x: 5, y: 30 },
+      ],
+    ]);
   });
 
-  it("returns null for degenerate quads", () => {
-    expect(quadBoundingRect([0, 0, 0, 0, 0, 0, 0, 0])).toBeNull();
-    expect(quadBoundingRect([1, 2, 3])).toBeNull();
-  });
-});
-
-describe("nodeBoundingRect", () => {
-  it("unions every visible content fragment", async () => {
+  it("preserves every visible content fragment", async () => {
     const send = async () => ({
       quads: [
         [10, 20, 60, 20, 60, 40, 10, 40],
@@ -30,12 +33,20 @@ describe("nodeBoundingRect", () => {
       send: send as CdpRunner["send"],
     };
 
-    await expect(nodeBoundingRect(cdp, 7, 555)).resolves.toEqual({
-      x: 10,
-      y: 20,
-      width: 80,
-      height: 60,
-    });
+    await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([
+      [
+        { x: 10, y: 20 },
+        { x: 60, y: 20 },
+        { x: 60, y: 40 },
+        { x: 10, y: 40 },
+      ],
+      [
+        { x: 15, y: 50 },
+        { x: 90, y: 50 },
+        { x: 90, y: 80 },
+        { x: 15, y: 80 },
+      ],
+    ]);
   });
 
   it("falls back to visible descendant bounds for zero-size containers", async () => {
@@ -57,12 +68,14 @@ describe("nodeBoundingRect", () => {
     };
     const cdp = { send: send as CdpRunner["send"] };
 
-    await expect(nodeBoundingRect(cdp, 7, 555)).resolves.toEqual({
-      x: 242,
-      y: 468,
-      width: 769,
-      height: 180,
-    });
+    await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([
+      [
+        { x: 242, y: 468 },
+        { x: 1011, y: 468 },
+        { x: 1011, y: 648 },
+        { x: 242, y: 648 },
+      ],
+    ]);
     expect(calls.map((c) => c.method)).toEqual([
       "DOM.getContentQuads",
       "DOM.getBoxModel",

apps/extension/src/tools/__tests__/frame-geometry.test.ts

@@ -1,15 +1,27 @@
 import { describe, expect, it, vi } from "vitest";
-import { type Quad, resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry";
+import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry";
 import {
   clipPolygon,
+  polygonArea,
   polygonBounds,
   projectAndClipRegion,
   projectUnitPoint,
+  type Quad,
   rectPolygon,
+  regionBounds,
 } from "../geometry";
 import type { CdpRunner } from "../shared";
 
 describe("frame geometry projection", () => {
+  it("keeps region bounds separate from polygon area", () => {
+    const region = [
+      rectPolygon({ x: 0, y: 0, w: 10, h: 10 }),
+      rectPolygon({ x: 20, y: 5, w: 5, h: 5 }),
+    ];
+    expect(regionBounds(region)).toEqual({ x: 0, y: 0, width: 25, height: 10 });
+    expect(polygonArea(region[0])).toBe(100);
+  });
+
   it("maps through the iframe content box instead of its border box", () => {
     const contentQuad: Quad = [
       { x: 204, y: 306 },

apps/extension/src/tools/element-geometry.ts

@@ -1,9 +1,15 @@
-// Shared CDP helpers for scrolling nodes into view and computing
-// viewport-space bounding rectangles. Used by interaction tools (click
-// centre) and observation tools (element screenshot clip).
+// Resolve and scroll nodes inside one CDP target. Cross-frame projection
+// belongs to frame-geometry.ts.
 
 import type { RpcError } from "@/transport/types";
 import { rpcError } from "./errors";
+import {
+  parseCdpQuad,
+  polygonBounds,
+  type Region,
+  rectPolygon,
+  type ViewportRect,
+} from "./geometry";
 import { type CdpRunner, isRpcError } from "./shared";
 
 const ELEMENT_NOT_VISIBLE_MESSAGE =
@@ -13,45 +19,6 @@ function elementNotVisibleError(): RpcError {
   return rpcError("permission_denied", "element_not_visible", ELEMENT_NOT_VISIBLE_MESSAGE);
 }
 
-/** Viewport-space axis-aligned bounding box for CDP `Page.captureScreenshot` clip. */
-export interface ViewportRect {
-  x: number;
-  y: number;
-  width: number;
-  height: number;
-}
-
-/**
- * Compute the axis-aligned bounding box of an 8-double CDP content
- * quad / box-model polygon. Exported for unit tests.
- */
-export function quadBoundingRect(quad: number[]): ViewportRect | null {
-  if (quad.length !== 8) return null;
-  const xs = [quad[0], quad[2], quad[4], quad[6]];
-  const ys = [quad[1], quad[3], quad[5], quad[7]];
-  const minX = Math.min(...xs);
-  const maxX = Math.max(...xs);
-  const minY = Math.min(...ys);
-  const maxY = Math.max(...ys);
-  const width = maxX - minX;
-  const height = maxY - minY;
-  if (width <= 0 || height <= 0) return null;
-  return { x: minX, y: minY, width, height };
-}
-
-function rectToQuad(rect: ViewportRect): number[] {
-  return [
-    rect.x,
-    rect.y,
-    rect.x + rect.width,
-    rect.y,
-    rect.x + rect.width,
-    rect.y + rect.height,
-    rect.x,
-    rect.y + rect.height,
-  ];
-}
-
 /**
  * Resolve `backendNodeId` → CDP `objectId` so we can invoke
  * `Runtime.callFunctionOn` against the live JS object.
@@ -112,52 +79,24 @@ export async function scrollNodeIntoView(
   }
 }
 
-/**
- * Viewport-space bounding box for `backendNodeId`. Prefers
- * `DOM.getContentQuads`, then `DOM.getBoxModel`, then the union of
- * visible descendant client rects for zero-size overflow containers.
- */
-export async function nodeBoundingRect(
+function validCdpRegion(quads: number[][] | undefined): Region {
+  return (quads ?? []).flatMap((raw) => {
+    const polygon = parseCdpQuad(raw);
+    return polygon && polygonBounds(polygon) ? [polygon] : [];
+  });
+}
+
+async function resolveVisibleContentRegion(
   cdp: CdpRunner,
   tabId: number,
   backendNodeId: number,
-): Promise<ViewportRect | RpcError> {
-  const polygons = await visibleContentPolygons(cdp, tabId, backendNodeId);
-  if (isRpcError(polygons)) return polygons;
-  const bounds = polygons.flatMap((polygon) => quadBoundingRect(polygon) ?? []);
-  const x = Math.min(...bounds.map((rect) => rect.x));
-  const y = Math.min(...bounds.map((rect) => rect.y));
-  const right = Math.max(...bounds.map((rect) => rect.x + rect.width));
-  const bottom = Math.max(...bounds.map((rect) => rect.y + rect.height));
-  return { x, y, width: right - x, height: bottom - y };
-}
-
-/**
- * Compute the centroid of a CDP content quad. Quads are reported as
- * 8 doubles in clockwise order: (x1, y1, x2, y2, x3, y3, x4, y4).
- */
-export function quadCentre(quad: number[]): { x: number; y: number } {
-  const x = (quad[0] + quad[4]) / 2;
-  const y = (quad[1] + quad[5]) / 2;
-  return { x, y };
-}
-
-/** Centre of a `DOM.getBoxModel` `content` polygon (also 8 doubles). */
-export function boxCentre(box: number[]): { x: number; y: number } {
-  return quadCentre(box);
-}
-
-async function visibleContentPolygons(
-  cdp: CdpRunner,
-  tabId: number,
-  backendNodeId: number,
-): Promise<number[][] | RpcError> {
+): Promise<Region | RpcError> {
   try {
     const quads = await cdp.send<{ quads?: number[][] }>(tabId, "DOM.getContentQuads", {
       backendNodeId,
     });
-    const visible = quads.quads?.filter((quad) => quad.length === 8 && quadBoundingRect(quad));
-    if (visible && visible.length > 0) return visible;
+    const visible = validCdpRegion(quads.quads);
+    if (visible.length > 0) return visible;
   } catch (err) {
     console.debug("[bsk element-geometry] getContentQuads failed", err);
   }
@@ -166,25 +105,32 @@ async function visibleContentPolygons(
       backendNodeId,
     });
     const content = box.model?.content;
-    if (content && content.length === 8 && quadBoundingRect(content)) return [content];
+    const fallback = validCdpRegion(content ? [content] : undefined);
+    if (fallback.length > 0) return fallback;
   } catch (err) {
     console.debug("[bsk element-geometry] getBoxModel failed", err);
   }
   const descendantRect = await descendantBoundingRect(cdp, tabId, backendNodeId);
-  if (descendantRect) return [rectToQuad(descendantRect)];
+  if (descendantRect) {
+    return [
+      rectPolygon({
+        x: descendantRect.x,
+        y: descendantRect.y,
+        w: descendantRect.width,
+        h: descendantRect.height,
+      }),
+    ];
+  }
   return elementNotVisibleError();
 }
 
-/**
- * Return a live content quad for a backend node in the target's own viewport.
- * Frame-aware callers can project this quad through the owning frame chain.
- */
-export async function nodeContentQuads(
+/** Resolve every visible content region in the node's target-local viewport. */
+export async function nodeContentRegion(
   cdp: CdpRunner,
   tabId: number,
   backendNodeId: number,
-): Promise<number[][] | RpcError> {
-  return visibleContentPolygons(cdp, tabId, backendNodeId);
+): Promise<Region | RpcError> {
+  return resolveVisibleContentRegion(cdp, tabId, backendNodeId);
 }
 
 async function descendantBoundingRect(
@@ -257,20 +203,3 @@ function parseViewportRect(value: unknown): ViewportRect | null {
   if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) return null;
   return { x, y, width, height };
 }
-
-/**
- * Get the click point for `backendNodeId`. Prefers
- * `DOM.getContentQuads` (richer for rotated / transformed elements);
- * falls back to `DOM.getBoxModel`, then visible descendant bounds.
- * Returns `permission_denied` with "element not visible" when every
- * geometry path comes back empty / fails.
- */
-export async function nodeCentre(
-  cdp: CdpRunner,
-  tabId: number,
-  backendNodeId: number,
-): Promise<{ x: number; y: number } | RpcError> {
-  const polygons = await visibleContentPolygons(cdp, tabId, backendNodeId);
-  if (isRpcError(polygons)) return polygons;
-  return quadCentre(polygons[0]);
-}

apps/extension/src/tools/frame-geometry.ts

@@ -1,6 +1,6 @@
 import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph";
 import type { RpcError } from "@/transport/types";
-import { nodeContentQuads, scrollNodeIntoView, type ViewportRect } from "./element-geometry";
+import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry";
 import {
   clipPolygon,
   type GeometryProjection,
@@ -8,21 +8,18 @@ import {
   type Polygon,
   type ProjectiveEdge,
   parseCdpQuad,
-  polygonBounds,
+  polygonArea,
   polygonCentroid,
   projectRegionToViewport,
   type Quad,
   type Region,
   rectPolygon,
+  regionBounds,
   type Size,
+  type ViewportRect,
 } from "./geometry";
 import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared";
 
-export type { Point, Quad, Region } from "./geometry";
-export { parseCdpQuad } from "./geometry";
-
-export type FrameProjection = GeometryProjection;
-
 export interface NodeAddress {
   target: CdpTarget;
   backendNodeId: number;
@@ -124,7 +121,7 @@ export async function resolveFrameProjection(
   cdp: CdpRunner,
   graph: CdpFrameGraph,
   frameId: string,
-): Promise<FrameProjection | null> {
+): Promise<GeometryProjection | null> {
   const byId = frameMap(graph);
   const frame = byId.get(frameId);
   if (!frame) return null;
@@ -154,7 +151,7 @@ export async function resolveFrameProjection(
 
   const topViewport =
     edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
-  return topViewport ? { sourceViewport, sourceClips, edges, topViewport } : null;
+  return topViewport ? { sourceClips, edges, topViewport } : null;
 }
 
 async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
@@ -215,9 +212,8 @@ export async function scrollElementAndFramesIntoView(
 function largestRegion(regions: Region): Polygon | null {
   let largest: { polygon: Polygon; area: number } | null = null;
   for (const polygon of regions) {
-    const bounds = polygonBounds(polygon);
-    if (!bounds) continue;
-    const area = bounds.width * bounds.height;
+    const area = polygonArea(polygon);
+    if (area <= 0) continue;
     if (!largest || area > largest.area) largest = { polygon, area };
   }
   return largest?.polygon ?? null;
@@ -241,16 +237,12 @@ export async function resolveNodeGeometry(
       if (scrollError) return scrollError;
     }
 
-    const local = await nodeContentQuads(
+    const localRegion = await nodeContentRegion(
       cdpRunnerForTarget(cdp, address.target),
       tabId,
       address.backendNodeId,
     );
-    if (isRpcError(local)) return local;
-    const localRegion = local
-      .map((quad) => parseCdpQuad(quad))
-      .filter((quad): quad is Quad => quad !== null);
-    if (localRegion.length === 0) return geometryError("DOM returned no valid content regions");
+    if (isRpcError(localRegion)) return localRegion;
 
     const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null;
     if (address.target.sessionId && !address.frameId) {
@@ -276,7 +268,7 @@ export async function resolveNodeGeometry(
         .filter((polygon) => polygon.length >= 3);
     }
 
-    const topBounds = polygonBounds(topVisibleRegions.flat());
+    const topBounds = regionBounds(topVisibleRegions);
     const actionRegion = largestRegion(topVisibleRegions);
     const actionPoint = actionRegion ? polygonCentroid(actionRegion) : null;
     if (!topBounds || !actionPoint) {

apps/extension/src/tools/geometry.ts

@@ -1,5 +1,3 @@
-import type { ViewportRect } from "./element-geometry";
-
 export interface Point {
   x: number;
   y: number;
@@ -14,6 +12,13 @@ export interface Size {
   height: number;
 }
 
+export interface ViewportRect {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+}
+
 export interface ProjectiveEdge {
   sourceViewport: Size;
   destinationQuad: Quad;
@@ -21,7 +26,6 @@ export interface ProjectiveEdge {
 }
 
 export interface GeometryProjection {
-  sourceViewport: Size;
   sourceClips: Polygon[];
   edges: ProjectiveEdge[];
   topViewport: Size;
@@ -175,6 +179,14 @@ export function polygonBounds(points: Polygon): ViewportRect | null {
   return { x, y, width: right - x, height: bottom - y };
 }
 
+export function regionBounds(region: Region): ViewportRect | null {
+  return polygonBounds(region.flat());
+}
+
+export function polygonArea(points: Polygon): number {
+  return Math.abs(polygonSignedArea(points));
+}
+
 export function polygonCentroid(points: Polygon): Point | null {
   if (points.length < 3) return null;
   let twiceArea = 0;
@@ -231,7 +243,7 @@ export function projectRectToViewport(
   projection: GeometryProjection,
 ): ViewportRect | null {
   if (!rect) return null;
-  return polygonBounds(projectRegionToViewport([rectPolygon(rect)], projection).flat());
+  return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection));
 }
 
 export function childFrameProjection(
@@ -240,7 +252,6 @@ export function childFrameProjection(
 ): GeometryProjection {
   const destinationQuad = rectPolygon(ownerRectInParent);
   return {
-    sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
     sourceClips: [],
     edges: [
       {

apps/extension/src/tools/interaction.ts

@@ -29,7 +29,7 @@ import type {
   SelectResult,
 } from "@/transport/types";
 import { attachDialogs, markDialogCursor } from "./dialogs";
-import { backendNodeToObject, boxCentre, nodeCentre, quadCentre } from "./element-geometry";
+import { backendNodeToObject } from "./element-geometry";
 import { rpcError } from "./errors";
 import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
 import {
@@ -1107,9 +1107,6 @@ export async function handleSelect(
 
 export const __testing__ = {
   DEFAULT_TIMEOUT_MS,
-  quadCentre,
-  boxCentre,
   resolveBackendNode,
-  nodeCentre,
   isFillable,
 };

apps/extension/src/tools/vom/capture.ts

@@ -1057,7 +1057,6 @@ export async function captureViewModel(
     frameId: snapshotFrameId(doc0, strings),
     ownerFrameBackendNodeId: null,
     projection: {
-      sourceViewport: viewport,
       sourceClips: [],
       edges: [],
       topViewport: viewport,

apps/extension/src/tools/vom/frame-capture.ts

@@ -1,6 +1,6 @@
 import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
-import { type FrameProjection, resolveFrameProjection } from "../frame-geometry";
-import { projectRectToViewport } from "../geometry";
+import { resolveFrameProjection } from "../frame-geometry";
+import { type GeometryProjection, projectRectToViewport } from "../geometry";
 import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
 import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
 import {
@@ -31,7 +31,10 @@ async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFra
   }
 }
 
-function transformFrameNodes(nodes: CapturedNode[], projection: FrameProjection): CapturedNode[] {
+function transformFrameNodes(
+  nodes: CapturedNode[],
+  projection: GeometryProjection,
+): CapturedNode[] {
   return nodes.map((node) => ({
     ...node,
     rect: (() => {