fix(vom): add new texts and fix bugs

apps/extension/src/session-manager/__tests__/ref-store.test.ts

@@ -77,4 +77,14 @@ describe("RefStore", () => {
     expect(s.resolveEntry("e1")?.capabilities).toEqual(["screenshot"]);
     expect(s.resolveEntry("e2")?.capabilities).toEqual(["screenshot"]);
   });
+
+  it("copies caller-owned capability arrays", () => {
+    const s = new RefStore();
+    const capabilities: Array<"interact" | "screenshot"> = ["screenshot"];
+    s.set("e1", 42, { tabId: 7, capabilities });
+
+    capabilities.push("interact");
+
+    expect(s.resolveEntry("e1")?.capabilities).toEqual(["screenshot"]);
+  });
 });

apps/extension/src/session-manager/ref-store.ts

@@ -45,7 +45,7 @@ function capabilitiesFor(
   capabilities: RefCapability[] | undefined,
 ): RefCapability[] {
   if (kind === "surface") return ["screenshot"];
-  return capabilities ?? ["interact", "screenshot"];
+  return capabilities ? [...capabilities] : ["interact", "screenshot"];
 }
 
 export class RefStore {

apps/extension/src/tools/__tests__/frame-geometry.test.ts

@@ -1,5 +1,9 @@
 import { describe, expect, it, vi } from "vitest";
-import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry";
+import {
+  createFrameGeometryContext,
+  resolveFrameProjection,
+  resolveNodeGeometry,
+} from "../frame-geometry";
 import {
   clipPolygon,
   polygonArea,
@@ -37,6 +41,53 @@ describe("frame geometry projection", () => {
     expect(polygonBounds(projected.flat())).toEqual({ x: 224, y: 346, width: 200, height: 80 });
   });
 
+  it("projects nested same-target frame-local bounds through authoritative content quads", async () => {
+    const boxCalls: number[] = [];
+    const cdp: CdpRunner = {
+      send: vi.fn(async (_tabId, method, params) => {
+        if (method === "Page.getLayoutMetrics") {
+          return { cssLayoutViewport: { clientWidth: 1_000, clientHeight: 800 } };
+        }
+        if (method === "Page.createIsolatedWorld") return { executionContextId: 91 };
+        if (method === "Runtime.evaluate") {
+          return { result: { value: { width: 100, height: 80 } } };
+        }
+        if (method === "DOM.getBoxModel") {
+          const backendNodeId = (params as { backendNodeId?: number }).backendNodeId ?? 0;
+          boxCalls.push(backendNodeId);
+          return backendNodeId === 10
+            ? { model: { content: [30, 100.5, 330, 100.5, 330, 300.5, 30, 300.5] } }
+            : { model: { content: [45, 120.5, 145, 120.5, 145, 200.5, 45, 200.5] } };
+        }
+        throw new Error(`unexpected ${method}`);
+      }) as CdpRunner["send"],
+    };
+    const geometry = createFrameGeometryContext(cdp, {
+      rootFrameId: "main",
+      frames: [
+        { frameId: "main", target: { tabId: 4 } },
+        {
+          frameId: "child",
+          parentFrameId: "main",
+          ownerBackendNodeId: 10,
+          target: { tabId: 4 },
+        },
+        {
+          frameId: "nested",
+          parentFrameId: "child",
+          ownerBackendNodeId: 20,
+          target: { tabId: 4 },
+        },
+      ],
+    });
+
+    await expect(
+      geometry.projectFrameLocalRect("nested", { x: 12, y: 12, w: 40, h: 20 }),
+    ).resolves.toEqual({ x: 57, y: 132.5, width: 40, height: 20 });
+    await geometry.projectFrameLocalRect("nested", { x: 1, y: 1, w: 2, h: 2 });
+    expect(boxCalls.sort()).toEqual([10, 20]);
+  });
+
   it("uses a projective mapping for perspective-transformed iframe quads", () => {
     const quad: Quad = [
       { x: 0, y: 0 },

apps/extension/src/tools/__tests__/observation.test.ts

@@ -330,6 +330,7 @@ describe("handleScreenshot", () => {
       tabId: 7,
       kind: "surface",
       capabilities: ["screenshot"],
+      visibleRect: { x: 0, y: 0, w: 4096, h: 4096 },
     });
     const { cdp, sent } = makeFakeCdp({
       "Page.getLayoutMetrics": () => ({
@@ -359,6 +360,86 @@ describe("handleScreenshot", () => {
     expect(clip.clip?.scale).toBeCloseTo(Math.sqrt(4_000_000 / (4096 * 4096)));
   });
 
+  it("retries once when actual surface PNG dimensions exceed the pixel budget", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    ctx.refStore.set("e5", 999, {
+      tabId: 7,
+      kind: "surface",
+      visibleRect: { x: 0, y: 0, w: 1_000, h: 1_000 },
+    });
+    let captures = 0;
+    const { cdp, sent } = makeFakeCdp({
+      "Runtime.evaluate": () => ({ result: { value: 1 } }),
+      "Page.getLayoutMetrics": () => ({
+        cssLayoutViewport: { clientWidth: 1_000, clientHeight: 1_000 },
+      }),
+      "DOM.getContentQuads": () => ({
+        quads: [[0, 0, 1_000, 0, 1_000, 1_000, 0, 1_000]],
+      }),
+      "Page.captureScreenshot": () => ({
+        data:
+          ++captures === 1
+            ? pngWithIhdrDimensions(4_000, 4_000)
+            : pngWithIhdrDimensions(1_998, 1_998),
+      }),
+    });
+
+    const res = await handleScreenshot(
+      sm,
+      { session_id: "aa11", ref: "@e5", tab_id: 7 },
+      makeScreenshotDeps({
+        cdp,
+        get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
+        query: vi.fn(),
+        captureVisibleTab: vi.fn(),
+      }),
+    );
+
+    if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
+    expect(res).toMatchObject({ width: 1_998, height: 1_998 });
+    const calls = sent.filter((call) => call.method === "Page.captureScreenshot");
+    expect(calls).toHaveLength(2);
+    expect((calls[1]?.params as { clip?: { scale?: number } }).clip?.scale).toBeLessThan(0.5);
+  });
+
+  it("fails closed when the retried surface PNG still exceeds the pixel budget", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    ctx.refStore.set("e5", 999, {
+      tabId: 7,
+      kind: "surface",
+      visibleRect: { x: 0, y: 0, w: 1_000, h: 1_000 },
+    });
+    const { cdp, sent } = makeFakeCdp({
+      "Runtime.evaluate": () => ({ result: { value: 1 } }),
+      "Page.getLayoutMetrics": () => ({
+        cssLayoutViewport: { clientWidth: 1_000, clientHeight: 1_000 },
+      }),
+      "DOM.getContentQuads": () => ({
+        quads: [[0, 0, 1_000, 0, 1_000, 1_000, 0, 1_000]],
+      }),
+      "Page.captureScreenshot": () => ({ data: pngWithIhdrDimensions(4_000, 4_000) }),
+    });
+
+    const res = await handleScreenshot(
+      sm,
+      { session_id: "aa11", ref: "@e5", tab_id: 7 },
+      makeScreenshotDeps({
+        cdp,
+        get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
+        query: vi.fn(),
+        captureVisibleTab: vi.fn(),
+      }),
+    );
+
+    expect(res).toEqual({
+      code: "cdp_failed",
+      message: "Page.captureScreenshot exceeded the visual surface pixel budget",
+    });
+    expect(sent.filter((call) => call.method === "Page.captureScreenshot")).toHaveLength(2);
+  });
+
   it("crops a visual surface screenshot to its observation-time visible region", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     const ctx = await sm.start("aa11");
@@ -462,7 +543,11 @@ describe("handleScreenshot", () => {
   it("fails closed when a ref capture is not a valid PNG", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     const ctx = await sm.start("aa11");
-    ctx.refStore.set("e5", 999, { tabId: 7, kind: "surface" });
+    ctx.refStore.set("e5", 999, {
+      tabId: 7,
+      kind: "surface",
+      visibleRect: { x: 0, y: 0, w: 50, h: 50 },
+    });
     const { cdp } = makeFakeCdp({
       "DOM.getContentQuads": () => ({ quads: [[0, 0, 50, 0, 50, 50, 0, 50]] }),
       "Page.captureScreenshot": () => ({ data: "bm90LWEtcG5n" }),
@@ -3094,6 +3179,150 @@ describe("handleSnapshot", () => {
     expect(snapshot.ref_count).toBe(0);
   });
 
+  it("preserves an overflow-clipped surface region through observation and screenshot", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    const root: CdpAxNode = {
+      nodeId: "1",
+      role: { type: "role", value: "RootWebArea" },
+      name: { type: "computedString", value: "Clipped canvas" },
+      backendDOMNodeId: 100,
+    };
+    const strings = [
+      "body",
+      "div",
+      "canvas",
+      "aria-label",
+      "clipped canvas",
+      "static",
+      "auto",
+      "visible",
+      "hidden",
+      "1",
+    ];
+    const i = (value: string) => strings.indexOf(value);
+    const send = vi.fn(async (_tabId: number, method: string, _params?: object) => {
+      if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
+      if (method === "Accessibility.getFullAXTree") return { nodes: [root] };
+      if (method === "Page.getLayoutMetrics") {
+        return {
+          cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
+          layoutViewport: { clientWidth: 1000, clientHeight: 800 },
+        };
+      }
+      if (method === "DOMSnapshot.captureSnapshot") {
+        return {
+          strings,
+          documents: [
+            {
+              nodes: {
+                parentIndex: [-1, 0, 1],
+                nodeName: [i("body"), i("div"), i("canvas")],
+                backendNodeId: [100, 150, 200],
+                attributes: [[], [], [i("aria-label"), i("clipped canvas")]],
+              },
+              layout: {
+                nodeIndex: [0, 1, 2],
+                styles: [
+                  [
+                    i("static"),
+                    i("auto"),
+                    i("auto"),
+                    i("visible"),
+                    i("1"),
+                    i("visible"),
+                    i("visible"),
+                  ],
+                  [
+                    i("static"),
+                    i("auto"),
+                    i("auto"),
+                    i("visible"),
+                    i("1"),
+                    i("hidden"),
+                    i("hidden"),
+                  ],
+                  [
+                    i("static"),
+                    i("auto"),
+                    i("auto"),
+                    i("visible"),
+                    i("1"),
+                    i("visible"),
+                    i("visible"),
+                  ],
+                ],
+                bounds: [
+                  [0, 0, 1000, 800],
+                  [10, 20, 100, 70],
+                  [55, 30, 140, 60],
+                ],
+                paintOrders: [0, 1, 2],
+              },
+            },
+          ],
+        };
+      }
+      if (method === "DOM.getContentQuads") {
+        return { quads: [[55, 30, 195, 30, 195, 90, 55, 90]] };
+      }
+      if (method === "Runtime.evaluate") return { result: { value: 1 } };
+      if (method === "Page.captureScreenshot") {
+        return { data: pngWithIhdrDimensions(55, 60) };
+      }
+      throw new Error(`unexpected CDP method ${method}`);
+    });
+    const cdp = {
+      send: send as unknown as CdpRunner["send"],
+      trackSessionTab: vi.fn(),
+      getFrameGraph: vi.fn(async () => ({
+        rootFrameId: "root",
+        frames: [{ frameId: "root", target: { tabId: 4 } }],
+      })),
+    } as CdpRunner;
+    const deps = {
+      cdp,
+      tabsApi: {
+        get: vi.fn(
+          async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
+        ),
+        query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
+      },
+      conditionalSurfaceProbe: false,
+    };
+
+    const observed = await handleObserve(sm, { session_id: "aa11" }, deps);
+    if ("code" in observed) throw new Error(`unexpected error: ${JSON.stringify(observed)}`);
+    expect(observed.text).toContain(
+      '@e1 surface "clipped canvas" [bounds=55,30,55,60; rendering=canvas;',
+    );
+    expect(ctx.refStore.resolveEntry("e1")).toMatchObject({
+      backendNodeId: 200,
+      kind: "surface",
+      visibleRect: { x: 55, y: 30, w: 55, h: 60 },
+    });
+
+    const screenshot = await handleScreenshot(
+      sm,
+      { session_id: "aa11", ref: "@e1", tab_id: 4 },
+      makeScreenshotDeps({
+        cdp,
+        get: vi.fn(async () => ({ id: 4, windowId: 100, active: true }) as chrome.tabs.Tab),
+      }),
+    );
+    if ("code" in screenshot) {
+      throw new Error(`unexpected error: ${JSON.stringify(screenshot)}`);
+    }
+    const captureCall = send.mock.calls.find(([, method]) => method === "Page.captureScreenshot");
+    expect((captureCall?.[2] as { clip?: object }).clip).toMatchObject({
+      x: 55,
+      y: 30,
+      width: 55,
+      height: 60,
+    });
+    expect(screenshot).toMatchObject({ width: 55, height: 60 });
+  });
+
   it("allows passive snapshots of explicit user-window tabs", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     await sm.start("aa11");

apps/extension/src/tools/__tests__/visual-screenshot-plan.test.ts

@@ -0,0 +1,37 @@
+import { describe, expect, it } from "vitest";
+import {
+  correctedVisualScreenshotScale,
+  planVisualScreenshot,
+  screenshotFitsPixelBudget,
+} from "../visual-screenshot-plan";
+
+const budget = { maxEdge: 2_048, maxPixels: 4_000_000 };
+
+describe("visual screenshot pixel planning", () => {
+  it.each([1, 1.25, 1.5, 2, 3])("accounts for raster scale %s", (rasterScaleEstimate) => {
+    const plan = planVisualScreenshot({
+      cssRect: { width: 2_000, height: 1_000 },
+      rasterScaleEstimate,
+      budget,
+    });
+    const width = 2_000 * rasterScaleEstimate * plan.clipScale;
+    const height = 1_000 * rasterScaleEstimate * plan.clipScale;
+
+    expect(Math.max(width, height)).toBeLessThanOrEqual(budget.maxEdge);
+    expect(width * height).toBeLessThanOrEqual(budget.maxPixels);
+  });
+
+  it("corrects a scale from actual PNG dimensions", () => {
+    const corrected = correctedVisualScreenshotScale({
+      previousClipScale: 1,
+      actualWidth: 4_000,
+      actualHeight: 2_000,
+      budget,
+    });
+
+    expect(corrected).not.toBeNull();
+    expect(corrected as number).toBeLessThan(2_048 / 4_000);
+    expect(screenshotFitsPixelBudget({ width: 2_000, height: 1_000 }, budget)).toBe(true);
+    expect(screenshotFitsPixelBudget({ width: 4_000, height: 2_000 }, budget)).toBe(false);
+  });
+});

apps/extension/src/tools/frame-geometry.ts

@@ -1,4 +1,9 @@
-import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph";
+import {
+  type CdpFrame,
+  type CdpFrameGraph,
+  type CdpTarget,
+  cdpTargetKey,
+} from "@/browser-driver/frame-graph";
 import type { RpcError } from "@/transport/types";
 import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry";
 import {
@@ -10,6 +15,7 @@ import {
   parseCdpQuad,
   polygonArea,
   polygonCentroid,
+  projectRectToViewport,
   projectRegionToViewport,
   type Quad,
   type Region,
@@ -84,6 +90,41 @@ async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise<Size |
   return width > 0 && height > 0 ? { width, height } : null;
 }
 
+async function sameTargetFrameViewport(cdp: CdpRunner, frame: CdpFrame): Promise<Size | null> {
+  const world = await sendToCdpTarget<{ executionContextId?: number }>(
+    cdp,
+    frame.target,
+    "Page.createIsolatedWorld",
+    {
+      frameId: frame.frameId,
+      worldName: "bsk-frame-geometry",
+      grantUniveralAccess: false,
+    },
+  );
+  if (world.executionContextId === undefined) return null;
+  const result = await sendToCdpTarget<{ result?: { value?: unknown } }>(
+    cdp,
+    frame.target,
+    "Runtime.evaluate",
+    {
+      expression: "({width:window.innerWidth,height:window.innerHeight})",
+      contextId: world.executionContextId,
+      returnByValue: true,
+    },
+  );
+  const value = result.result?.value as { width?: unknown; height?: unknown } | undefined;
+  const width = value?.width;
+  const height = value?.height;
+  return typeof width === "number" &&
+    Number.isFinite(width) &&
+    width > 0 &&
+    typeof height === "number" &&
+    Number.isFinite(height) &&
+    height > 0
+    ? { width, height }
+    : null;
+}
+
 async function ownerContentQuad(
   cdp: CdpRunner,
   parent: CdpFrame,
@@ -98,26 +139,150 @@ async function ownerContentQuad(
   return parseCdpQuad(result.model?.content);
 }
 
-async function sameTargetFrameClips(
+export interface FrameGeometryContext {
+  resolveFrameProjection(frameId: string): Promise<GeometryProjection | null>;
+  projectFrameLocalRect(
+    frameId: string,
+    rect: { x: number; y: number; w: number; h: number },
+  ): Promise<ViewportRect | null>;
+}
+
+export function createFrameGeometryContext(
   cdp: CdpRunner,
-  byId: Map<string, CdpFrame>,
-  frame: CdpFrame,
-  targetRoot: CdpFrame,
-): Promise<Polygon[] | null> {
-  const clips: Polygon[] = [];
-  let current = frame;
-  const seen = new Set<string>();
-  while (current.frameId !== targetRoot.frameId) {
-    if (seen.has(current.frameId) || !current.parentFrameId) return null;
-    seen.add(current.frameId);
-    const parent = byId.get(current.parentFrameId);
-    if (!parent || current.ownerBackendNodeId === undefined) return null;
-    const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId);
-    if (!clip) return null;
-    clips.push(clip);
-    current = parent;
-  }
-  return clips;
+  graph: CdpFrameGraph,
+): FrameGeometryContext {
+  const byId = frameMap(graph);
+  const targetViewports = new Map<string, Promise<Size | null>>();
+  const frameViewports = new Map<string, Promise<Size | null>>();
+  const ownerQuads = new Map<string, Promise<Quad | null>>();
+  const projections = new Map<string, Promise<GeometryProjection | null>>();
+  const localProjections = new Map<string, Promise<GeometryProjection | null>>();
+
+  const cachedTargetViewport = (target: CdpTarget): Promise<Size | null> => {
+    const key = cdpTargetKey(target);
+    let value = targetViewports.get(key);
+    if (!value) {
+      value = targetViewport(cdp, target);
+      targetViewports.set(key, value);
+    }
+    return value;
+  };
+  const cachedFrameViewport = (frame: CdpFrame): Promise<Size | null> => {
+    let value = frameViewports.get(frame.frameId);
+    if (!value) {
+      const root = targetRootFrame(byId, frame);
+      value =
+        root?.frameId === frame.frameId
+          ? cachedTargetViewport(frame.target)
+          : sameTargetFrameViewport(cdp, frame);
+      frameViewports.set(frame.frameId, value);
+    }
+    return value;
+  };
+  const cachedOwnerQuad = (parent: CdpFrame, backendNodeId: number): Promise<Quad | null> => {
+    const key = `${cdpTargetKey(parent.target)}:${backendNodeId}`;
+    let value = ownerQuads.get(key);
+    if (!value) {
+      value = ownerContentQuad(cdp, parent, backendNodeId);
+      ownerQuads.set(key, value);
+    }
+    return value;
+  };
+  const cachedSameTargetClips = async (
+    frame: CdpFrame,
+    root: CdpFrame,
+  ): Promise<Polygon[] | null> => {
+    const clips: Polygon[] = [];
+    let current = frame;
+    const seen = new Set<string>();
+    while (current.frameId !== root.frameId) {
+      if (seen.has(current.frameId) || !current.parentFrameId) return null;
+      seen.add(current.frameId);
+      const parent = byId.get(current.parentFrameId);
+      if (!parent || current.ownerBackendNodeId === undefined) return null;
+      const clip = await cachedOwnerQuad(parent, current.ownerBackendNodeId);
+      if (!clip) return null;
+      clips.push(clip);
+      current = parent;
+    }
+    return clips;
+  };
+
+  const buildTargetProjection = async (frameId: string): Promise<GeometryProjection | null> => {
+    const frame = byId.get(frameId);
+    if (!frame) return null;
+    let root = targetRootFrame(byId, frame);
+    if (!root) return null;
+    const sourceViewport = await cachedTargetViewport(root.target);
+    if (!sourceViewport) return null;
+    const sourceClips = await cachedSameTargetClips(frame, root);
+    if (!sourceClips) return null;
+    const edges: ProjectiveEdge[] = [];
+    while (root.parentFrameId) {
+      const parent = byId.get(root.parentFrameId);
+      if (!parent || root.ownerBackendNodeId === undefined) return null;
+      const destinationQuad = await cachedOwnerQuad(parent, root.ownerBackendNodeId);
+      if (!destinationQuad) return null;
+      const source = await cachedTargetViewport(root.target);
+      if (!source) return null;
+      const parentRoot = targetRootFrame(byId, parent);
+      if (!parentRoot) return null;
+      const destinationClips = await cachedSameTargetClips(parent, parentRoot);
+      if (!destinationClips) return null;
+      edges.push({ sourceViewport: source, destinationQuad, destinationClips });
+      root = parentRoot;
+    }
+    const topViewport = await cachedTargetViewport(root.target);
+    return topViewport ? { sourceClips, edges, topViewport } : null;
+  };
+
+  const resolveFrameProjectionCached = (frameId: string): Promise<GeometryProjection | null> => {
+    let value = projections.get(frameId);
+    if (!value) {
+      value = buildTargetProjection(frameId);
+      projections.set(frameId, value);
+    }
+    return value;
+  };
+
+  const buildLocalProjection = async (frameId: string): Promise<GeometryProjection | null> => {
+    const frame = byId.get(frameId);
+    if (!frame) return null;
+    const root = targetRootFrame(byId, frame);
+    if (!root) return null;
+    const targetProjection = await resolveFrameProjectionCached(root.frameId);
+    if (!targetProjection) return null;
+    if (root.frameId === frame.frameId) return targetProjection;
+    if (!frame.parentFrameId || frame.ownerBackendNodeId === undefined) return null;
+    const parent = byId.get(frame.parentFrameId);
+    if (!parent) return null;
+    const sourceViewport = await cachedFrameViewport(frame);
+    const destinationQuad = await cachedOwnerQuad(parent, frame.ownerBackendNodeId);
+    const destinationClips = await cachedSameTargetClips(frame, root);
+    if (!sourceViewport || !destinationQuad || !destinationClips) return null;
+    return {
+      sourceClips: [],
+      edges: [{ sourceViewport, destinationQuad, destinationClips }, ...targetProjection.edges],
+      topViewport: targetProjection.topViewport,
+    };
+  };
+
+  const localProjection = (frameId: string): Promise<GeometryProjection | null> => {
+    let value = localProjections.get(frameId);
+    if (!value) {
+      value = buildLocalProjection(frameId);
+      localProjections.set(frameId, value);
+    }
+    return value;
+  };
+
+  return {
+    resolveFrameProjection: resolveFrameProjectionCached,
+    async projectFrameLocalRect(frameId, rect) {
+      const projection = await localProjection(frameId);
+      return projection ? projectRectToViewport(rect, projection) : null;
+    },
+  };
 }
 
 export async function resolveFrameProjection(
@@ -125,36 +290,7 @@ export async function resolveFrameProjection(
   graph: CdpFrameGraph,
   frameId: string,
 ): Promise<GeometryProjection | null> {
-  const byId = frameMap(graph);
-  const frame = byId.get(frameId);
-  if (!frame) return null;
-  let targetRoot = targetRootFrame(byId, frame);
-  if (!targetRoot) return null;
-  const sourceViewport = await targetViewport(cdp, targetRoot.target);
-  if (!sourceViewport) return null;
-  const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot);
-  if (!sourceClips) return null;
-
-  const edges: ProjectiveEdge[] = [];
-  while (targetRoot.parentFrameId) {
-    const parent = byId.get(targetRoot.parentFrameId);
-    if (!parent || targetRoot.ownerBackendNodeId === undefined) return null;
-    const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId);
-    if (!destinationQuad) return null;
-    const source =
-      edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
-    if (!source) return null;
-    const parentTargetRoot = targetRootFrame(byId, parent);
-    if (!parentTargetRoot) return null;
-    const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot);
-    if (!destinationClips) return null;
-    edges.push({ sourceViewport: source, destinationQuad, destinationClips });
-    targetRoot = parentTargetRoot;
-  }
-
-  const topViewport =
-    edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
-  return topViewport ? { sourceClips, edges, topViewport } : null;
+  return createFrameGeometryContext(cdp, graph).resolveFrameProjection(frameId);
 }
 
 async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {

apps/extension/src/tools/geometry.ts

@@ -245,22 +245,3 @@ export function projectRectToViewport(
   if (!rect) return null;
   return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection));
 }
-
-export function childFrameProjection(
-  parent: GeometryProjection,
-  ownerRectInParent: { x: number; y: number; w: number; h: number },
-): GeometryProjection {
-  const destinationQuad = rectPolygon(ownerRectInParent);
-  return {
-    sourceClips: [],
-    edges: [
-      {
-        sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
-        destinationQuad,
-        destinationClips: parent.sourceClips,
-      },
-      ...parent.edges,
-    ],
-    topViewport: parent.topViewport,
-  };
-}

apps/extension/src/tools/observation.ts

@@ -48,6 +48,13 @@ import {
   type ToolEffect,
 } from "./shared";
 import { resolveSnapshotRef } from "./snapshot-ref";
+import { topLevelScreenshotRasterScale } from "./viewport-metrics";
+import {
+  correctedVisualScreenshotScale,
+  planVisualScreenshot,
+  type ScreenshotPixelBudget,
+  screenshotFitsPixelBudget,
+} from "./visual-screenshot-plan";
 import {
   type CapturedNode,
   type CapturedSurfaceProbe,
@@ -101,6 +108,10 @@ export const normaliseRef = sharedNormaliseRef;
 
 const VISUAL_SURFACE_SCREENSHOT_MAX_EDGE = 2_048;
 const VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS = 4_000_000;
+const VISUAL_SURFACE_SCREENSHOT_BUDGET: ScreenshotPixelBudget = {
+  maxEdge: VISUAL_SURFACE_SCREENSHOT_MAX_EDGE,
+  maxPixels: VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS,
+};
 
 /**
  * Strip the `data:image/...;base64,` prefix from a Chrome
@@ -225,36 +236,52 @@ async function captureElementScreenshot(
         })()
       : liveRect;
   if (!rect) return { code: "permission_denied", message: "surface is no longer visible" };
-  const scale = visualSurface
-    ? Math.min(
-        1,
-        VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.width,
-        VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.height,
-        Math.sqrt(VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS / (rect.width * rect.height)),
-      )
+  const rasterScaleEstimate = visualSurface ? await topLevelScreenshotRasterScale(cdp, tabId) : 1;
+  let scale = visualSurface
+    ? planVisualScreenshot({
+        cssRect: rect,
+        rasterScaleEstimate,
+        budget: VISUAL_SURFACE_SCREENSHOT_BUDGET,
+      }).clipScale
     : 1;
 
   try {
-    const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
-      format: "png",
-      clip: {
-        x: rect.x,
-        y: rect.y,
-        width: rect.width,
-        height: rect.height,
-        scale,
-      },
-    });
-    if (signal?.aborted) return cancelled("screenshot");
-    const image_base64 = shot.data ?? "";
-    if (!image_base64) {
-      return { code: "cdp_failed", message: "Page.captureScreenshot returned no data" };
+    for (let attempt = 0; attempt < (visualSurface ? 2 : 1); attempt += 1) {
+      const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
+        format: "png",
+        clip: {
+          x: rect.x,
+          y: rect.y,
+          width: rect.width,
+          height: rect.height,
+          scale,
+        },
+      });
+      if (signal?.aborted) return cancelled("screenshot");
+      const image_base64 = shot.data ?? "";
+      if (!image_base64) {
+        return { code: "cdp_failed", message: "Page.captureScreenshot returned no data" };
+      }
+      const dims = parsePngDimensions(image_base64);
+      if (!dims) {
+        return { code: "cdp_failed", message: "Page.captureScreenshot returned invalid PNG data" };
+      }
+      if (!visualSurface || screenshotFitsPixelBudget(dims, VISUAL_SURFACE_SCREENSHOT_BUDGET)) {
+        return { image_base64, width: dims.width, height: dims.height };
+      }
+      const corrected = correctedVisualScreenshotScale({
+        previousClipScale: scale,
+        actualWidth: dims.width,
+        actualHeight: dims.height,
+        budget: VISUAL_SURFACE_SCREENSHOT_BUDGET,
+      });
+      if (attempt === 1 || corrected === null) break;
+      scale = corrected;
     }
-    const dims = parsePngDimensions(image_base64);
-    if (!dims) {
-      return { code: "cdp_failed", message: "Page.captureScreenshot returned invalid PNG data" };
-    }
-    return { image_base64, width: dims.width, height: dims.height };
+    return {
+      code: "cdp_failed",
+      message: "Page.captureScreenshot exceeded the visual surface pixel budget",
+    };
   } catch (err) {
     return {
       code: "cdp_failed",
@@ -351,6 +378,12 @@ export async function handleScreenshot(
     }
     const node = resolveSnapshotRef(ctx, ref, target.tabId, "screenshot");
     if (isRpcError(node)) return node;
+    if (node.kind === "surface" && !node.visibleRect) {
+      return {
+        code: "permission_denied",
+        message: `surface ref ${ref} has no observation-time visible bounds`,
+      };
+    }
     if (signal?.aborted) return cancelled("screenshot");
     deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
     await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);

apps/extension/src/tools/viewport-metrics.ts

@@ -0,0 +1,18 @@
+import type { CdpRunner } from "./shared";
+
+/** Estimate the raster/CSS ratio used by top-level Page.captureScreenshot. */
+export async function topLevelScreenshotRasterScale(
+  cdp: CdpRunner,
+  tabId: number,
+): Promise<number> {
+  try {
+    const reply = await cdp.send<{ result?: { value?: unknown } }>(tabId, "Runtime.evaluate", {
+      expression: "window.devicePixelRatio",
+      returnByValue: true,
+    });
+    const value = reply.result?.value;
+    return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 1;
+  } catch {
+    return 1;
+  }
+}

apps/extension/src/tools/visual-screenshot-plan.ts

@@ -0,0 +1,52 @@
+export interface ScreenshotPixelBudget {
+  maxEdge: number;
+  maxPixels: number;
+}
+
+export interface VisualScreenshotPlan {
+  clipScale: number;
+}
+
+export function planVisualScreenshot(input: {
+  cssRect: { width: number; height: number };
+  rasterScaleEstimate: number;
+  budget: ScreenshotPixelBudget;
+}): VisualScreenshotPlan {
+  const rasterScale =
+    Number.isFinite(input.rasterScaleEstimate) && input.rasterScaleEstimate > 0
+      ? input.rasterScaleEstimate
+      : 1;
+  const { width, height } = input.cssRect;
+  const scale = Math.min(
+    1,
+    input.budget.maxEdge / (width * rasterScale),
+    input.budget.maxEdge / (height * rasterScale),
+    Math.sqrt(input.budget.maxPixels / (width * height * rasterScale * rasterScale)),
+  );
+  return { clipScale: Math.max(Number.EPSILON, scale) };
+}
+
+export function correctedVisualScreenshotScale(input: {
+  previousClipScale: number;
+  actualWidth: number;
+  actualHeight: number;
+  budget: ScreenshotPixelBudget;
+}): number | null {
+  const correction = Math.min(
+    input.budget.maxEdge / input.actualWidth,
+    input.budget.maxEdge / input.actualHeight,
+    Math.sqrt(input.budget.maxPixels / (input.actualWidth * input.actualHeight)),
+  );
+  if (!Number.isFinite(correction) || correction >= 1 || correction <= 0) return null;
+  return Math.max(Number.EPSILON, input.previousClipScale * correction * 0.999);
+}
+
+export function screenshotFitsPixelBudget(
+  dimensions: { width: number; height: number },
+  budget: ScreenshotPixelBudget,
+): boolean {
+  return (
+    Math.max(dimensions.width, dimensions.height) <= budget.maxEdge &&
+    dimensions.width * dimensions.height <= budget.maxPixels
+  );
+}

apps/extension/src/tools/vom/__tests__/capture.test.ts

@@ -793,8 +793,10 @@ describe("captureViewModel", () => {
     expect(div?.rect).toMatchObject({ y: 0, h: 600 });
   });
 
-  it("normalizes device-pixel bounds by devicePixelRatio", async () => {
-    const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
+  it.each([
+    1, 2,
+  ])("keeps CSS-space bounds stable at devicePixelRatio=%i", async (deviceScaleFactor) => {
+    const S = ["html", "body", "canvas", "position", "fixed", "static", "pointer-events", "auto"];
     const i = (s: string) => S.indexOf(s);
     const snapshot = {
       strings: S,
@@ -802,7 +804,7 @@ describe("captureViewModel", () => {
         {
           nodes: {
             parentIndex: [-1, 0, 1],
-            nodeName: [i("html"), i("body"), i("div")],
+            nodeName: [i("html"), i("body"), i("canvas")],
             backendNodeId: [10, 11, 12],
             attributes: [[], [], []],
           },
@@ -813,8 +815,8 @@ describe("captureViewModel", () => {
               [i("fixed"), i("auto")],
             ],
             bounds: [
-              [0, 0, 2000, 3200],
-              [0, 0, 2000, 1600],
+              [0, 0, 1000 * deviceScaleFactor, 1600 * deviceScaleFactor],
+              [0, 0, 1000 * deviceScaleFactor, 800 * deviceScaleFactor],
             ],
             paintOrders: [0, 50],
           },
@@ -828,14 +830,19 @@ describe("captureViewModel", () => {
         if (method === "Page.getLayoutMetrics") {
           return {
             cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
-            layoutViewport: { clientWidth: 2000, clientHeight: 1600 },
+            layoutViewport: {
+              clientWidth: 1000 * deviceScaleFactor,
+              clientHeight: 800 * deviceScaleFactor,
+            },
           };
         }
         throw new Error(method);
       }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
     };
     const { nodes } = await captureViewModel(cdp, 4);
-    expect(nodes.find((n) => n.backendNodeId === 12)?.rect).toEqual({
+    const canvas = nodes.find((node) => node.backendNodeId === 12);
+    expect(canvas?.tag).toBe("canvas");
+    expect(canvas?.rect).toEqual({
       x: 0,
       y: 0,
       w: 1000,
@@ -1011,7 +1018,7 @@ describe("captureViewModel", () => {
     expect(input?.attrs.type).toBe("text");
     expect(input?.ownerFrameBackendNodeId).toBe(13);
     expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
-    expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
+    expect(input?.rect).toBeNull();
     expect(rootFrameId).toBe("main-frame");
     expect(frameNodes?.get("child-frame")).toBe(subNodes);
     expect(frameOwnerBackendNodeIds?.get("child-frame")).toBe(13);
@@ -1183,13 +1190,13 @@ describe("captureViewModel", () => {
     const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31);
 
     expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 });
-    expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
+    expect(nestedIframe?.rect).toBeNull();
     expect(input?.ownerFrameBackendNodeId).toBe(23);
     expect(input?.localRect).toEqual({ x: 1, y: 2, w: 40, h: 20 });
-    expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
+    expect(input?.rect).toBeNull();
   });
 
-  it("keeps same-origin iframe canvas discovery in top-level CSS coordinates at dpr 1 and 2", async () => {
+  it("keeps same-origin iframe canvas frame-local until frame geometry is resolved", async () => {
     const captureAtDpr = async (dpr: number) => {
       const S = ["html", "body", "iframe", "canvas", "static", "auto"];
       const i = (s: string) => S.indexOf(s);
@@ -1270,13 +1277,7 @@ describe("captureViewModel", () => {
     const atDpr1 = await captureAtDpr(1);
     const atDpr2 = await captureAtDpr(2);
 
-    expect(atDpr1).toEqual([
-      expect.objectContaining({
-        backendNodeId: 21,
-        frameId: "child",
-        visibleRect: { x: 120, y: 180, w: 120, h: 60 },
-      }),
-    ]);
+    expect(atDpr1).toEqual([]);
     expect(atDpr2).toEqual(atDpr1);
   });
 

apps/extension/src/tools/vom/__tests__/frame-capture.test.ts

@@ -50,6 +50,105 @@ function childSnapshot(frameId: string, backendNodeId: number) {
 }
 
 describe("captureFrameData", () => {
+  it("does not request frame geometry for a root-only page", async () => {
+    const root = { ...ownerNode(1, 0), tag: "body" };
+    const captured: CapturedViewModel = {
+      nodes: [root],
+      viewport: { width: 1000, height: 800 },
+      iframeNodes: new Map(),
+      frameNodes: new Map([["main", [root]]]),
+      rootFrameId: "main",
+      excludedBackendNodeIds: new Set(),
+    };
+    const send = vi.fn(async (_tabId, method) => {
+      if (method === "Accessibility.enable") return {};
+      if (method === "Accessibility.getFullAXTree") return { nodes: [] };
+      throw new Error(`unexpected ${method}`);
+    });
+    const cdp: CdpRunner = {
+      send: send as CdpRunner["send"],
+      getFrameGraph: vi.fn(async () => ({
+        rootFrameId: "main",
+        frames: [{ frameId: "main", target: { tabId: 4 } }],
+      })),
+    };
+
+    await captureFrameData(cdp, 4, captured);
+
+    expect(send.mock.calls.some(([, method]) => method === "Page.createIsolatedWorld")).toBe(false);
+    expect(send.mock.calls.some(([, method]) => method === "DOM.getBoxModel")).toBe(false);
+    expect(send.mock.calls.some(([, method]) => method === "Page.getLayoutMetrics")).toBe(false);
+  });
+
+  it("projects same-origin frame-local rects through the iframe content box", async () => {
+    const owner = ownerNode(10, 50);
+    const canvas: CapturedNode = {
+      backendNodeId: 101,
+      parentBackendNodeId: 100,
+      frameId: "child",
+      ownerFrameBackendNodeId: 10,
+      tag: "canvas",
+      attrs: {},
+      rect: null,
+      localRect: { x: 12, y: 12, w: 224, h: 94 },
+      paintOrder: 1,
+      position: "static",
+      pointerEvents: "auto",
+    };
+    const captured: CapturedViewModel = {
+      nodes: [owner],
+      viewport: { width: 1000, height: 800 },
+      iframeNodes: new Map([[10, [canvas]]]),
+      frameNodes: new Map([
+        ["main", [owner]],
+        ["child", [canvas]],
+      ]),
+      frameOwnerBackendNodeIds: new Map([["child", 10]]),
+      frameParentIds: new Map([["child", "main"]]),
+      rootFrameId: "main",
+      excludedBackendNodeIds: new Set(),
+    };
+    const cdp: CdpRunner = {
+      send: vi.fn(async (_tabId, method) => {
+        if (method === "Accessibility.enable") return {};
+        if (method === "Accessibility.getFullAXTree") return { nodes: [] };
+        if (method === "Page.getLayoutMetrics") {
+          return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
+        }
+        if (method === "DOM.getBoxModel") {
+          return { model: { content: [52, 102, 352, 102, 352, 302, 52, 302] } };
+        }
+        if (method === "Page.createIsolatedWorld") return { executionContextId: 91 };
+        if (method === "Runtime.evaluate") {
+          return { result: { value: { width: 300, height: 200 } } };
+        }
+        throw new Error(`unexpected ${method}`);
+      }) as CdpRunner["send"],
+      getFrameGraph: vi.fn(async () => ({
+        rootFrameId: "main",
+        frames: [
+          { frameId: "main", target: { tabId: 4 } },
+          {
+            frameId: "child",
+            parentFrameId: "main",
+            ownerBackendNodeId: 10,
+            target: { tabId: 4 },
+          },
+        ],
+      })),
+    };
+
+    await captureFrameData(cdp, 4, captured);
+
+    expect(captured.frameNodes?.get("child")?.[0]?.rect).toEqual({
+      x: 64,
+      y: 114,
+      w: 224,
+      h: 94,
+    });
+    expect(captured.iframeNodes.get(10)?.[0]?.rect).toEqual({ x: 64, y: 114, w: 224, h: 94 });
+  });
+
   it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
     const leftOwner = ownerNode(10, 50);
     const rightOwner = ownerNode(20, 500);
@@ -133,6 +232,87 @@ describe("captureFrameData", () => {
     );
   });
 
+  it("bounds concurrent same-target frame geometry resolution", async () => {
+    const owners = Array.from({ length: 8 }, (_, index) => ownerNode(10 + index, index * 40));
+    const childFrames = owners.map((owner, index) => {
+      const frameId = `child-${index}`;
+      const node: CapturedNode = {
+        backendNodeId: 100 + index,
+        parentBackendNodeId: null,
+        frameId,
+        ownerFrameBackendNodeId: owner.backendNodeId,
+        tag: "canvas",
+        attrs: {},
+        rect: null,
+        localRect: { x: 1, y: 2, w: 20, h: 10 },
+        paintOrder: 1,
+        position: "static",
+        pointerEvents: "auto",
+      };
+      return { frameId, owner, node };
+    });
+    const captured: CapturedViewModel = {
+      nodes: owners,
+      viewport: { width: 1000, height: 800 },
+      iframeNodes: new Map(childFrames.map(({ owner, node }) => [owner.backendNodeId, [node]])),
+      frameNodes: new Map([
+        ["main", owners],
+        ...childFrames.map(({ frameId, node }): [string, CapturedNode[]] => [frameId, [node]]),
+      ]),
+      frameOwnerBackendNodeIds: new Map(
+        childFrames.map(({ frameId, owner }) => [frameId, owner.backendNodeId]),
+      ),
+      rootFrameId: "main",
+      excludedBackendNodeIds: new Set(),
+    };
+    let activeWorlds = 0;
+    let maxActiveWorlds = 0;
+    const cdp: CdpRunner = {
+      send: vi.fn(async (_tabId, method, params) => {
+        if (method === "Page.getLayoutMetrics") {
+          return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
+        }
+        if (method === "DOM.getBoxModel") {
+          const backendNodeId = (params as { backendNodeId?: number }).backendNodeId ?? 10;
+          const x = (backendNodeId - 10) * 40;
+          return { model: { content: [x, 100, x + 30, 100, x + 30, 120, x, 120] } };
+        }
+        if (method === "Page.createIsolatedWorld") {
+          activeWorlds += 1;
+          maxActiveWorlds = Math.max(maxActiveWorlds, activeWorlds);
+          await new Promise((resolve) => setTimeout(resolve, 5));
+          activeWorlds -= 1;
+          return { executionContextId: 90 };
+        }
+        if (method === "Runtime.evaluate") {
+          return { result: { value: { width: 30, height: 20 } } };
+        }
+        if (method === "Accessibility.enable") return {};
+        if (method === "Accessibility.getFullAXTree") return { nodes: [] };
+        throw new Error(`unexpected ${method}`);
+      }) as CdpRunner["send"],
+      getFrameGraph: vi.fn(async () => ({
+        rootFrameId: "main",
+        frames: [
+          { frameId: "main", target: { tabId: 4 } },
+          ...childFrames.map(({ frameId, owner }) => ({
+            frameId,
+            parentFrameId: "main",
+            ownerBackendNodeId: owner.backendNodeId,
+            target: { tabId: 4 },
+          })),
+        ],
+      })),
+    };
+
+    await captureFrameData(cdp, 4, captured);
+
+    expect(maxActiveWorlds).toBe(4);
+    expect(childFrames.every(({ frameId }) => captured.frameNodes?.get(frameId)?.[0]?.rect)).toBe(
+      true,
+    );
+  });
+
   it("keeps OOPIF semantics when viewport projection is unavailable", async () => {
     const owner = { ...ownerNode(10, 50), rect: null, localRect: null };
     const captured: CapturedViewModel = {

apps/extension/src/tools/vom/__tests__/rendered-surfaces.test.ts

@@ -183,7 +183,7 @@ describe("rendered surface clustering and projection", () => {
       ],
       { width: 1280, height: 720 },
     );
-    const groups = clusterRenderedSurfaces(surfaces);
+    const clustered = clusterRenderedSurfaces(surfaces);
     const scene: VomScene = {
       viewport: { width: 1280, height: 720 },
       nodes: [
@@ -202,14 +202,14 @@ describe("rendered surface clustering and projection", () => {
       ],
     };
 
-    expect(groups).toHaveLength(1);
-    expect(groups[0]?.members).toHaveLength(2);
-    expect(groups[0]?.label).toBeUndefined();
+    expect(clustered.groups).toHaveLength(1);
+    expect(clustered.groups[0]?.memberCount).toBe(2);
+    expect(clustered.groups[0]?.label).toBeUndefined();
     expect(
       projectRenderedSurfaces(
         scene,
         [document([body, ...surfaces.map((s) => domNode(s.backendNodeId))])],
-        groups,
+        clustered,
       ),
     ).toMatchObject({
       visualSurfaces: [expect.objectContaining({ parentId: 10, backendNodeId: 3, memberCount: 2 })],
@@ -217,7 +217,7 @@ describe("rendered surface clustering and projection", () => {
   });
 
   it("does not merge a contained canvas with a larger independent canvas", () => {
-    const groups = clusterRenderedSurfaces([
+    const clustered = clusterRenderedSurfaces([
       ...discoverRenderedSurfaces(
         [
           document([
@@ -233,11 +233,34 @@ describe("rendered surface clustering and projection", () => {
       ),
     ]);
 
-    expect(groups).toHaveLength(2);
+    expect(clustered.groups).toHaveLength(2);
+  });
+
+  it("keeps the existing inclusive 90 percent stack threshold", () => {
+    const clustered = clusterRenderedSurfaces([
+      {
+        renderingKind: "canvas",
+        frameId: "main",
+        backendNodeId: 2,
+        parentBackendNodeId: 1,
+        visibleRect: { x: 0, y: 0, w: 100, h: 100 },
+        paintOrder: 1,
+      },
+      {
+        renderingKind: "canvas",
+        frameId: "main",
+        backendNodeId: 3,
+        parentBackendNodeId: 1,
+        visibleRect: { x: 5, y: 0, w: 90, h: 100 },
+        paintOrder: 2,
+      },
+    ]);
+
+    expect(clustered.groups).toEqual([expect.objectContaining({ memberCount: 2 })]);
   });
 
   it("does not merge coincident canvases owned by different containers", () => {
-    const groups = clusterRenderedSurfaces(
+    const clustered = clusterRenderedSurfaces(
       discoverRenderedSurfaces(
         [
           document([
@@ -251,6 +274,113 @@ describe("rendered surface clustering and projection", () => {
       ),
     );
 
-    expect(groups).toHaveLength(2);
+    expect(clustered.groups).toHaveLength(2);
+  });
+
+  it("keeps adjacent map tiles and twenty sparklines as independent surfaces", () => {
+    const surfaces = Array.from({ length: 20 }, (_, index) =>
+      domNode(100 + index, {
+        parentBackendNodeId: 1,
+        rect: { x: index * 12, y: 20, w: 10, h: 8 },
+      }),
+    );
+    const clustered = clusterRenderedSurfaces(
+      discoverRenderedSurfaces(
+        [document([domNode(1, { tag: "main", rect: null, localRect: null }), ...surfaces])],
+        { width: 1280, height: 720 },
+      ),
+    );
+
+    expect(clustered.groups).toHaveLength(20);
+    expect(clustered.groups.every((group) => group.memberCount === 1)).toBe(true);
+    expect(clustered.truncated).toBe(false);
+  });
+
+  it("clusters a large coincident stack without retaining every member", () => {
+    const surfaces = Array.from({ length: 10_000 }, (_, index) => ({
+      renderingKind: "canvas" as const,
+      frameId: "main",
+      backendNodeId: index + 1,
+      parentBackendNodeId: 1,
+      visibleRect: { x: 10, y: 20, w: 800, h: 500 },
+      paintOrder: index,
+    }));
+
+    const clustered = clusterRenderedSurfaces(surfaces);
+
+    expect(clustered.groups).toEqual([
+      expect.objectContaining({ memberCount: 10_000, representative: surfaces[9_999] }),
+    ]);
+    expect(clustered.truncated).toBe(false);
+  });
+
+  it("keeps large non-overlapping surface sets linear and independent", () => {
+    const surfaces = Array.from({ length: 10_000 }, (_, index) => ({
+      renderingKind: "canvas" as const,
+      frameId: "main",
+      backendNodeId: index + 1,
+      parentBackendNodeId: 1,
+      visibleRect: { x: index * 20, y: 0, w: 10, h: 10 },
+      paintOrder: index,
+    }));
+
+    const clustered = clusterRenderedSurfaces(surfaces);
+
+    expect(clustered.groups).toHaveLength(10_000);
+    expect(clustered.groups.every((group) => group.memberCount === 1)).toBe(true);
+    expect(clustered.truncated).toBe(false);
+  });
+
+  it("keeps the highest-priority surfaces when the computation guard truncates input", () => {
+    const surfaces = [
+      {
+        renderingKind: "canvas" as const,
+        frameId: "main",
+        backendNodeId: 1,
+        parentBackendNodeId: 10,
+        visibleRect: { x: 0, y: 0, w: 10, h: 10 },
+        paintOrder: 1,
+      },
+      {
+        renderingKind: "canvas" as const,
+        frameId: "main",
+        backendNodeId: 2,
+        parentBackendNodeId: 10,
+        visibleRect: { x: 20, y: 0, w: 100, h: 100 },
+        paintOrder: 1,
+      },
+      {
+        renderingKind: "canvas" as const,
+        frameId: "main",
+        backendNodeId: 3,
+        parentBackendNodeId: 10,
+        visibleRect: { x: 140, y: 0, w: 8, h: 8 },
+        paintOrder: 1,
+        label: "Important status",
+      },
+    ];
+
+    const clustered = clusterRenderedSurfaces(surfaces, { maxSurfaces: 2 });
+
+    expect(clustered.groups.map((group) => group.representative.backendNodeId).sort()).toEqual([
+      2, 3,
+    ]);
+    expect(clustered).toMatchObject({ truncated: true, omittedCount: 1 });
+  });
+
+  it("stops at the comparison budget without dropping already completed groups", () => {
+    const surfaces = Array.from({ length: 3 }, (_, index) => ({
+      renderingKind: "canvas" as const,
+      frameId: "main",
+      backendNodeId: index + 1,
+      parentBackendNodeId: 10,
+      visibleRect: { x: 0, y: 0, w: 100, h: 100 },
+      paintOrder: index + 1,
+    }));
+
+    const clustered = clusterRenderedSurfaces(surfaces, { maxComparisons: 1 });
+
+    expect(clustered.groups).toEqual([expect.objectContaining({ memberCount: 2 })]);
+    expect(clustered).toMatchObject({ truncated: true, omittedCount: 1 });
   });
 });

apps/extension/src/tools/vom/capture.ts

@@ -7,7 +7,7 @@
 import type { Rect, Viewport } from "@browser-skill/vom";
 import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
 import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
-import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
+import { type GeometryProjection, projectRectToViewport } from "../geometry";
 import type { CdpRunner } from "../shared";
 import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
 
@@ -1014,7 +1014,6 @@ function parseChildFrameDocuments(
   strings: string[],
   dpr: number,
   parentDocIndex: number,
-  parentNodes: CapturedNode[],
   parentContext: FrameContext,
   visited = new Set<number>(),
 ): ParseFrameDocumentsResult {
@@ -1029,19 +1028,17 @@ function parseChildFrameDocuments(
   }
 
   const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
-  const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
-
   for (const [nodeArrayIdx, childDocIndex] of cdi) {
     if (visited.has(childDocIndex)) continue;
     const childDoc = documents[childDocIndex];
     if (!childDoc) continue;
     const iframeBackendId = parentBackendIds[nodeArrayIdx];
     if (iframeBackendId === undefined) continue;
-    const iframeNode = parentNodeByBackendId.get(iframeBackendId);
-    const projection =
-      parentContext.projection && iframeNode?.localRect
-        ? childFrameProjection(parentContext.projection, iframeNode.localRect)
-        : null;
+    // DOMSnapshot reports the iframe owner's border-box bounds, but child
+    // document coordinates start at the content-box origin. Preserve the
+    // frame-local rectangles here and let frame-capture normalize them with
+    // authoritative content quads once the frame graph is available.
+    const projection = null;
 
     const childContext: FrameContext = {
       frameId: snapshotFrameId(childDoc, strings),
@@ -1065,7 +1062,6 @@ function parseChildFrameDocuments(
       strings,
       dpr,
       childDocIndex,
-      parsed.nodes,
       childContext,
       nextVisited,
     );
@@ -1143,7 +1139,7 @@ export async function captureViewModel(
   const nodes = mainParsed.nodes;
   const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
 
-  const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext);
+  const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, topContext);
   const iframeNodes = frameParsed.iframeNodes;
   for (const id of frameParsed.excludedBackendNodeIds) {
     excludedBackendNodeIds.add(id);

apps/extension/src/tools/vom/frame-capture.ts

@@ -1,6 +1,5 @@
 import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
-import { resolveFrameProjection } from "../frame-geometry";
-import { type GeometryProjection, projectRectToViewport } from "../geometry";
+import { createFrameGeometryContext } from "../frame-geometry";
 import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
 import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
 import {
@@ -13,6 +12,8 @@ import {
 export type FrameAxNode = FrameOwnedAxNode;
 export type CapturedFrameDocument<T extends FrameAxNode> = FrameDocument<T>;
 
+const FRAME_GEOMETRY_CONCURRENCY = 4;
+
 function throwIfAborted(signal?: AbortSignal): void {
   if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
 }
@@ -21,6 +22,22 @@ function isAbortError(err: unknown): boolean {
   return err instanceof Error && err.name === "AbortError";
 }
 
+async function forEachWithConcurrency<T>(
+  items: readonly T[],
+  concurrency: number,
+  worker: (item: T) => Promise<void>,
+): Promise<void> {
+  let next = 0;
+  const run = async () => {
+    while (next < items.length) {
+      const index = next;
+      next += 1;
+      await worker(items[index]);
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, run));
+}
+
 async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
   if (!cdp.getFrameGraph) return null;
   try {
@@ -31,21 +48,6 @@ async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFra
   }
 }
 
-function transformFrameNodes(
-  nodes: CapturedNode[],
-  projection: GeometryProjection | null,
-): CapturedNode[] {
-  return nodes.map((node) => ({
-    ...node,
-    rect: projection
-      ? (() => {
-          const bounds = projectRectToViewport(node.rect, projection);
-          return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
-        })()
-      : null,
-  }));
-}
-
 async function captureMissingFrameDocuments(
   cdp: CdpRunner,
   tabId: number,
@@ -63,16 +65,6 @@ async function captureMissingFrameDocuments(
       const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
         signal,
       });
-      let projection: GeometryProjection | null = null;
-      try {
-        projection = await resolveFrameProjection(cdp, graph, frame.frameId);
-      } catch (err) {
-        if (isAbortError(err)) throw err;
-        console.debug("[bsk observation] child frame geometry projection failed", {
-          frameId: frame.frameId,
-          err,
-        });
-      }
       const rootFrameId = child.rootFrameId ?? frame.frameId;
       const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
       if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
@@ -83,7 +75,7 @@ async function captureMissingFrameDocuments(
         );
       }
       for (const [childFrameId, nodes] of childFrames) {
-        captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
+        captured.frameNodes.set(childFrameId, nodes);
       }
       if (frame.ownerBackendNodeId !== undefined) {
         captured.iframeNodes.set(
@@ -109,6 +101,60 @@ async function captureMissingFrameDocuments(
   }
 }
 
+async function normalizeFrameLocalRects(
+  cdp: CdpRunner,
+  graph: CdpFrameGraph,
+  captured: CapturedViewModel,
+  signal?: AbortSignal,
+): Promise<void> {
+  if (!captured.frameNodes) return;
+  const geometry = createFrameGeometryContext(cdp, graph);
+  const frameById = new Map(graph.frames.map((frame) => [frame.frameId, frame]));
+  await forEachWithConcurrency(
+    [...captured.frameNodes],
+    FRAME_GEOMETRY_CONCURRENCY,
+    async ([frameId, nodes]) => {
+      if (frameId === graph.rootFrameId || !frameById.has(frameId)) return;
+      try {
+        const normalized: CapturedNode[] = [];
+        for (const node of nodes) {
+          if (!node.localRect) {
+            normalized.push({ ...node, rect: null });
+          } else {
+            const bounds = await geometry.projectFrameLocalRect(frameId, node.localRect);
+            normalized.push({
+              ...node,
+              rect: bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null,
+            });
+          }
+        }
+        throwIfAborted(signal);
+        captured.frameNodes?.set(frameId, normalized);
+        const ownerBackendNodeId =
+          frameById.get(frameId)?.ownerBackendNodeId ??
+          captured.frameOwnerBackendNodeIds?.get(frameId);
+        if (ownerBackendNodeId !== undefined) {
+          captured.iframeNodes.set(ownerBackendNodeId, normalized);
+        }
+      } catch (err) {
+        if (isAbortError(err)) throw err;
+        console.debug("[bsk observation] frame-local geometry normalization failed", {
+          frameId,
+          err,
+        });
+        const unavailable = nodes.map((node) => ({ ...node, rect: null }));
+        captured.frameNodes?.set(frameId, unavailable);
+        const ownerBackendNodeId =
+          frameById.get(frameId)?.ownerBackendNodeId ??
+          captured.frameOwnerBackendNodeIds?.get(frameId);
+        if (ownerBackendNodeId !== undefined) {
+          captured.iframeNodes.set(ownerBackendNodeId, unavailable);
+        }
+      }
+    },
+  );
+}
+
 async function captureAxTrees<T extends FrameAxNode>(
   cdp: CdpRunner,
   tabId: number,
@@ -174,7 +220,10 @@ export async function captureFrameData<T extends FrameAxNode>(
 ): Promise<CapturedFrameDocument<T>[]> {
   const graph = await discoverFrameGraph(cdp, tabId);
   const frames = graph?.frames ?? [];
-  if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal);
+  if (graph) {
+    await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal);
+    await normalizeFrameLocalRects(cdp, graph, captured, signal);
+  }
   throwIfAborted(signal);
   const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
   return buildFrameDocuments(graph, batches, captured);

apps/extension/src/tools/vom/rendered-surfaces/cluster.ts

@@ -1,8 +1,22 @@
-import type { Rect } from "@browser-skill/vom";
-import type { RenderedSurface, RenderedSurfaceGroup } from "./types";
+import { type Rect, selectHighestPriorityVisualSurfaces } from "@browser-skill/vom";
+import type { ClusteredRenderedSurfaces, RenderedSurface, RenderedSurfaceGroup } from "./types";
 
 const STACK_IOU_RATIO = 0.9;
 const STACK_SIZE_RATIO = 0.9;
+const SIZE_BAND_BASE = 1 / STACK_SIZE_RATIO;
+const POSITION_CELL_RATIO = 0.1;
+const MAX_CLUSTER_COMPARISONS = 1_000_000;
+const MAX_CLUSTER_SURFACES = 50_000;
+
+export interface ClusterRenderedSurfaceOptions {
+  maxComparisons?: number;
+  maxSurfaces?: number;
+}
+
+interface IndexedGroup extends RenderedSurfaceGroup {
+  creationIndex: number;
+  indexKey: string;
+}
 
 function area(rect: Rect): number {
   return Math.max(0, rect.w) * Math.max(0, rect.h);
@@ -14,9 +28,7 @@ function intersectionArea(a: Rect, b: Rect): number {
   return width * height;
 }
 
-function sameVisualStack(a: RenderedSurface, b: RenderedSurface): boolean {
-  if (a.frameId !== b.frameId) return false;
-  if (a.parentBackendNodeId !== b.parentBackendNodeId) return false;
+function stackScore(a: RenderedSurface, b: RenderedSurface): number | null {
   const aArea = area(a.visibleRect);
   const bArea = area(b.visibleRect);
   const intersection = intersectionArea(a.visibleRect, b.visibleRect);
@@ -25,41 +37,171 @@ function sameVisualStack(a: RenderedSurface, b: RenderedSurface): boolean {
     Math.min(a.visibleRect.w, b.visibleRect.w) / Math.max(a.visibleRect.w, b.visibleRect.w);
   const heightRatio =
     Math.min(a.visibleRect.h, b.visibleRect.h) / Math.max(a.visibleRect.h, b.visibleRect.h);
-  return (
-    union > 0 &&
-    intersection / union >= STACK_IOU_RATIO &&
-    widthRatio >= STACK_SIZE_RATIO &&
-    heightRatio >= STACK_SIZE_RATIO
-  );
+  const iou = union > 0 ? intersection / union : 0;
+  return iou >= STACK_IOU_RATIO && widthRatio >= STACK_SIZE_RATIO && heightRatio >= STACK_SIZE_RATIO
+    ? iou
+    : null;
 }
 
-function toGroup(members: RenderedSurface[]): RenderedSurfaceGroup {
-  const representative = [...members].sort(
-    (a, b) => area(b.visibleRect) - area(a.visibleRect) || b.paintOrder - a.paintOrder,
-  )[0];
+function sizeBand(value: number): number {
+  return Math.floor(Math.log(Math.max(Number.EPSILON, value)) / Math.log(SIZE_BAND_BASE));
+}
+
+function bandSize(band: number): number {
+  return SIZE_BAND_BASE ** band;
+}
+
+function spatialKey(rect: Rect): string {
+  const widthBand = sizeBand(rect.w);
+  const heightBand = sizeBand(rect.h);
+  const cellWidth = bandSize(widthBand) * POSITION_CELL_RATIO;
+  const cellHeight = bandSize(heightBand) * POSITION_CELL_RATIO;
+  const centerX = rect.x + rect.w / 2;
+  const centerY = rect.y + rect.h / 2;
+  return `${widthBand}:${heightBand}:${Math.floor(centerX / cellWidth)}:${Math.floor(centerY / cellHeight)}`;
+}
+
+function candidateKeys(rect: Rect): string[] {
+  const keys: string[] = [];
+  const centerX = rect.x + rect.w / 2;
+  const centerY = rect.y + rect.h / 2;
+  const widthBand = sizeBand(rect.w);
+  const heightBand = sizeBand(rect.h);
+  // A >=0.9 size ratio crosses at most one logarithmic band. IoU >=0.9
+  // likewise keeps centers within one 10%-of-size position cell.
+  for (let dw = -1; dw <= 1; dw += 1) {
+    for (let dh = -1; dh <= 1; dh += 1) {
+      const candidateWidthBand = widthBand + dw;
+      const candidateHeightBand = heightBand + dh;
+      const cellWidth = bandSize(candidateWidthBand) * POSITION_CELL_RATIO;
+      const cellHeight = bandSize(candidateHeightBand) * POSITION_CELL_RATIO;
+      const cellX = Math.floor(centerX / cellWidth);
+      const cellY = Math.floor(centerY / cellHeight);
+      for (let dx = -1; dx <= 1; dx += 1) {
+        for (let dy = -1; dy <= 1; dy += 1) {
+          keys.push(`${candidateWidthBand}:${candidateHeightBand}:${cellX + dx}:${cellY + dy}`);
+        }
+      }
+    }
+  }
+  return keys;
+}
+
+function preferredRepresentative(a: RenderedSurface, b: RenderedSurface): RenderedSurface {
+  const aArea = area(a.visibleRect);
+  const bArea = area(b.visibleRect);
+  return bArea > aArea || (bArea === aArea && b.paintOrder > a.paintOrder) ? b : a;
+}
+
+function partitionKey(surface: RenderedSurface): string {
+  return `${surface.frameId}\u0000${surface.parentBackendNodeId ?? "root"}`;
+}
+
+function normalizedLimit(value: number | undefined, fallback: number): number {
+  const resolved = value ?? fallback;
+  return Number.isFinite(resolved) ? Math.max(0, Math.floor(resolved)) : fallback;
+}
+
+export function clusterRenderedSurfaces(
+  surfaces: RenderedSurface[],
+  options: ClusterRenderedSurfaceOptions = {},
+): ClusteredRenderedSurfaces {
+  const maxComparisons = normalizedLimit(options.maxComparisons, MAX_CLUSTER_COMPARISONS);
+  const maxSurfaces = normalizedLimit(options.maxSurfaces, MAX_CLUSTER_SURFACES);
+  const prioritized = selectHighestPriorityVisualSurfaces(surfaces, maxSurfaces);
+  const partitions = new Map<string, Map<string, Set<IndexedGroup>>>();
+  const groups: IndexedGroup[] = [];
+  let comparisons = 0;
+  let processed = 0;
+  let truncated = prioritized.length < surfaces.length;
+  let comparisonLimitReached = false;
+
+  for (const surface of prioritized) {
+    if (comparisons >= maxComparisons) {
+      truncated = true;
+      break;
+    }
+    const key = partitionKey(surface);
+    let index = partitions.get(key);
+    if (!index) {
+      index = new Map();
+      partitions.set(key, index);
+    }
+    const candidates = new Set<IndexedGroup>();
+    for (const candidateKey of candidateKeys(surface.visibleRect)) {
+      for (const candidate of index.get(candidateKey) ?? []) candidates.add(candidate);
+    }
+
+    let best: { group: IndexedGroup; score: number; areaDelta: number } | null = null;
+    for (const group of candidates) {
+      if (comparisons >= maxComparisons) {
+        truncated = true;
+        comparisonLimitReached = true;
+        break;
+      }
+      comparisons += 1;
+      const score = stackScore(group.representative, surface);
+      if (score === null) continue;
+      const areaDelta = Math.abs(
+        area(group.representative.visibleRect) - area(surface.visibleRect),
+      );
+      if (
+        !best ||
+        score > best.score ||
+        (score === best.score && areaDelta < best.areaDelta) ||
+        (score === best.score &&
+          areaDelta === best.areaDelta &&
+          group.creationIndex < best.group.creationIndex)
+      ) {
+        best = { group, score, areaDelta };
+      }
+    }
+    if (comparisonLimitReached) break;
+
+    if (best) {
+      const group = best.group;
+      const representative = preferredRepresentative(group.representative, surface);
+      group.memberCount += 1;
+      group.label ??= surface.label;
+      if (representative !== group.representative) {
+        index.get(group.indexKey)?.delete(group);
+        group.representative = representative;
+        group.indexKey = spatialKey(representative.visibleRect);
+        const indexed = index.get(group.indexKey) ?? new Set<IndexedGroup>();
+        indexed.add(group);
+        index.set(group.indexKey, indexed);
+      }
+      processed += 1;
+      continue;
+    }
+
+    const indexKey = spatialKey(surface.visibleRect);
+    const group: IndexedGroup = {
+      frameId: surface.frameId,
+      parentBackendNodeId: surface.parentBackendNodeId,
+      representative: surface,
+      ...(surface.label ? { label: surface.label } : {}),
+      memberCount: 1,
+      creationIndex: groups.length,
+      indexKey,
+    };
+    groups.push(group);
+    const indexed = index.get(indexKey) ?? new Set<IndexedGroup>();
+    indexed.add(group);
+    index.set(indexKey, indexed);
+    processed += 1;
+  }
+
   return {
-    frameId: representative.frameId,
-    members,
-    representative,
-    label: members.map((surface) => surface.label).find((label) => label !== undefined),
+    groups: groups
+      .map(({ creationIndex: _creationIndex, indexKey: _indexKey, ...group }) => group)
+      .sort(
+        (a, b) =>
+          a.frameId.localeCompare(b.frameId) ||
+          a.representative.visibleRect.y - b.representative.visibleRect.y ||
+          a.representative.visibleRect.x - b.representative.visibleRect.x,
+      ),
+    truncated,
+    ...(truncated ? { omittedCount: surfaces.length - processed } : {}),
   };
 }
-
-export function clusterRenderedSurfaces(surfaces: RenderedSurface[]): RenderedSurfaceGroup[] {
-  const groups: RenderedSurface[][] = [];
-  for (const surface of surfaces) {
-    const matching = groups.find((group) =>
-      group.some((member) => sameVisualStack(member, surface)),
-    );
-    if (matching) matching.push(surface);
-    else groups.push([surface]);
-  }
-  return groups
-    .map(toGroup)
-    .sort(
-      (a, b) =>
-        a.frameId.localeCompare(b.frameId) ||
-        a.representative.visibleRect.y - b.representative.visibleRect.y ||
-        a.representative.visibleRect.x - b.representative.visibleRect.x,
-    );
-}

apps/extension/src/tools/vom/rendered-surfaces/index.ts

@@ -1,4 +1,8 @@
-export { clusterRenderedSurfaces } from "./cluster";
+export { type ClusterRenderedSurfaceOptions, clusterRenderedSurfaces } from "./cluster";
 export { discoverRenderedSurfaces } from "./discovery";
 export { projectRenderedSurfaces } from "./project";
-export type { RenderedSurface, RenderedSurfaceGroup } from "./types";
+export type {
+  ClusteredRenderedSurfaces,
+  RenderedSurface,
+  RenderedSurfaceGroup,
+} from "./types";

apps/extension/src/tools/vom/rendered-surfaces/project.ts

@@ -1,7 +1,7 @@
 import type { VomScene, VomVisualSurface } from "@browser-skill/vom";
 import type { CapturedFrameDocument } from "../frame-capture";
 import type { SemanticAxNode } from "../semantic-graph";
-import type { RenderedSurfaceGroup } from "./types";
+import type { ClusteredRenderedSurfaces, RenderedSurfaceGroup } from "./types";
 
 function frameBackendKey(frameId: string, backendNodeId: number): string {
   return `${frameId}\u0000${backendNodeId}`;
@@ -75,12 +75,12 @@ function buildProjectionIndex(
 export function projectRenderedSurfaces(
   scene: VomScene,
   documents: CapturedFrameDocument<SemanticAxNode>[],
-  groups: RenderedSurfaceGroup[],
+  clustered: ClusteredRenderedSurfaces,
 ): VomScene {
   const index = buildProjectionIndex(scene, documents);
   const visualSurfaces: VomVisualSurface[] = [];
 
-  for (const group of groups) {
+  for (const group of clustered.groups) {
     const parentId = nearestSceneParent(index, group);
     visualSurfaces.push({
       parentId,
@@ -89,12 +89,20 @@ export function projectRenderedSurfaces(
       renderingKind: "canvas",
       visibleRect: group.representative.visibleRect,
       ...(group.label ? { label: group.label } : {}),
-      memberCount: group.members.length,
+      memberCount: group.memberCount,
     });
   }
 
   return {
     ...scene,
     ...(visualSurfaces.length > 0 ? { visualSurfaces } : {}),
+    ...(clustered.truncated
+      ? {
+          visualSurfacesTruncated: true,
+          ...(clustered.omittedCount !== undefined
+            ? { omittedVisualSurfaceCount: clustered.omittedCount }
+            : {}),
+        }
+      : {}),
   };
 }

apps/extension/src/tools/vom/rendered-surfaces/types.ts

@@ -12,7 +12,14 @@ export interface RenderedSurface {
 
 export interface RenderedSurfaceGroup {
   frameId: string;
-  members: RenderedSurface[];
+  parentBackendNodeId: number | null;
   representative: RenderedSurface;
   label?: string;
+  memberCount: number;
+}
+
+export interface ClusteredRenderedSurfaces {
+  groups: RenderedSurfaceGroup[];
+  truncated: boolean;
+  omittedCount?: number;
 }

apps/extension/test-fixtures/rendered-surfaces/README.md

@@ -95,6 +95,11 @@ Use the normal DOM buttons and re-observe after each action:
 - After **Restore size**: present with a fresh ref.
 - After **Remove canvas**: absent.
 
+### `sparklines.html`
+
+Expected Surface labels `fixture-sparkline-01` through `fixture-sparkline-20` must all be present.
+The adjacent, non-overlapping canvases remain twenty independent screenshot-only Surface refs.
+
 ## Invariants
 
 - `snapshot` must not emit any of these Surface markers.

apps/extension/test-fixtures/rendered-surfaces/sparklines.html

@@ -0,0 +1,28 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Rendered Surface sparkline matrix</title>
+    <style>
+      html, body { margin: 0; font: 14px system-ui, sans-serif; }
+      body { padding: 16px; }
+      main { display: grid; grid-template-columns: repeat(5, 120px); gap: 12px; }
+      canvas { display: block; width: 100px; height: 30px; background: #eaf2f8; }
+    </style>
+  </head>
+  <body>
+    <h1>Twenty independent sparklines</h1>
+    <main id="sparklines"></main>
+    <script>
+      const host = document.querySelector("#sparklines");
+      for (let index = 1; index <= 20; index += 1) {
+        const canvas = document.createElement("canvas");
+        canvas.width = 100;
+        canvas.height = 30;
+        canvas.setAttribute("aria-label", `fixture-sparkline-${String(index).padStart(2, "0")}`);
+        host.append(canvas);
+      }
+    </script>
+  </body>
+</html>

packages/vom/src/__tests__/render.test.ts

@@ -49,7 +49,7 @@ describe("renderVom single-layer page", () => {
 
     expect(out.text).toContain("[visual surfaces]");
     expect(out.text).toContain(
-      '@e1 surface "Data grid" [bounds=10,20,800,500; rendering=canvas; layers=2; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
+      '@e1 surface "Data grid" [bounds=10,20,800,500; rendering=canvas; layers=2; visual-only; screenshot-only; requires=image-understanding]',
     );
     expect(out.refs).toContainEqual(
       expect.objectContaining({
@@ -78,7 +78,7 @@ describe("renderVom single-layer page", () => {
     const out = renderVom(input);
 
     expect(out.text).toContain(
-      '@e1 surface "canvas visual surface" [bounds=10,20,800,500; rendering=canvas; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
+      '@e1 surface "canvas visual surface" [bounds=10,20,800,500; rendering=canvas; visual-only; screenshot-only; requires=image-understanding]',
     );
     expect(out.refs).toContainEqual(
       expect.objectContaining({
@@ -117,7 +117,7 @@ describe("renderVom single-layer page", () => {
     );
   });
 
-  it("bounds visual output and prefers explicitly labeled surfaces", () => {
+  it("uses the appendix token budget and prefers explicitly labeled surfaces", () => {
     const input = scene([node({ id: 1, role: "RootWebArea", frameId: "main" })]);
     input.visualSurfaces = Array.from({ length: 12 }, (_, index) => ({
       parentId: 1,
@@ -132,10 +132,30 @@ describe("renderVom single-layer page", () => {
     const out = renderVom(input);
     const surfaceRefs = out.refs.filter((ref) => ref.kind === "surface");
 
-    expect(surfaceRefs).toHaveLength(8);
+    expect(surfaceRefs).toHaveLength(12);
     expect(surfaceRefs[0]).toMatchObject({ backendNodeId: 111, name: "Important status" });
-    expect(out.text).toContain("… 4 additional visual surfaces omitted");
-    expect(out.truncated).toBe(true);
+    expect(out.truncated).toBe(false);
+  });
+
+  it("renders twenty independent sparklines without changing DOM refs", () => {
+    const input = scene([
+      node({ id: 1, role: "RootWebArea", frameId: "main" }),
+      node({ id: 2, parentId: 1, role: "button", name: "Submit", tag: "button" }),
+    ]);
+    input.visualSurfaces = Array.from({ length: 20 }, (_, index) => ({
+      parentId: 1,
+      backendNodeId: 100 + index,
+      frameId: "main",
+      renderingKind: "canvas" as const,
+      visibleRect: { x: index * 12, y: 20, w: 10, h: 8 },
+      memberCount: 1,
+    }));
+
+    const out = renderVom(input);
+
+    expect(out.refs[0]).toMatchObject({ ref: "e1", backendNodeId: 2, kind: "dom" });
+    expect(out.refs.filter((ref) => ref.kind === "surface")).toHaveLength(20);
+    expect(out.truncated).toBe(false);
   });
 
   it("does not let a visual surface exhaust the budget before later DOM content", () => {

packages/vom/src/__tests__/visual-surface-priority.test.ts

@@ -0,0 +1,35 @@
+import { describe, expect, it } from "vitest";
+import {
+  compareVisualSurfacePriority,
+  selectHighestPriorityVisualSurfaces,
+} from "../visual-surface-priority";
+
+function surface(backendNodeId: number, area: number, label?: string) {
+  return {
+    backendNodeId,
+    frameId: "main",
+    visibleRect: { x: backendNodeId, y: 0, w: area, h: 1 },
+    ...(label ? { label } : {}),
+  };
+}
+
+describe("visual surface priority selection", () => {
+  it("matches a full priority sort when selecting a bounded prefix", () => {
+    const surfaces = Array.from({ length: 600 }, (_, index) => surface(index, index + 1));
+    surfaces[599] = surface(599, 1, "Important status");
+
+    const selected = selectHighestPriorityVisualSurfaces(surfaces, 512);
+    const expected = [...surfaces].sort(compareVisualSurfacePriority).slice(0, 512);
+
+    expect(selected).toEqual(expected);
+    expect(selected[0]?.backendNodeId).toBe(599);
+  });
+
+  it("normalizes empty, fractional, and unbounded limits", () => {
+    const surfaces = [surface(1, 1), surface(2, 2), surface(3, 3)];
+
+    expect(selectHighestPriorityVisualSurfaces(surfaces, 0)).toEqual([]);
+    expect(selectHighestPriorityVisualSurfaces(surfaces, 1.9)).toEqual([surface(3, 3)]);
+    expect(selectHighestPriorityVisualSurfaces(surfaces, Number.POSITIVE_INFINITY)).toHaveLength(3);
+  });
+});

packages/vom/src/index.ts

@@ -20,3 +20,8 @@ export type {
   VomScene,
   VomVisualSurface,
 } from "./types";
+export {
+  compareVisualSurfacePriority,
+  selectHighestPriorityVisualSurfaces,
+  type VisualSurfacePriorityInput,
+} from "./visual-surface-priority";

packages/vom/src/render.ts

@@ -11,6 +11,7 @@ import type {
   VomScene,
   VomVisualSurface,
 } from "./types";
+import { selectHighestPriorityVisualSurfaces } from "./visual-surface-priority";
 
 const SKIP_ROLES = new Set(["generic", "none", "presentation", "inlinetextbox"]);
 
@@ -88,7 +89,8 @@ const MAX_CUSTOM_RECOVERY_AREA = 180_000;
 const MAX_RECOVERABLE_SINGLE_INTENT_AREA = 360_000;
 const MAX_HANDLE_CONTEXT_ITEMS = 3;
 const MAX_SURFACE_ITEMS = 12;
-const MAX_VISUAL_SURFACES = 8;
+const DEFAULT_VISUAL_APPENDIX_TOKENS = 2_048;
+const MAX_VISUAL_SURFACE_REFS = 512;
 const MAX_SCOPE_LINES = 40;
 const RECOVERY_HANDLER_ATTRS = ["onclick", "onmousedown", "onkeydown", "onkeyup", "onkeypress"];
 const NATIVE_TAGS = new Set(["button", "input", "select", "textarea"]);
@@ -796,23 +798,16 @@ interface RenderState {
   surfaceMap: Map<number, CondSurface>;
   scopeMap: Map<number, ActiveScopeBlock>;
   visualSurfaces: VomVisualSurface[];
+  visualSurfacesTruncated: boolean;
+  omittedVisualSurfaceCount?: number;
 }
 
-function visualSurfaceArea(surface: VomVisualSurface): number {
-  return Math.max(0, surface.visibleRect.w) * Math.max(0, surface.visibleRect.h);
-}
-
-function compareVisualSurfaces(a: VomVisualSurface, b: VomVisualSurface): number {
-  const aLabeled = cleaned(a.label) !== undefined;
-  const bLabeled = cleaned(b.label) !== undefined;
-  return (
-    Number(bLabeled) - Number(aLabeled) ||
-    visualSurfaceArea(b) - visualSurfaceArea(a) ||
-    a.frameId.localeCompare(b.frameId) ||
-    a.visibleRect.y - b.visibleRect.y ||
-    a.visibleRect.x - b.visibleRect.x ||
-    a.backendNodeId - b.backendNodeId
-  );
+function tryAppendLine(state: RenderState, line: string, tokenLimit = state.maxTokens): boolean {
+  const nextTokens = state.tokens + estimateTokens(line);
+  if (nextTokens > tokenLimit) return false;
+  state.lines.push(line);
+  state.tokens = nextTokens;
+  return true;
 }
 
 function visualSurfaceLine(surface: VomVisualSurface, ref: string): string {
@@ -820,55 +815,50 @@ function visualSurfaceLine(surface: VomVisualSurface, ref: string): string {
   const layers = surface.memberCount > 1 ? `; layers=${surface.memberCount}` : "";
   const { x, y, w, h } = surface.visibleRect;
   const bounds = [x, y, w, h].map(Math.round).join(",");
-  return `  @${ref} surface ${JSON.stringify(label)} [bounds=${bounds}; rendering=${surface.renderingKind}${layers}; visual-only; requires=image-understanding; use: bsk screenshot --ref @${ref}]`;
+  return `  @${ref} surface ${JSON.stringify(label)} [bounds=${bounds}; rendering=${surface.renderingKind}${layers}; visual-only; screenshot-only; requires=image-understanding]`;
 }
 
 function selectVisualSurfaces(
   surfaces: readonly VomVisualSurface[],
   nodesById: ReadonlyMap<number, VomNode>,
 ): { selected: VomVisualSurface[]; eligibleCount: number } {
-  const selected: VomVisualSurface[] = [];
   let eligibleCount = 0;
-
-  for (const surface of surfaces) {
-    if (surface.parentId !== null && !nodesById.has(surface.parentId)) continue;
-    eligibleCount += 1;
-
-    const insertionIndex = selected.findIndex(
-      (candidate) => compareVisualSurfaces(surface, candidate) < 0,
-    );
-    if (insertionIndex < 0) {
-      if (selected.length < MAX_VISUAL_SURFACES) selected.push(surface);
-      continue;
+  function* eligibleSurfaces(): Generator<VomVisualSurface> {
+    for (const surface of surfaces) {
+      if (surface.parentId !== null && !nodesById.has(surface.parentId)) continue;
+      eligibleCount += 1;
+      yield surface;
     }
-    selected.splice(insertionIndex, 0, surface);
-    if (selected.length > MAX_VISUAL_SURFACES) selected.pop();
   }
-
-  return { selected, eligibleCount };
+  return {
+    selected: selectHighestPriorityVisualSurfaces(eligibleSurfaces(), MAX_VISUAL_SURFACE_REFS),
+    eligibleCount,
+  };
 }
 
 function emitVisualAppendix(state: RenderState): void {
   if (state.stopped) return;
   const { selected, eligibleCount } = selectVisualSurfaces(state.visualSurfaces, state.nodesById);
-  if (eligibleCount === 0) return;
+  if (eligibleCount === 0 && !state.visualSurfacesTruncated) return;
 
   const header = "[visual surfaces]";
+  const appendixStartTokens = state.tokens;
+  const appendixTokenLimit = Number.isFinite(state.maxTokens)
+    ? state.maxTokens
+    : appendixStartTokens + DEFAULT_VISUAL_APPENDIX_TOKENS;
   let headerEmitted = false;
   let emitted = 0;
   for (const surface of selected) {
+    if (emitted >= MAX_VISUAL_SURFACE_REFS) break;
     const ref = `e${state.nextRef}`;
     const line = visualSurfaceLine(surface, ref);
-    const nextTokens =
-      state.tokens + estimateTokens(line) + (headerEmitted ? 0 : estimateTokens(header));
-    if (nextTokens > state.maxTokens) break;
+    const requiredTokens = estimateTokens(line) + (headerEmitted ? 0 : estimateTokens(header));
+    if (state.tokens + requiredTokens > appendixTokenLimit) break;
     if (!headerEmitted) {
-      state.lines.push(header);
-      state.tokens += estimateTokens(header);
+      if (!tryAppendLine(state, header, appendixTokenLimit)) break;
       headerEmitted = true;
     }
-    state.lines.push(line);
-    state.tokens += estimateTokens(line);
+    if (!tryAppendLine(state, line, appendixTokenLimit)) break;
     const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`;
     state.refs.push({
       ref,
@@ -885,16 +875,15 @@ function emitVisualAppendix(state: RenderState): void {
     emitted += 1;
   }
 
-  const omitted = eligibleCount - emitted;
-  if (omitted > 0) {
+  const locallyOmitted = eligibleCount - emitted;
+  const upstreamOmitted = state.omittedVisualSurfaceCount ?? 0;
+  const omitted = locallyOmitted + upstreamOmitted;
+  if (omitted > 0 || state.visualSurfacesTruncated) {
     state.truncated = true;
     if (headerEmitted) {
-      const summary = `  … ${omitted} additional visual surfaces omitted`;
-      const nextTokens = state.tokens + estimateTokens(summary);
-      if (nextTokens <= state.maxTokens) {
-        state.lines.push(summary);
-        state.tokens = nextTokens;
-      }
+      const count =
+        omitted > 0 ? `${state.visualSurfacesTruncated ? "at least " : ""}${omitted}` : "some";
+      tryAppendLine(state, `  … ${count} additional visual surfaces omitted`, appendixTokenLimit);
     }
   }
 }
@@ -903,28 +892,22 @@ function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: Ren
   if (state.stopped || scope.lines.length === 0) return;
   const indent = "  ".repeat(depth);
   const header = `${indent}[§ active: ${scope.label}]`;
-  const headerTokens = estimateTokens(header);
-  if (state.tokens + headerTokens > state.maxTokens) {
+  if (!tryAppendLine(state, header)) {
     state.truncated = true;
     state.stopped = true;
     return;
   }
-  state.lines.push(header);
-  state.tokens += headerTokens;
 
   for (const text of scope.lines.slice(0, MAX_SCOPE_LINES)) {
     if (state.stopped) return;
     const clean = cleaned(text);
     if (!clean) continue;
     const line = `${indent}  ${clean}`;
-    const nextTokens = state.tokens + estimateTokens(line);
-    if (nextTokens > state.maxTokens) {
+    if (!tryAppendLine(state, line)) {
       state.truncated = true;
       state.stopped = true;
       return;
     }
-    state.lines.push(line);
-    state.tokens = nextTokens;
   }
 }
 
@@ -959,15 +942,11 @@ function renderTree(children: Map<number | null, VomNode[]>, state: RenderState)
     const context = ref ? handleContext(node, state) : [];
     const surface = state.surfaceMap.get(node.id);
     const line = renderNodeLine(node, depth, ref, context, surface, state.redactValues);
-    const nextTokens = state.tokens + estimateTokens(line);
-    if (nextTokens > state.maxTokens) {
+    if (!tryAppendLine(state, line)) {
       state.truncated = true;
       state.stopped = true;
       break;
     }
-
-    state.lines.push(line);
-    state.tokens = nextTokens;
     if (ref) {
       const name = cleaned(node.name);
       state.refs.push({
@@ -1001,6 +980,8 @@ function renderNodes(
   surfaces: CondSurface[] = [],
   activeScopeBlocks: ActiveScopeBlock[] = [],
   visualSurfaces: VomVisualSurface[] = [],
+  visualSurfacesTruncated = false,
+  omittedVisualSurfaceCount?: number,
 ): RenderState {
   const children = buildChildren(nodes);
   const state: RenderState = {
@@ -1021,6 +1002,8 @@ function renderNodes(
     surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])),
     scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])),
     visualSurfaces,
+    visualSurfacesTruncated,
+    omittedVisualSurfaceCount,
   };
 
   renderTree(children, state);
@@ -1241,10 +1224,11 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
     scene.surfaces,
     scene.activeScopeBlocks,
     scene.visualSurfaces?.filter((surface) => visibleSurface(surface.parentId)),
+    scene.visualSurfacesTruncated,
+    scene.omittedVisualSurfaceCount,
   );
   const occlusionLine = renderPageOcclusionLine(hiddenCount);
-  state.lines.push(occlusionLine);
-  state.tokens += estimateTokens(occlusionLine);
+  if (!tryAppendLine(state, occlusionLine)) state.truncated = true;
   emitVisualAppendix(state);
 
   return {
@@ -1273,6 +1257,8 @@ export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult
     scene.surfaces,
     scene.activeScopeBlocks,
     scene.visualSurfaces,
+    scene.visualSurfacesTruncated,
+    scene.omittedVisualSurfaceCount,
   );
   emitVisualAppendix(state);
 

packages/vom/src/types.ts

@@ -63,6 +63,10 @@ export interface VomScene {
   activeScopeBlocks?: ActiveScopeBlock[];
   /** Non-semantic rendered regions discovered alongside the AX/DOM tree. */
   visualSurfaces?: VomVisualSurface[];
+  /** Upstream surface discovery/aggregation stopped at a resource guardrail. */
+  visualSurfacesTruncated?: boolean;
+  /** Exact omitted count when upstream could determine it. */
+  omittedVisualSurfaceCount?: number;
 }
 
 export interface VomOptions {

packages/vom/src/visual-surface-priority.ts

@@ -0,0 +1,80 @@
+export interface VisualSurfacePriorityInput {
+  label?: string;
+  visibleRect: { x: number; y: number; w: number; h: number };
+  frameId: string;
+  backendNodeId: number;
+}
+
+function hasLabel(label: string | undefined): boolean {
+  return (label?.trim().length ?? 0) > 0;
+}
+
+function area(surface: VisualSurfacePriorityInput): number {
+  return Math.max(0, surface.visibleRect.w) * Math.max(0, surface.visibleRect.h);
+}
+
+/** Negative means `a` should be emitted before `b`. */
+export function compareVisualSurfacePriority(
+  a: VisualSurfacePriorityInput,
+  b: VisualSurfacePriorityInput,
+): number {
+  return (
+    Number(hasLabel(b.label)) - Number(hasLabel(a.label)) ||
+    area(b) - area(a) ||
+    a.frameId.localeCompare(b.frameId) ||
+    a.visibleRect.y - b.visibleRect.y ||
+    a.visibleRect.x - b.visibleRect.x ||
+    a.backendNodeId - b.backendNodeId
+  );
+}
+
+function siftWorstUp<T extends VisualSurfacePriorityInput>(heap: T[], start: number): void {
+  let index = start;
+  while (index > 0) {
+    const parent = Math.floor((index - 1) / 2);
+    if (compareVisualSurfacePriority(heap[index], heap[parent]) <= 0) break;
+    [heap[index], heap[parent]] = [heap[parent], heap[index]];
+    index = parent;
+  }
+}
+
+function siftWorstDown<T extends VisualSurfacePriorityInput>(heap: T[], start: number): void {
+  let index = start;
+  while (true) {
+    const left = index * 2 + 1;
+    if (left >= heap.length) return;
+    const right = left + 1;
+    let worst = left;
+    if (right < heap.length && compareVisualSurfacePriority(heap[right], heap[left]) > 0) {
+      worst = right;
+    }
+    if (compareVisualSurfacePriority(heap[worst], heap[index]) <= 0) return;
+    [heap[index], heap[worst]] = [heap[worst], heap[index]];
+    index = worst;
+  }
+}
+
+/** Selects the best `limit` surfaces and returns them in priority order. */
+export function selectHighestPriorityVisualSurfaces<T extends VisualSurfacePriorityInput>(
+  surfaces: Iterable<T>,
+  limit: number,
+): T[] {
+  if (!Number.isFinite(limit)) return [...surfaces].sort(compareVisualSurfacePriority);
+  const normalizedLimit = Math.max(0, Math.floor(limit));
+  if (normalizedLimit === 0) return [];
+  if (Array.isArray(surfaces) && surfaces.length <= normalizedLimit) {
+    return [...surfaces].sort(compareVisualSurfacePriority);
+  }
+
+  const heap: T[] = [];
+  for (const surface of surfaces) {
+    if (heap.length < normalizedLimit) {
+      heap.push(surface);
+      siftWorstUp(heap, heap.length - 1);
+    } else if (compareVisualSurfacePriority(surface, heap[0]) < 0) {
+      heap[0] = surface;
+      siftWorstDown(heap, 0);
+    }
+  }
+  return heap.sort(compareVisualSurfacePriority);
+}