fix(vom): add fixtures, remove fixed canvas numbers under budget, improve efficiency

apps/extension/src/session-manager/__tests__/ref-store.test.ts

@@ -77,4 +77,14 @@ describe("RefStore", () => {
     expect(s.resolveEntry("e1")?.capabilities).toEqual(["screenshot"]);
     expect(s.resolveEntry("e2")?.capabilities).toEqual(["screenshot"]);
   });
+
+  it("copies caller-owned capability arrays", () => {
+    const s = new RefStore();
+    const capabilities: Array<"interact" | "screenshot"> = ["screenshot"];
+    s.set("e1", 42, { tabId: 7, capabilities });
+
+    capabilities.push("interact");
+
+    expect(s.resolveEntry("e1")?.capabilities).toEqual(["screenshot"]);
+  });
 });

apps/extension/src/session-manager/ref-store.ts

@@ -45,7 +45,7 @@ function capabilitiesFor(
   capabilities: RefCapability[] | undefined,
 ): RefCapability[] {
   if (kind === "surface") return ["screenshot"];
-  return capabilities ?? ["interact", "screenshot"];
+  return capabilities ? [...capabilities] : ["interact", "screenshot"];
 }
 
 export class RefStore {

apps/extension/src/tools/__tests__/frame-geometry.test.ts

@@ -1,5 +1,9 @@
 import { describe, expect, it, vi } from "vitest";
-import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry";
+import {
+  createFrameGeometryContext,
+  resolveFrameProjection,
+  resolveNodeGeometry,
+} from "../frame-geometry";
 import {
   clipPolygon,
   polygonArea,
@@ -37,6 +41,53 @@ describe("frame geometry projection", () => {
     expect(polygonBounds(projected.flat())).toEqual({ x: 224, y: 346, width: 200, height: 80 });
   });
 
+  it("projects nested same-target frame-local bounds through authoritative content quads", async () => {
+    const boxCalls: number[] = [];
+    const cdp: CdpRunner = {
+      send: vi.fn(async (_tabId, method, params) => {
+        if (method === "Page.getLayoutMetrics") {
+          return { cssLayoutViewport: { clientWidth: 1_000, clientHeight: 800 } };
+        }
+        if (method === "Page.createIsolatedWorld") return { executionContextId: 91 };
+        if (method === "Runtime.evaluate") {
+          return { result: { value: { width: 100, height: 80 } } };
+        }
+        if (method === "DOM.getBoxModel") {
+          const backendNodeId = (params as { backendNodeId?: number }).backendNodeId ?? 0;
+          boxCalls.push(backendNodeId);
+          return backendNodeId === 10
+            ? { model: { content: [30, 100.5, 330, 100.5, 330, 300.5, 30, 300.5] } }
+            : { model: { content: [45, 120.5, 145, 120.5, 145, 200.5, 45, 200.5] } };
+        }
+        throw new Error(`unexpected ${method}`);
+      }) as CdpRunner["send"],
+    };
+    const geometry = createFrameGeometryContext(cdp, {
+      rootFrameId: "main",
+      frames: [
+        { frameId: "main", target: { tabId: 4 } },
+        {
+          frameId: "child",
+          parentFrameId: "main",
+          ownerBackendNodeId: 10,
+          target: { tabId: 4 },
+        },
+        {
+          frameId: "nested",
+          parentFrameId: "child",
+          ownerBackendNodeId: 20,
+          target: { tabId: 4 },
+        },
+      ],
+    });
+
+    await expect(
+      geometry.projectFrameLocalRect("nested", { x: 12, y: 12, w: 40, h: 20 }),
+    ).resolves.toEqual({ x: 57, y: 132.5, width: 40, height: 20 });
+    await geometry.projectFrameLocalRect("nested", { x: 1, y: 1, w: 2, h: 2 });
+    expect(boxCalls.sort()).toEqual([10, 20]);
+  });
+
   it("uses a projective mapping for perspective-transformed iframe quads", () => {
     const quad: Quad = [
       { x: 0, y: 0 },

apps/extension/src/tools/__tests__/observation.test.ts

@@ -330,6 +330,7 @@ describe("handleScreenshot", () => {
       tabId: 7,
       kind: "surface",
       capabilities: ["screenshot"],
+      visibleRect: { x: 0, y: 0, w: 4096, h: 4096 },
     });
     const { cdp, sent } = makeFakeCdp({
       "Page.getLayoutMetrics": () => ({
@@ -359,6 +360,49 @@ describe("handleScreenshot", () => {
     expect(clip.clip?.scale).toBeCloseTo(Math.sqrt(4_000_000 / (4096 * 4096)));
   });
 
+  it("retries once when actual surface PNG dimensions exceed the pixel budget", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    ctx.refStore.set("e5", 999, {
+      tabId: 7,
+      kind: "surface",
+      visibleRect: { x: 0, y: 0, w: 1_000, h: 1_000 },
+    });
+    let captures = 0;
+    const { cdp, sent } = makeFakeCdp({
+      "Runtime.evaluate": () => ({ result: { value: 1 } }),
+      "Page.getLayoutMetrics": () => ({
+        cssLayoutViewport: { clientWidth: 1_000, clientHeight: 1_000 },
+      }),
+      "DOM.getContentQuads": () => ({
+        quads: [[0, 0, 1_000, 0, 1_000, 1_000, 0, 1_000]],
+      }),
+      "Page.captureScreenshot": () => ({
+        data:
+          ++captures === 1
+            ? pngWithIhdrDimensions(4_000, 4_000)
+            : pngWithIhdrDimensions(1_998, 1_998),
+      }),
+    });
+
+    const res = await handleScreenshot(
+      sm,
+      { session_id: "aa11", ref: "@e5", tab_id: 7 },
+      makeScreenshotDeps({
+        cdp,
+        get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
+        query: vi.fn(),
+        captureVisibleTab: vi.fn(),
+      }),
+    );
+
+    if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
+    expect(res).toMatchObject({ width: 1_998, height: 1_998 });
+    const calls = sent.filter((call) => call.method === "Page.captureScreenshot");
+    expect(calls).toHaveLength(2);
+    expect((calls[1]?.params as { clip?: { scale?: number } }).clip?.scale).toBeLessThan(0.5);
+  });
+
   it("crops a visual surface screenshot to its observation-time visible region", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     const ctx = await sm.start("aa11");
@@ -462,7 +506,11 @@ describe("handleScreenshot", () => {
   it("fails closed when a ref capture is not a valid PNG", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     const ctx = await sm.start("aa11");
-    ctx.refStore.set("e5", 999, { tabId: 7, kind: "surface" });
+    ctx.refStore.set("e5", 999, {
+      tabId: 7,
+      kind: "surface",
+      visibleRect: { x: 0, y: 0, w: 50, h: 50 },
+    });
     const { cdp } = makeFakeCdp({
       "DOM.getContentQuads": () => ({ quads: [[0, 0, 50, 0, 50, 50, 0, 50]] }),
       "Page.captureScreenshot": () => ({ data: "bm90LWEtcG5n" }),

apps/extension/src/tools/__tests__/visual-screenshot-plan.test.ts

@@ -0,0 +1,37 @@
+import { describe, expect, it } from "vitest";
+import {
+  correctedVisualScreenshotScale,
+  planVisualScreenshot,
+  screenshotFitsPixelBudget,
+} from "../visual-screenshot-plan";
+
+const budget = { maxEdge: 2_048, maxPixels: 4_000_000 };
+
+describe("visual screenshot pixel planning", () => {
+  it.each([1, 1.25, 1.5, 2, 3])("accounts for raster scale %s", (rasterScaleEstimate) => {
+    const plan = planVisualScreenshot({
+      cssRect: { width: 2_000, height: 1_000 },
+      rasterScaleEstimate,
+      budget,
+    });
+    const width = 2_000 * rasterScaleEstimate * plan.clipScale;
+    const height = 1_000 * rasterScaleEstimate * plan.clipScale;
+
+    expect(Math.max(width, height)).toBeLessThanOrEqual(budget.maxEdge);
+    expect(width * height).toBeLessThanOrEqual(budget.maxPixels);
+  });
+
+  it("corrects a scale from actual PNG dimensions", () => {
+    const corrected = correctedVisualScreenshotScale({
+      previousClipScale: 1,
+      actualWidth: 4_000,
+      actualHeight: 2_000,
+      budget,
+    });
+
+    expect(corrected).not.toBeNull();
+    expect(corrected as number).toBeLessThan(2_048 / 4_000);
+    expect(screenshotFitsPixelBudget({ width: 2_000, height: 1_000 }, budget)).toBe(true);
+    expect(screenshotFitsPixelBudget({ width: 4_000, height: 2_000 }, budget)).toBe(false);
+  });
+});

apps/extension/src/tools/frame-geometry.ts

@@ -1,4 +1,9 @@
-import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph";
+import {
+  type CdpFrame,
+  type CdpFrameGraph,
+  type CdpTarget,
+  cdpTargetKey,
+} from "@/browser-driver/frame-graph";
 import type { RpcError } from "@/transport/types";
 import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry";
 import {
@@ -10,6 +15,7 @@ import {
   parseCdpQuad,
   polygonArea,
   polygonCentroid,
+  projectRectToViewport,
   projectRegionToViewport,
   type Quad,
   type Region,
@@ -84,6 +90,41 @@ async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise<Size |
   return width > 0 && height > 0 ? { width, height } : null;
 }
 
+async function sameTargetFrameViewport(cdp: CdpRunner, frame: CdpFrame): Promise<Size | null> {
+  const world = await sendToCdpTarget<{ executionContextId?: number }>(
+    cdp,
+    frame.target,
+    "Page.createIsolatedWorld",
+    {
+      frameId: frame.frameId,
+      worldName: "bsk-frame-geometry",
+      grantUniveralAccess: false,
+    },
+  );
+  if (world.executionContextId === undefined) return null;
+  const result = await sendToCdpTarget<{ result?: { value?: unknown } }>(
+    cdp,
+    frame.target,
+    "Runtime.evaluate",
+    {
+      expression: "({width:window.innerWidth,height:window.innerHeight})",
+      contextId: world.executionContextId,
+      returnByValue: true,
+    },
+  );
+  const value = result.result?.value as { width?: unknown; height?: unknown } | undefined;
+  const width = value?.width;
+  const height = value?.height;
+  return typeof width === "number" &&
+    Number.isFinite(width) &&
+    width > 0 &&
+    typeof height === "number" &&
+    Number.isFinite(height) &&
+    height > 0
+    ? { width, height }
+    : null;
+}
+
 async function ownerContentQuad(
   cdp: CdpRunner,
   parent: CdpFrame,
@@ -98,26 +139,150 @@ async function ownerContentQuad(
   return parseCdpQuad(result.model?.content);
 }
 
-async function sameTargetFrameClips(
+export interface FrameGeometryContext {
+  resolveFrameProjection(frameId: string): Promise<GeometryProjection | null>;
+  projectFrameLocalRect(
+    frameId: string,
+    rect: { x: number; y: number; w: number; h: number },
+  ): Promise<ViewportRect | null>;
+}
+
+export function createFrameGeometryContext(
   cdp: CdpRunner,
-  byId: Map<string, CdpFrame>,
+  graph: CdpFrameGraph,
-  frame: CdpFrame,
+): FrameGeometryContext {
-  targetRoot: CdpFrame,
+  const byId = frameMap(graph);
-): Promise<Polygon[] | null> {
+  const targetViewports = new Map<string, Promise<Size | null>>();
-  const clips: Polygon[] = [];
+  const frameViewports = new Map<string, Promise<Size | null>>();
-  let current = frame;
+  const ownerQuads = new Map<string, Promise<Quad | null>>();
-  const seen = new Set<string>();
+  const projections = new Map<string, Promise<GeometryProjection | null>>();
-  while (current.frameId !== targetRoot.frameId) {
+  const localProjections = new Map<string, Promise<GeometryProjection | null>>();
-    if (seen.has(current.frameId) || !current.parentFrameId) return null;
+
-    seen.add(current.frameId);
+  const cachedTargetViewport = (target: CdpTarget): Promise<Size | null> => {
-    const parent = byId.get(current.parentFrameId);
+    const key = cdpTargetKey(target);
-    if (!parent || current.ownerBackendNodeId === undefined) return null;
+    let value = targetViewports.get(key);
-    const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId);
+    if (!value) {
-    if (!clip) return null;
+      value = targetViewport(cdp, target);
-    clips.push(clip);
+      targetViewports.set(key, value);
-    current = parent;
+    }
-  }
+    return value;
-  return clips;
+  };
+  const cachedFrameViewport = (frame: CdpFrame): Promise<Size | null> => {
+    let value = frameViewports.get(frame.frameId);
+    if (!value) {
+      const root = targetRootFrame(byId, frame);
+      value =
+        root?.frameId === frame.frameId
+          ? cachedTargetViewport(frame.target)
+          : sameTargetFrameViewport(cdp, frame);
+      frameViewports.set(frame.frameId, value);
+    }
+    return value;
+  };
+  const cachedOwnerQuad = (parent: CdpFrame, backendNodeId: number): Promise<Quad | null> => {
+    const key = `${cdpTargetKey(parent.target)}:${backendNodeId}`;
+    let value = ownerQuads.get(key);
+    if (!value) {
+      value = ownerContentQuad(cdp, parent, backendNodeId);
+      ownerQuads.set(key, value);
+    }
+    return value;
+  };
+  const cachedSameTargetClips = async (
+    frame: CdpFrame,
+    root: CdpFrame,
+  ): Promise<Polygon[] | null> => {
+    const clips: Polygon[] = [];
+    let current = frame;
+    const seen = new Set<string>();
+    while (current.frameId !== root.frameId) {
+      if (seen.has(current.frameId) || !current.parentFrameId) return null;
+      seen.add(current.frameId);
+      const parent = byId.get(current.parentFrameId);
+      if (!parent || current.ownerBackendNodeId === undefined) return null;
+      const clip = await cachedOwnerQuad(parent, current.ownerBackendNodeId);
+      if (!clip) return null;
+      clips.push(clip);
+      current = parent;
+    }
+    return clips;
+  };
+
+  const buildTargetProjection = async (frameId: string): Promise<GeometryProjection | null> => {
+    const frame = byId.get(frameId);
+    if (!frame) return null;
+    let root = targetRootFrame(byId, frame);
+    if (!root) return null;
+    const sourceViewport = await cachedTargetViewport(root.target);
+    if (!sourceViewport) return null;
+    const sourceClips = await cachedSameTargetClips(frame, root);
+    if (!sourceClips) return null;
+    const edges: ProjectiveEdge[] = [];
+    while (root.parentFrameId) {
+      const parent = byId.get(root.parentFrameId);
+      if (!parent || root.ownerBackendNodeId === undefined) return null;
+      const destinationQuad = await cachedOwnerQuad(parent, root.ownerBackendNodeId);
+      if (!destinationQuad) return null;
+      const source = await cachedTargetViewport(root.target);
+      if (!source) return null;
+      const parentRoot = targetRootFrame(byId, parent);
+      if (!parentRoot) return null;
+      const destinationClips = await cachedSameTargetClips(parent, parentRoot);
+      if (!destinationClips) return null;
+      edges.push({ sourceViewport: source, destinationQuad, destinationClips });
+      root = parentRoot;
+    }
+    const topViewport = await cachedTargetViewport(root.target);
+    return topViewport ? { sourceClips, edges, topViewport } : null;
+  };
+
+  const resolveFrameProjectionCached = (frameId: string): Promise<GeometryProjection | null> => {
+    let value = projections.get(frameId);
+    if (!value) {
+      value = buildTargetProjection(frameId);
+      projections.set(frameId, value);
+    }
+    return value;
+  };
+
+  const buildLocalProjection = async (frameId: string): Promise<GeometryProjection | null> => {
+    const frame = byId.get(frameId);
+    if (!frame) return null;
+    const root = targetRootFrame(byId, frame);
+    if (!root) return null;
+    const targetProjection = await resolveFrameProjectionCached(root.frameId);
+    if (!targetProjection) return null;
+    if (root.frameId === frame.frameId) return targetProjection;
+    if (!frame.parentFrameId || frame.ownerBackendNodeId === undefined) return null;
+    const parent = byId.get(frame.parentFrameId);
+    if (!parent) return null;
+    const sourceViewport = await cachedFrameViewport(frame);
+    const destinationQuad = await cachedOwnerQuad(parent, frame.ownerBackendNodeId);
+    const destinationClips = await cachedSameTargetClips(frame, root);
+    if (!sourceViewport || !destinationQuad || !destinationClips) return null;
+    return {
+      sourceClips: [],
+      edges: [{ sourceViewport, destinationQuad, destinationClips }, ...targetProjection.edges],
+      topViewport: targetProjection.topViewport,
+    };
+  };
+
+  const localProjection = (frameId: string): Promise<GeometryProjection | null> => {
+    let value = localProjections.get(frameId);
+    if (!value) {
+      value = buildLocalProjection(frameId);
+      localProjections.set(frameId, value);
+    }
+    return value;
+  };
+
+  return {
+    resolveFrameProjection: resolveFrameProjectionCached,
+    async projectFrameLocalRect(frameId, rect) {
+      const projection = await localProjection(frameId);
+      return projection ? projectRectToViewport(rect, projection) : null;
+    },
+  };
 }
 
 export async function resolveFrameProjection(
@@ -125,36 +290,7 @@ export async function resolveFrameProjection(
   graph: CdpFrameGraph,
   frameId: string,
 ): Promise<GeometryProjection | null> {
-  const byId = frameMap(graph);
+  return createFrameGeometryContext(cdp, graph).resolveFrameProjection(frameId);
-  const frame = byId.get(frameId);
-  if (!frame) return null;
-  let targetRoot = targetRootFrame(byId, frame);
-  if (!targetRoot) return null;
-  const sourceViewport = await targetViewport(cdp, targetRoot.target);
-  if (!sourceViewport) return null;
-  const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot);
-  if (!sourceClips) return null;
-
-  const edges: ProjectiveEdge[] = [];
-  while (targetRoot.parentFrameId) {
-    const parent = byId.get(targetRoot.parentFrameId);
-    if (!parent || targetRoot.ownerBackendNodeId === undefined) return null;
-    const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId);
-    if (!destinationQuad) return null;
-    const source =
-      edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
-    if (!source) return null;
-    const parentTargetRoot = targetRootFrame(byId, parent);
-    if (!parentTargetRoot) return null;
-    const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot);
-    if (!destinationClips) return null;
-    edges.push({ sourceViewport: source, destinationQuad, destinationClips });
-    targetRoot = parentTargetRoot;
-  }
-
-  const topViewport =
-    edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
-  return topViewport ? { sourceClips, edges, topViewport } : null;
 }
 
 async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {

apps/extension/src/tools/geometry.ts

@@ -245,22 +245,3 @@ export function projectRectToViewport(
   if (!rect) return null;
   return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection));
 }
-
-export function childFrameProjection(
-  parent: GeometryProjection,
-  ownerRectInParent: { x: number; y: number; w: number; h: number },
-): GeometryProjection {
-  const destinationQuad = rectPolygon(ownerRectInParent);
-  return {
-    sourceClips: [],
-    edges: [
-      {
-        sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
-        destinationQuad,
-        destinationClips: parent.sourceClips,
-      },
-      ...parent.edges,
-    ],
-    topViewport: parent.topViewport,
-  };
-}

apps/extension/src/tools/observation.ts

@@ -48,6 +48,13 @@ import {
   type ToolEffect,
 } from "./shared";
 import { resolveSnapshotRef } from "./snapshot-ref";
+import { topLevelScreenshotRasterScale } from "./viewport-metrics";
+import {
+  correctedVisualScreenshotScale,
+  planVisualScreenshot,
+  type ScreenshotPixelBudget,
+  screenshotFitsPixelBudget,
+} from "./visual-screenshot-plan";
 import {
   type CapturedNode,
   type CapturedSurfaceProbe,
@@ -101,6 +108,10 @@ export const normaliseRef = sharedNormaliseRef;
 
 const VISUAL_SURFACE_SCREENSHOT_MAX_EDGE = 2_048;
 const VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS = 4_000_000;
+const VISUAL_SURFACE_SCREENSHOT_BUDGET: ScreenshotPixelBudget = {
+  maxEdge: VISUAL_SURFACE_SCREENSHOT_MAX_EDGE,
+  maxPixels: VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS,
+};
 
 /**
  * Strip the `data:image/...;base64,` prefix from a Chrome
@@ -225,36 +236,52 @@ async function captureElementScreenshot(
         })()
       : liveRect;
   if (!rect) return { code: "permission_denied", message: "surface is no longer visible" };
-  const scale = visualSurface
+  const rasterScaleEstimate = visualSurface ? await topLevelScreenshotRasterScale(cdp, tabId) : 1;
-    ? Math.min(
+  let scale = visualSurface
-        1,
+    ? planVisualScreenshot({
-        VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.width,
+        cssRect: rect,
-        VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.height,
+        rasterScaleEstimate,
-        Math.sqrt(VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS / (rect.width * rect.height)),
+        budget: VISUAL_SURFACE_SCREENSHOT_BUDGET,
-      )
+      }).clipScale
     : 1;
 
   try {
-    const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
+    for (let attempt = 0; attempt < (visualSurface ? 2 : 1); attempt += 1) {
-      format: "png",
+      const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
-      clip: {
+        format: "png",
-        x: rect.x,
+        clip: {
-        y: rect.y,
+          x: rect.x,
-        width: rect.width,
+          y: rect.y,
-        height: rect.height,
+          width: rect.width,
-        scale,
+          height: rect.height,
-      },
+          scale,
-    });
+        },
-    if (signal?.aborted) return cancelled("screenshot");
+      });
-    const image_base64 = shot.data ?? "";
+      if (signal?.aborted) return cancelled("screenshot");
-    if (!image_base64) {
+      const image_base64 = shot.data ?? "";
-      return { code: "cdp_failed", message: "Page.captureScreenshot returned no data" };
+      if (!image_base64) {
+        return { code: "cdp_failed", message: "Page.captureScreenshot returned no data" };
+      }
+      const dims = parsePngDimensions(image_base64);
+      if (!dims) {
+        return { code: "cdp_failed", message: "Page.captureScreenshot returned invalid PNG data" };
+      }
+      if (!visualSurface || screenshotFitsPixelBudget(dims, VISUAL_SURFACE_SCREENSHOT_BUDGET)) {
+        return { image_base64, width: dims.width, height: dims.height };
+      }
+      const corrected = correctedVisualScreenshotScale({
+        previousClipScale: scale,
+        actualWidth: dims.width,
+        actualHeight: dims.height,
+        budget: VISUAL_SURFACE_SCREENSHOT_BUDGET,
+      });
+      if (attempt === 1 || corrected === null) break;
+      scale = corrected;
     }
-    const dims = parsePngDimensions(image_base64);
+    return {
-    if (!dims) {
+      code: "cdp_failed",
-      return { code: "cdp_failed", message: "Page.captureScreenshot returned invalid PNG data" };
+      message: "Page.captureScreenshot exceeded the visual surface pixel budget",
-    }
+    };
-    return { image_base64, width: dims.width, height: dims.height };
   } catch (err) {
     return {
       code: "cdp_failed",
@@ -351,6 +378,12 @@ export async function handleScreenshot(
     }
     const node = resolveSnapshotRef(ctx, ref, target.tabId, "screenshot");
     if (isRpcError(node)) return node;
+    if (node.kind === "surface" && !node.visibleRect) {
+      return {
+        code: "permission_denied",
+        message: `surface ref ${ref} has no observation-time visible bounds`,
+      };
+    }
     if (signal?.aborted) return cancelled("screenshot");
     deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
     await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);

apps/extension/src/tools/viewport-metrics.ts

@@ -0,0 +1,18 @@
+import type { CdpRunner } from "./shared";
+
+/** Estimate the raster/CSS ratio used by top-level Page.captureScreenshot. */
+export async function topLevelScreenshotRasterScale(
+  cdp: CdpRunner,
+  tabId: number,
+): Promise<number> {
+  try {
+    const reply = await cdp.send<{ result?: { value?: unknown } }>(tabId, "Runtime.evaluate", {
+      expression: "window.devicePixelRatio",
+      returnByValue: true,
+    });
+    const value = reply.result?.value;
+    return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 1;
+  } catch {
+    return 1;
+  }
+}

apps/extension/src/tools/visual-screenshot-plan.ts

@@ -0,0 +1,52 @@
+export interface ScreenshotPixelBudget {
+  maxEdge: number;
+  maxPixels: number;
+}
+
+export interface VisualScreenshotPlan {
+  clipScale: number;
+}
+
+export function planVisualScreenshot(input: {
+  cssRect: { width: number; height: number };
+  rasterScaleEstimate: number;
+  budget: ScreenshotPixelBudget;
+}): VisualScreenshotPlan {
+  const rasterScale =
+    Number.isFinite(input.rasterScaleEstimate) && input.rasterScaleEstimate > 0
+      ? input.rasterScaleEstimate
+      : 1;
+  const { width, height } = input.cssRect;
+  const scale = Math.min(
+    1,
+    input.budget.maxEdge / (width * rasterScale),
+    input.budget.maxEdge / (height * rasterScale),
+    Math.sqrt(input.budget.maxPixels / (width * height * rasterScale * rasterScale)),
+  );
+  return { clipScale: Math.max(Number.EPSILON, scale) };
+}
+
+export function correctedVisualScreenshotScale(input: {
+  previousClipScale: number;
+  actualWidth: number;
+  actualHeight: number;
+  budget: ScreenshotPixelBudget;
+}): number | null {
+  const correction = Math.min(
+    input.budget.maxEdge / input.actualWidth,
+    input.budget.maxEdge / input.actualHeight,
+    Math.sqrt(input.budget.maxPixels / (input.actualWidth * input.actualHeight)),
+  );
+  if (!Number.isFinite(correction) || correction >= 1 || correction <= 0) return null;
+  return Math.max(Number.EPSILON, input.previousClipScale * correction * 0.999);
+}
+
+export function screenshotFitsPixelBudget(
+  dimensions: { width: number; height: number },
+  budget: ScreenshotPixelBudget,
+): boolean {
+  return (
+    Math.max(dimensions.width, dimensions.height) <= budget.maxEdge &&
+    dimensions.width * dimensions.height <= budget.maxPixels
+  );
+}

apps/extension/src/tools/vom/__tests__/capture.test.ts

@@ -1011,7 +1011,7 @@ describe("captureViewModel", () => {
     expect(input?.attrs.type).toBe("text");
     expect(input?.ownerFrameBackendNodeId).toBe(13);
     expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
-    expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
+    expect(input?.rect).toBeNull();
     expect(rootFrameId).toBe("main-frame");
     expect(frameNodes?.get("child-frame")).toBe(subNodes);
     expect(frameOwnerBackendNodeIds?.get("child-frame")).toBe(13);
@@ -1183,13 +1183,13 @@ describe("captureViewModel", () => {
     const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31);
 
     expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 });
-    expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
+    expect(nestedIframe?.rect).toBeNull();
     expect(input?.ownerFrameBackendNodeId).toBe(23);
     expect(input?.localRect).toEqual({ x: 1, y: 2, w: 40, h: 20 });
-    expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
+    expect(input?.rect).toBeNull();
   });
 
-  it("keeps same-origin iframe canvas discovery in top-level CSS coordinates at dpr 1 and 2", async () => {
+  it("keeps same-origin iframe canvas frame-local until frame geometry is resolved", async () => {
     const captureAtDpr = async (dpr: number) => {
       const S = ["html", "body", "iframe", "canvas", "static", "auto"];
       const i = (s: string) => S.indexOf(s);
@@ -1270,13 +1270,7 @@ describe("captureViewModel", () => {
     const atDpr1 = await captureAtDpr(1);
     const atDpr2 = await captureAtDpr(2);
 
-    expect(atDpr1).toEqual([
+    expect(atDpr1).toEqual([]);
-      expect.objectContaining({
-        backendNodeId: 21,
-        frameId: "child",
-        visibleRect: { x: 120, y: 180, w: 120, h: 60 },
-      }),
-    ]);
     expect(atDpr2).toEqual(atDpr1);
   });
 

apps/extension/src/tools/vom/__tests__/frame-capture.test.ts

@@ -50,6 +50,75 @@ function childSnapshot(frameId: string, backendNodeId: number) {
 }
 
 describe("captureFrameData", () => {
+  it("projects same-origin frame-local rects through the iframe content box", async () => {
+    const owner = ownerNode(10, 50);
+    const canvas: CapturedNode = {
+      backendNodeId: 101,
+      parentBackendNodeId: 100,
+      frameId: "child",
+      ownerFrameBackendNodeId: 10,
+      tag: "canvas",
+      attrs: {},
+      rect: null,
+      localRect: { x: 12, y: 12, w: 224, h: 94 },
+      paintOrder: 1,
+      position: "static",
+      pointerEvents: "auto",
+    };
+    const captured: CapturedViewModel = {
+      nodes: [owner],
+      viewport: { width: 1000, height: 800 },
+      iframeNodes: new Map([[10, [canvas]]]),
+      frameNodes: new Map([
+        ["main", [owner]],
+        ["child", [canvas]],
+      ]),
+      frameOwnerBackendNodeIds: new Map([["child", 10]]),
+      frameParentIds: new Map([["child", "main"]]),
+      rootFrameId: "main",
+      excludedBackendNodeIds: new Set(),
+    };
+    const cdp: CdpRunner = {
+      send: vi.fn(async (_tabId, method) => {
+        if (method === "Accessibility.enable") return {};
+        if (method === "Accessibility.getFullAXTree") return { nodes: [] };
+        if (method === "Page.getLayoutMetrics") {
+          return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
+        }
+        if (method === "DOM.getBoxModel") {
+          return { model: { content: [52, 102, 352, 102, 352, 302, 52, 302] } };
+        }
+        if (method === "Page.createIsolatedWorld") return { executionContextId: 91 };
+        if (method === "Runtime.evaluate") {
+          return { result: { value: { width: 300, height: 200 } } };
+        }
+        throw new Error(`unexpected ${method}`);
+      }) as CdpRunner["send"],
+      getFrameGraph: vi.fn(async () => ({
+        rootFrameId: "main",
+        frames: [
+          { frameId: "main", target: { tabId: 4 } },
+          {
+            frameId: "child",
+            parentFrameId: "main",
+            ownerBackendNodeId: 10,
+            target: { tabId: 4 },
+          },
+        ],
+      })),
+    };
+
+    await captureFrameData(cdp, 4, captured);
+
+    expect(captured.frameNodes?.get("child")?.[0]?.rect).toEqual({
+      x: 64,
+      y: 114,
+      w: 224,
+      h: 94,
+    });
+    expect(captured.iframeNodes.get(10)?.[0]?.rect).toEqual({ x: 64, y: 114, w: 224, h: 94 });
+  });
+
   it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
     const leftOwner = ownerNode(10, 50);
     const rightOwner = ownerNode(20, 500);

apps/extension/src/tools/vom/__tests__/rendered-surfaces.test.ts

@@ -183,7 +183,7 @@ describe("rendered surface clustering and projection", () => {
       ],
       { width: 1280, height: 720 },
     );
-    const groups = clusterRenderedSurfaces(surfaces);
+    const clustered = clusterRenderedSurfaces(surfaces);
     const scene: VomScene = {
       viewport: { width: 1280, height: 720 },
       nodes: [
@@ -202,14 +202,14 @@ describe("rendered surface clustering and projection", () => {
       ],
     };
 
-    expect(groups).toHaveLength(1);
+    expect(clustered.groups).toHaveLength(1);
-    expect(groups[0]?.members).toHaveLength(2);
+    expect(clustered.groups[0]?.memberCount).toBe(2);
-    expect(groups[0]?.label).toBeUndefined();
+    expect(clustered.groups[0]?.label).toBeUndefined();
     expect(
       projectRenderedSurfaces(
         scene,
         [document([body, ...surfaces.map((s) => domNode(s.backendNodeId))])],
-        groups,
+        clustered,
       ),
     ).toMatchObject({
       visualSurfaces: [expect.objectContaining({ parentId: 10, backendNodeId: 3, memberCount: 2 })],
@@ -217,7 +217,7 @@ describe("rendered surface clustering and projection", () => {
   });
 
   it("does not merge a contained canvas with a larger independent canvas", () => {
-    const groups = clusterRenderedSurfaces([
+    const clustered = clusterRenderedSurfaces([
       ...discoverRenderedSurfaces(
         [
           document([
@@ -233,11 +233,34 @@ describe("rendered surface clustering and projection", () => {
       ),
     ]);
 
-    expect(groups).toHaveLength(2);
+    expect(clustered.groups).toHaveLength(2);
+  });
+
+  it("keeps the existing inclusive 90 percent stack threshold", () => {
+    const clustered = clusterRenderedSurfaces([
+      {
+        renderingKind: "canvas",
+        frameId: "main",
+        backendNodeId: 2,
+        parentBackendNodeId: 1,
+        visibleRect: { x: 0, y: 0, w: 100, h: 100 },
+        paintOrder: 1,
+      },
+      {
+        renderingKind: "canvas",
+        frameId: "main",
+        backendNodeId: 3,
+        parentBackendNodeId: 1,
+        visibleRect: { x: 5, y: 0, w: 90, h: 100 },
+        paintOrder: 2,
+      },
+    ]);
+
+    expect(clustered.groups).toEqual([expect.objectContaining({ memberCount: 2 })]);
   });
 
   it("does not merge coincident canvases owned by different containers", () => {
-    const groups = clusterRenderedSurfaces(
+    const clustered = clusterRenderedSurfaces(
       discoverRenderedSurfaces(
         [
           document([
@@ -251,6 +274,43 @@ describe("rendered surface clustering and projection", () => {
       ),
     );
 
-    expect(groups).toHaveLength(2);
+    expect(clustered.groups).toHaveLength(2);
+  });
+
+  it("keeps adjacent map tiles and twenty sparklines as independent surfaces", () => {
+    const surfaces = Array.from({ length: 20 }, (_, index) =>
+      domNode(100 + index, {
+        parentBackendNodeId: 1,
+        rect: { x: index * 12, y: 20, w: 10, h: 8 },
+      }),
+    );
+    const clustered = clusterRenderedSurfaces(
+      discoverRenderedSurfaces(
+        [document([domNode(1, { tag: "main", rect: null, localRect: null }), ...surfaces])],
+        { width: 1280, height: 720 },
+      ),
+    );
+
+    expect(clustered.groups).toHaveLength(20);
+    expect(clustered.groups.every((group) => group.memberCount === 1)).toBe(true);
+    expect(clustered.truncated).toBe(false);
+  });
+
+  it("clusters a large coincident stack without retaining every member", () => {
+    const surfaces = Array.from({ length: 10_000 }, (_, index) => ({
+      renderingKind: "canvas" as const,
+      frameId: "main",
+      backendNodeId: index + 1,
+      parentBackendNodeId: 1,
+      visibleRect: { x: 10, y: 20, w: 800, h: 500 },
+      paintOrder: index,
+    }));
+
+    const clustered = clusterRenderedSurfaces(surfaces);
+
+    expect(clustered.groups).toEqual([
+      expect.objectContaining({ memberCount: 10_000, representative: surfaces[9_999] }),
+    ]);
+    expect(clustered.truncated).toBe(false);
   });
 });

apps/extension/src/tools/vom/capture.ts

@@ -7,7 +7,7 @@
 import type { Rect, Viewport } from "@browser-skill/vom";
 import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
 import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
-import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
+import { type GeometryProjection, projectRectToViewport } from "../geometry";
 import type { CdpRunner } from "../shared";
 import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
 
@@ -1014,7 +1014,6 @@ function parseChildFrameDocuments(
   strings: string[],
   dpr: number,
   parentDocIndex: number,
-  parentNodes: CapturedNode[],
   parentContext: FrameContext,
   visited = new Set<number>(),
 ): ParseFrameDocumentsResult {
@@ -1029,19 +1028,17 @@ function parseChildFrameDocuments(
   }
 
   const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
-  const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
-
   for (const [nodeArrayIdx, childDocIndex] of cdi) {
     if (visited.has(childDocIndex)) continue;
     const childDoc = documents[childDocIndex];
     if (!childDoc) continue;
     const iframeBackendId = parentBackendIds[nodeArrayIdx];
     if (iframeBackendId === undefined) continue;
-    const iframeNode = parentNodeByBackendId.get(iframeBackendId);
+    // DOMSnapshot reports the iframe owner's border-box bounds, but child
-    const projection =
+    // document coordinates start at the content-box origin. Preserve the
-      parentContext.projection && iframeNode?.localRect
+    // frame-local rectangles here and let frame-capture normalize them with
-        ? childFrameProjection(parentContext.projection, iframeNode.localRect)
+    // authoritative content quads once the frame graph is available.
-        : null;
+    const projection = null;
 
     const childContext: FrameContext = {
       frameId: snapshotFrameId(childDoc, strings),
@@ -1065,7 +1062,6 @@ function parseChildFrameDocuments(
       strings,
       dpr,
       childDocIndex,
-      parsed.nodes,
       childContext,
       nextVisited,
     );
@@ -1143,7 +1139,7 @@ export async function captureViewModel(
   const nodes = mainParsed.nodes;
   const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
 
-  const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext);
+  const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, topContext);
   const iframeNodes = frameParsed.iframeNodes;
   for (const id of frameParsed.excludedBackendNodeIds) {
     excludedBackendNodeIds.add(id);

apps/extension/src/tools/vom/frame-capture.ts

@@ -1,6 +1,5 @@
 import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
-import { resolveFrameProjection } from "../frame-geometry";
+import { createFrameGeometryContext } from "../frame-geometry";
-import { type GeometryProjection, projectRectToViewport } from "../geometry";
 import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
 import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
 import {
@@ -31,21 +30,6 @@ async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFra
   }
 }
 
-function transformFrameNodes(
-  nodes: CapturedNode[],
-  projection: GeometryProjection | null,
-): CapturedNode[] {
-  return nodes.map((node) => ({
-    ...node,
-    rect: projection
-      ? (() => {
-          const bounds = projectRectToViewport(node.rect, projection);
-          return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
-        })()
-      : null,
-  }));
-}
-
 async function captureMissingFrameDocuments(
   cdp: CdpRunner,
   tabId: number,
@@ -63,16 +47,6 @@ async function captureMissingFrameDocuments(
       const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
         signal,
       });
-      let projection: GeometryProjection | null = null;
-      try {
-        projection = await resolveFrameProjection(cdp, graph, frame.frameId);
-      } catch (err) {
-        if (isAbortError(err)) throw err;
-        console.debug("[bsk observation] child frame geometry projection failed", {
-          frameId: frame.frameId,
-          err,
-        });
-      }
       const rootFrameId = child.rootFrameId ?? frame.frameId;
       const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
       if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
@@ -83,7 +57,7 @@ async function captureMissingFrameDocuments(
         );
       }
       for (const [childFrameId, nodes] of childFrames) {
-        captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
+        captured.frameNodes.set(childFrameId, nodes);
       }
       if (frame.ownerBackendNodeId !== undefined) {
         captured.iframeNodes.set(
@@ -109,6 +83,56 @@ async function captureMissingFrameDocuments(
   }
 }
 
+async function normalizeFrameLocalRects(
+  cdp: CdpRunner,
+  graph: CdpFrameGraph,
+  captured: CapturedViewModel,
+  signal?: AbortSignal,
+): Promise<void> {
+  if (!captured.frameNodes) return;
+  const geometry = createFrameGeometryContext(cdp, graph);
+  const frameById = new Map(graph.frames.map((frame) => [frame.frameId, frame]));
+  await Promise.all(
+    [...captured.frameNodes].map(async ([frameId, nodes]) => {
+      if (frameId === graph.rootFrameId || !frameById.has(frameId)) return;
+      try {
+        const normalized = await Promise.all(
+          nodes.map(async (node) => {
+            if (!node.localRect) return { ...node, rect: null };
+            const bounds = await geometry.projectFrameLocalRect(frameId, node.localRect);
+            return {
+              ...node,
+              rect: bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null,
+            };
+          }),
+        );
+        throwIfAborted(signal);
+        captured.frameNodes?.set(frameId, normalized);
+        const ownerBackendNodeId =
+          frameById.get(frameId)?.ownerBackendNodeId ??
+          captured.frameOwnerBackendNodeIds?.get(frameId);
+        if (ownerBackendNodeId !== undefined) {
+          captured.iframeNodes.set(ownerBackendNodeId, normalized);
+        }
+      } catch (err) {
+        if (isAbortError(err)) throw err;
+        console.debug("[bsk observation] frame-local geometry normalization failed", {
+          frameId,
+          err,
+        });
+        const unavailable = nodes.map((node) => ({ ...node, rect: null }));
+        captured.frameNodes?.set(frameId, unavailable);
+        const ownerBackendNodeId =
+          frameById.get(frameId)?.ownerBackendNodeId ??
+          captured.frameOwnerBackendNodeIds?.get(frameId);
+        if (ownerBackendNodeId !== undefined) {
+          captured.iframeNodes.set(ownerBackendNodeId, unavailable);
+        }
+      }
+    }),
+  );
+}
+
 async function captureAxTrees<T extends FrameAxNode>(
   cdp: CdpRunner,
   tabId: number,
@@ -174,7 +198,10 @@ export async function captureFrameData<T extends FrameAxNode>(
 ): Promise<CapturedFrameDocument<T>[]> {
   const graph = await discoverFrameGraph(cdp, tabId);
   const frames = graph?.frames ?? [];
-  if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal);
+  if (graph) {
+    await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal);
+    await normalizeFrameLocalRects(cdp, graph, captured, signal);
+  }
   throwIfAborted(signal);
   const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
   return buildFrameDocuments(graph, batches, captured);

apps/extension/src/tools/vom/rendered-surfaces/cluster.ts

@@ -1,8 +1,17 @@
 import type { Rect } from "@browser-skill/vom";
-import type { RenderedSurface, RenderedSurfaceGroup } from "./types";
+import type { ClusteredRenderedSurfaces, RenderedSurface, RenderedSurfaceGroup } from "./types";
 
 const STACK_IOU_RATIO = 0.9;
 const STACK_SIZE_RATIO = 0.9;
+const SIZE_BAND_BASE = 1 / STACK_SIZE_RATIO;
+const POSITION_CELL_RATIO = 0.1;
+const MAX_CLUSTER_COMPARISONS = 1_000_000;
+const MAX_CLUSTER_SURFACES = 50_000;
+
+interface IndexedGroup extends RenderedSurfaceGroup {
+  creationIndex: number;
+  indexKey: string;
+}
 
 function area(rect: Rect): number {
   return Math.max(0, rect.w) * Math.max(0, rect.h);
@@ -14,9 +23,7 @@ function intersectionArea(a: Rect, b: Rect): number {
   return width * height;
 }
 
-function sameVisualStack(a: RenderedSurface, b: RenderedSurface): boolean {
+function stackScore(a: RenderedSurface, b: RenderedSurface): number | null {
-  if (a.frameId !== b.frameId) return false;
-  if (a.parentBackendNodeId !== b.parentBackendNodeId) return false;
   const aArea = area(a.visibleRect);
   const bArea = area(b.visibleRect);
   const intersection = intersectionArea(a.visibleRect, b.visibleRect);
@@ -25,41 +32,158 @@ function sameVisualStack(a: RenderedSurface, b: RenderedSurface): boolean {
     Math.min(a.visibleRect.w, b.visibleRect.w) / Math.max(a.visibleRect.w, b.visibleRect.w);
   const heightRatio =
     Math.min(a.visibleRect.h, b.visibleRect.h) / Math.max(a.visibleRect.h, b.visibleRect.h);
-  return (
+  const iou = union > 0 ? intersection / union : 0;
-    union > 0 &&
+  return iou >= STACK_IOU_RATIO && widthRatio >= STACK_SIZE_RATIO && heightRatio >= STACK_SIZE_RATIO
-    intersection / union >= STACK_IOU_RATIO &&
+    ? iou
-    widthRatio >= STACK_SIZE_RATIO &&
+    : null;
-    heightRatio >= STACK_SIZE_RATIO
-  );
 }
 
-function toGroup(members: RenderedSurface[]): RenderedSurfaceGroup {
+function sizeBand(value: number): number {
-  const representative = [...members].sort(
+  return Math.floor(Math.log(Math.max(Number.EPSILON, value)) / Math.log(SIZE_BAND_BASE));
-    (a, b) => area(b.visibleRect) - area(a.visibleRect) || b.paintOrder - a.paintOrder,
+}
-  )[0];
+
+function bandSize(band: number): number {
+  return SIZE_BAND_BASE ** band;
+}
+
+function spatialKey(rect: Rect): string {
+  const widthBand = sizeBand(rect.w);
+  const heightBand = sizeBand(rect.h);
+  const cellWidth = bandSize(widthBand) * POSITION_CELL_RATIO;
+  const cellHeight = bandSize(heightBand) * POSITION_CELL_RATIO;
+  const centerX = rect.x + rect.w / 2;
+  const centerY = rect.y + rect.h / 2;
+  return `${widthBand}:${heightBand}:${Math.floor(centerX / cellWidth)}:${Math.floor(centerY / cellHeight)}`;
+}
+
+function candidateKeys(rect: Rect): string[] {
+  const keys: string[] = [];
+  const centerX = rect.x + rect.w / 2;
+  const centerY = rect.y + rect.h / 2;
+  const widthBand = sizeBand(rect.w);
+  const heightBand = sizeBand(rect.h);
+  // A >=0.9 size ratio crosses at most one logarithmic band. IoU >=0.9
+  // likewise keeps centers within one 10%-of-size position cell.
+  for (let dw = -1; dw <= 1; dw += 1) {
+    for (let dh = -1; dh <= 1; dh += 1) {
+      const candidateWidthBand = widthBand + dw;
+      const candidateHeightBand = heightBand + dh;
+      const cellWidth = bandSize(candidateWidthBand) * POSITION_CELL_RATIO;
+      const cellHeight = bandSize(candidateHeightBand) * POSITION_CELL_RATIO;
+      const cellX = Math.floor(centerX / cellWidth);
+      const cellY = Math.floor(centerY / cellHeight);
+      for (let dx = -1; dx <= 1; dx += 1) {
+        for (let dy = -1; dy <= 1; dy += 1) {
+          keys.push(`${candidateWidthBand}:${candidateHeightBand}:${cellX + dx}:${cellY + dy}`);
+        }
+      }
+    }
+  }
+  return keys;
+}
+
+function preferredRepresentative(a: RenderedSurface, b: RenderedSurface): RenderedSurface {
+  const aArea = area(a.visibleRect);
+  const bArea = area(b.visibleRect);
+  return bArea > aArea || (bArea === aArea && b.paintOrder > a.paintOrder) ? b : a;
+}
+
+function partitionKey(surface: RenderedSurface): string {
+  return `${surface.frameId}\u0000${surface.parentBackendNodeId ?? "root"}`;
+}
+
+export function clusterRenderedSurfaces(surfaces: RenderedSurface[]): ClusteredRenderedSurfaces {
+  const partitions = new Map<string, Map<string, Set<IndexedGroup>>>();
+  const groups: IndexedGroup[] = [];
+  let comparisons = 0;
+  let processed = 0;
+  let truncated = false;
+
+  for (const surface of surfaces) {
+    if (processed >= MAX_CLUSTER_SURFACES || comparisons >= MAX_CLUSTER_COMPARISONS) {
+      truncated = true;
+      break;
+    }
+    const key = partitionKey(surface);
+    let index = partitions.get(key);
+    if (!index) {
+      index = new Map();
+      partitions.set(key, index);
+    }
+    const candidates = new Set<IndexedGroup>();
+    for (const candidateKey of candidateKeys(surface.visibleRect)) {
+      for (const candidate of index.get(candidateKey) ?? []) candidates.add(candidate);
+    }
+
+    let best: { group: IndexedGroup; score: number; areaDelta: number } | null = null;
+    for (const group of candidates) {
+      comparisons += 1;
+      if (comparisons > MAX_CLUSTER_COMPARISONS) {
+        truncated = true;
+        break;
+      }
+      const score = stackScore(group.representative, surface);
+      if (score === null) continue;
+      const areaDelta = Math.abs(
+        area(group.representative.visibleRect) - area(surface.visibleRect),
+      );
+      if (
+        !best ||
+        score > best.score ||
+        (score === best.score && areaDelta < best.areaDelta) ||
+        (score === best.score &&
+          areaDelta === best.areaDelta &&
+          group.creationIndex < best.group.creationIndex)
+      ) {
+        best = { group, score, areaDelta };
+      }
+    }
+    if (truncated) break;
+
+    if (best) {
+      const group = best.group;
+      const representative = preferredRepresentative(group.representative, surface);
+      group.memberCount += 1;
+      group.label ??= surface.label;
+      if (representative !== group.representative) {
+        index.get(group.indexKey)?.delete(group);
+        group.representative = representative;
+        group.indexKey = spatialKey(representative.visibleRect);
+        const indexed = index.get(group.indexKey) ?? new Set<IndexedGroup>();
+        indexed.add(group);
+        index.set(group.indexKey, indexed);
+      }
+      processed += 1;
+      continue;
+    }
+
+    const indexKey = spatialKey(surface.visibleRect);
+    const group: IndexedGroup = {
+      frameId: surface.frameId,
+      parentBackendNodeId: surface.parentBackendNodeId,
+      representative: surface,
+      ...(surface.label ? { label: surface.label } : {}),
+      memberCount: 1,
+      creationIndex: groups.length,
+      indexKey,
+    };
+    groups.push(group);
+    const indexed = index.get(indexKey) ?? new Set<IndexedGroup>();
+    indexed.add(group);
+    index.set(indexKey, indexed);
+    processed += 1;
+  }
+
   return {
-    frameId: representative.frameId,
+    groups: groups
-    members,
+      .map(({ creationIndex: _creationIndex, indexKey: _indexKey, ...group }) => group)
-    representative,
+      .sort(
-    label: members.map((surface) => surface.label).find((label) => label !== undefined),
+        (a, b) =>
+          a.frameId.localeCompare(b.frameId) ||
+          a.representative.visibleRect.y - b.representative.visibleRect.y ||
+          a.representative.visibleRect.x - b.representative.visibleRect.x,
+      ),
+    truncated,
+    ...(truncated ? { omittedCount: surfaces.length - processed } : {}),
   };
 }
-
-export function clusterRenderedSurfaces(surfaces: RenderedSurface[]): RenderedSurfaceGroup[] {
-  const groups: RenderedSurface[][] = [];
-  for (const surface of surfaces) {
-    const matching = groups.find((group) =>
-      group.some((member) => sameVisualStack(member, surface)),
-    );
-    if (matching) matching.push(surface);
-    else groups.push([surface]);
-  }
-  return groups
-    .map(toGroup)
-    .sort(
-      (a, b) =>
-        a.frameId.localeCompare(b.frameId) ||
-        a.representative.visibleRect.y - b.representative.visibleRect.y ||
-        a.representative.visibleRect.x - b.representative.visibleRect.x,
-    );
-}

apps/extension/src/tools/vom/rendered-surfaces/index.ts

@@ -1,4 +1,8 @@
 export { clusterRenderedSurfaces } from "./cluster";
 export { discoverRenderedSurfaces } from "./discovery";
 export { projectRenderedSurfaces } from "./project";
-export type { RenderedSurface, RenderedSurfaceGroup } from "./types";
+export type {
+  ClusteredRenderedSurfaces,
+  RenderedSurface,
+  RenderedSurfaceGroup,
+} from "./types";

apps/extension/src/tools/vom/rendered-surfaces/project.ts

@@ -1,7 +1,7 @@
 import type { VomScene, VomVisualSurface } from "@browser-skill/vom";
 import type { CapturedFrameDocument } from "../frame-capture";
 import type { SemanticAxNode } from "../semantic-graph";
-import type { RenderedSurfaceGroup } from "./types";
+import type { ClusteredRenderedSurfaces, RenderedSurfaceGroup } from "./types";
 
 function frameBackendKey(frameId: string, backendNodeId: number): string {
   return `${frameId}\u0000${backendNodeId}`;
@@ -75,12 +75,12 @@ function buildProjectionIndex(
 export function projectRenderedSurfaces(
   scene: VomScene,
   documents: CapturedFrameDocument<SemanticAxNode>[],
-  groups: RenderedSurfaceGroup[],
+  clustered: ClusteredRenderedSurfaces,
 ): VomScene {
   const index = buildProjectionIndex(scene, documents);
   const visualSurfaces: VomVisualSurface[] = [];
 
-  for (const group of groups) {
+  for (const group of clustered.groups) {
     const parentId = nearestSceneParent(index, group);
     visualSurfaces.push({
       parentId,
@@ -89,12 +89,20 @@ export function projectRenderedSurfaces(
       renderingKind: "canvas",
       visibleRect: group.representative.visibleRect,
       ...(group.label ? { label: group.label } : {}),
-      memberCount: group.members.length,
+      memberCount: group.memberCount,
     });
   }
 
   return {
     ...scene,
     ...(visualSurfaces.length > 0 ? { visualSurfaces } : {}),
+    ...(clustered.truncated
+      ? {
+          visualSurfacesTruncated: true,
+          ...(clustered.omittedCount !== undefined
+            ? { omittedVisualSurfaceCount: clustered.omittedCount }
+            : {}),
+        }
+      : {}),
   };
 }

apps/extension/src/tools/vom/rendered-surfaces/types.ts

@@ -12,7 +12,14 @@ export interface RenderedSurface {
 
 export interface RenderedSurfaceGroup {
   frameId: string;
-  members: RenderedSurface[];
+  parentBackendNodeId: number | null;
   representative: RenderedSurface;
   label?: string;
+  memberCount: number;
+}
+
+export interface ClusteredRenderedSurfaces {
+  groups: RenderedSurfaceGroup[];
+  truncated: boolean;
+  omittedCount?: number;
 }

apps/extension/test-fixtures/rendered-surfaces/README.md

@@ -95,6 +95,11 @@ Use the normal DOM buttons and re-observe after each action:
 - After **Restore size**: present with a fresh ref.
 - After **Remove canvas**: absent.
 
+### `sparklines.html`
+
+Expected Surface labels `fixture-sparkline-01` through `fixture-sparkline-20` must all be present.
+The adjacent, non-overlapping canvases remain twenty independent screenshot-only Surface refs.
+
 ## Invariants
 
 - `snapshot` must not emit any of these Surface markers.

apps/extension/test-fixtures/rendered-surfaces/sparklines.html

@@ -0,0 +1,28 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Rendered Surface sparkline matrix</title>
+    <style>
+      html, body { margin: 0; font: 14px system-ui, sans-serif; }
+      body { padding: 16px; }
+      main { display: grid; grid-template-columns: repeat(5, 120px); gap: 12px; }
+      canvas { display: block; width: 100px; height: 30px; background: #eaf2f8; }
+    </style>
+  </head>
+  <body>
+    <h1>Twenty independent sparklines</h1>
+    <main id="sparklines"></main>
+    <script>
+      const host = document.querySelector("#sparklines");
+      for (let index = 1; index <= 20; index += 1) {
+        const canvas = document.createElement("canvas");
+        canvas.width = 100;
+        canvas.height = 30;
+        canvas.setAttribute("aria-label", `fixture-sparkline-${String(index).padStart(2, "0")}`);
+        host.append(canvas);
+      }
+    </script>
+  </body>
+</html>

packages/vom/src/__tests__/render.test.ts

@@ -117,7 +117,7 @@ describe("renderVom single-layer page", () => {
     );
   });
 
-  it("bounds visual output and prefers explicitly labeled surfaces", () => {
+  it("uses the appendix token budget and prefers explicitly labeled surfaces", () => {
     const input = scene([node({ id: 1, role: "RootWebArea", frameId: "main" })]);
     input.visualSurfaces = Array.from({ length: 12 }, (_, index) => ({
       parentId: 1,
@@ -132,10 +132,30 @@ describe("renderVom single-layer page", () => {
     const out = renderVom(input);
     const surfaceRefs = out.refs.filter((ref) => ref.kind === "surface");
 
-    expect(surfaceRefs).toHaveLength(8);
+    expect(surfaceRefs).toHaveLength(12);
     expect(surfaceRefs[0]).toMatchObject({ backendNodeId: 111, name: "Important status" });
-    expect(out.text).toContain("… 4 additional visual surfaces omitted");
+    expect(out.truncated).toBe(false);
-    expect(out.truncated).toBe(true);
+  });
+
+  it("renders twenty independent sparklines without changing DOM refs", () => {
+    const input = scene([
+      node({ id: 1, role: "RootWebArea", frameId: "main" }),
+      node({ id: 2, parentId: 1, role: "button", name: "Submit", tag: "button" }),
+    ]);
+    input.visualSurfaces = Array.from({ length: 20 }, (_, index) => ({
+      parentId: 1,
+      backendNodeId: 100 + index,
+      frameId: "main",
+      renderingKind: "canvas" as const,
+      visibleRect: { x: index * 12, y: 20, w: 10, h: 8 },
+      memberCount: 1,
+    }));
+
+    const out = renderVom(input);
+
+    expect(out.refs[0]).toMatchObject({ ref: "e1", backendNodeId: 2, kind: "dom" });
+    expect(out.refs.filter((ref) => ref.kind === "surface")).toHaveLength(20);
+    expect(out.truncated).toBe(false);
   });
 
   it("does not let a visual surface exhaust the budget before later DOM content", () => {

packages/vom/src/render.ts

@@ -88,7 +88,8 @@ const MAX_CUSTOM_RECOVERY_AREA = 180_000;
 const MAX_RECOVERABLE_SINGLE_INTENT_AREA = 360_000;
 const MAX_HANDLE_CONTEXT_ITEMS = 3;
 const MAX_SURFACE_ITEMS = 12;
-const MAX_VISUAL_SURFACES = 8;
+const DEFAULT_VISUAL_APPENDIX_TOKENS = 2_048;
+const MAX_VISUAL_SURFACE_REFS = 512;
 const MAX_SCOPE_LINES = 40;
 const RECOVERY_HANDLER_ATTRS = ["onclick", "onmousedown", "onkeydown", "onkeyup", "onkeypress"];
 const NATIVE_TAGS = new Set(["button", "input", "select", "textarea"]);
@@ -796,6 +797,16 @@ interface RenderState {
   surfaceMap: Map<number, CondSurface>;
   scopeMap: Map<number, ActiveScopeBlock>;
   visualSurfaces: VomVisualSurface[];
+  visualSurfacesTruncated: boolean;
+  omittedVisualSurfaceCount?: number;
+}
+
+function tryAppendLine(state: RenderState, line: string, tokenLimit = state.maxTokens): boolean {
+  const nextTokens = state.tokens + estimateTokens(line);
+  if (nextTokens > tokenLimit) return false;
+  state.lines.push(line);
+  state.tokens = nextTokens;
+  return true;
 }
 
 function visualSurfaceArea(surface: VomVisualSurface): number {
@@ -834,41 +845,35 @@ function selectVisualSurfaces(
     if (surface.parentId !== null && !nodesById.has(surface.parentId)) continue;
     eligibleCount += 1;
 
-    const insertionIndex = selected.findIndex(
+    selected.push(surface);
-      (candidate) => compareVisualSurfaces(surface, candidate) < 0,
-    );
-    if (insertionIndex < 0) {
-      if (selected.length < MAX_VISUAL_SURFACES) selected.push(surface);
-      continue;
-    }
-    selected.splice(insertionIndex, 0, surface);
-    if (selected.length > MAX_VISUAL_SURFACES) selected.pop();
   }
-
+  selected.sort(compareVisualSurfaces);
   return { selected, eligibleCount };
 }
 
 function emitVisualAppendix(state: RenderState): void {
   if (state.stopped) return;
   const { selected, eligibleCount } = selectVisualSurfaces(state.visualSurfaces, state.nodesById);
-  if (eligibleCount === 0) return;
+  if (eligibleCount === 0 && !state.visualSurfacesTruncated) return;
 
   const header = "[visual surfaces]";
+  const appendixStartTokens = state.tokens;
+  const appendixTokenLimit = Number.isFinite(state.maxTokens)
+    ? state.maxTokens
+    : appendixStartTokens + DEFAULT_VISUAL_APPENDIX_TOKENS;
   let headerEmitted = false;
   let emitted = 0;
   for (const surface of selected) {
+    if (emitted >= MAX_VISUAL_SURFACE_REFS) break;
     const ref = `e${state.nextRef}`;
     const line = visualSurfaceLine(surface, ref);
-    const nextTokens =
+    const requiredTokens = estimateTokens(line) + (headerEmitted ? 0 : estimateTokens(header));
-      state.tokens + estimateTokens(line) + (headerEmitted ? 0 : estimateTokens(header));
+    if (state.tokens + requiredTokens > appendixTokenLimit) break;
-    if (nextTokens > state.maxTokens) break;
     if (!headerEmitted) {
-      state.lines.push(header);
+      if (!tryAppendLine(state, header, appendixTokenLimit)) break;
-      state.tokens += estimateTokens(header);
       headerEmitted = true;
     }
-    state.lines.push(line);
+    if (!tryAppendLine(state, line, appendixTokenLimit)) break;
-    state.tokens += estimateTokens(line);
     const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`;
     state.refs.push({
       ref,
@@ -885,16 +890,15 @@ function emitVisualAppendix(state: RenderState): void {
     emitted += 1;
   }
 
-  const omitted = eligibleCount - emitted;
+  const locallyOmitted = eligibleCount - emitted;
-  if (omitted > 0) {
+  const upstreamOmitted = state.omittedVisualSurfaceCount ?? 0;
+  const omitted = locallyOmitted + upstreamOmitted;
+  if (omitted > 0 || state.visualSurfacesTruncated) {
     state.truncated = true;
     if (headerEmitted) {
-      const summary = `  … ${omitted} additional visual surfaces omitted`;
+      const count =
-      const nextTokens = state.tokens + estimateTokens(summary);
+        omitted > 0 ? `${state.visualSurfacesTruncated ? "at least " : ""}${omitted}` : "some";
-      if (nextTokens <= state.maxTokens) {
+      tryAppendLine(state, `  … ${count} additional visual surfaces omitted`, appendixTokenLimit);
-        state.lines.push(summary);
-        state.tokens = nextTokens;
-      }
     }
   }
 }
@@ -903,28 +907,22 @@ function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: Ren
   if (state.stopped || scope.lines.length === 0) return;
   const indent = "  ".repeat(depth);
   const header = `${indent}[§ active: ${scope.label}]`;
-  const headerTokens = estimateTokens(header);
+  if (!tryAppendLine(state, header)) {
-  if (state.tokens + headerTokens > state.maxTokens) {
     state.truncated = true;
     state.stopped = true;
     return;
   }
-  state.lines.push(header);
-  state.tokens += headerTokens;
 
   for (const text of scope.lines.slice(0, MAX_SCOPE_LINES)) {
     if (state.stopped) return;
     const clean = cleaned(text);
     if (!clean) continue;
     const line = `${indent}  ${clean}`;
-    const nextTokens = state.tokens + estimateTokens(line);
+    if (!tryAppendLine(state, line)) {
-    if (nextTokens > state.maxTokens) {
       state.truncated = true;
       state.stopped = true;
       return;
     }
-    state.lines.push(line);
-    state.tokens = nextTokens;
   }
 }
 
@@ -959,15 +957,11 @@ function renderTree(children: Map<number | null, VomNode[]>, state: RenderState)
     const context = ref ? handleContext(node, state) : [];
     const surface = state.surfaceMap.get(node.id);
     const line = renderNodeLine(node, depth, ref, context, surface, state.redactValues);
-    const nextTokens = state.tokens + estimateTokens(line);
+    if (!tryAppendLine(state, line)) {
-    if (nextTokens > state.maxTokens) {
       state.truncated = true;
       state.stopped = true;
       break;
     }
-
-    state.lines.push(line);
-    state.tokens = nextTokens;
     if (ref) {
       const name = cleaned(node.name);
       state.refs.push({
@@ -1001,6 +995,8 @@ function renderNodes(
   surfaces: CondSurface[] = [],
   activeScopeBlocks: ActiveScopeBlock[] = [],
   visualSurfaces: VomVisualSurface[] = [],
+  visualSurfacesTruncated = false,
+  omittedVisualSurfaceCount?: number,
 ): RenderState {
   const children = buildChildren(nodes);
   const state: RenderState = {
@@ -1021,6 +1017,8 @@ function renderNodes(
     surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])),
     scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])),
     visualSurfaces,
+    visualSurfacesTruncated,
+    omittedVisualSurfaceCount,
   };
 
   renderTree(children, state);
@@ -1241,10 +1239,11 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
     scene.surfaces,
     scene.activeScopeBlocks,
     scene.visualSurfaces?.filter((surface) => visibleSurface(surface.parentId)),
+    scene.visualSurfacesTruncated,
+    scene.omittedVisualSurfaceCount,
   );
   const occlusionLine = renderPageOcclusionLine(hiddenCount);
-  state.lines.push(occlusionLine);
+  if (!tryAppendLine(state, occlusionLine)) state.truncated = true;
-  state.tokens += estimateTokens(occlusionLine);
   emitVisualAppendix(state);
 
   return {
@@ -1273,6 +1272,8 @@ export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult
     scene.surfaces,
     scene.activeScopeBlocks,
     scene.visualSurfaces,
+    scene.visualSurfacesTruncated,
+    scene.omittedVisualSurfaceCount,
   );
   emitVisualAppendix(state);
 

packages/vom/src/types.ts

@@ -63,6 +63,10 @@ export interface VomScene {
   activeScopeBlocks?: ActiveScopeBlock[];
   /** Non-semantic rendered regions discovered alongside the AX/DOM tree. */
   visualSurfaces?: VomVisualSurface[];
+  /** Upstream surface discovery/aggregation stopped at a resource guardrail. */
+  visualSurfacesTruncated?: boolean;
+  /** Exact omitted count when upstream could determine it. */
+  omittedVisualSurfaceCount?: number;
 }
 
 export interface VomOptions {