From 309d270045541b43b7105e6eebcecd5ee3c8d700 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Tue, 8 Sep 2026 13:13:25 +0800 Subject: [PATCH] fix(vom): add fixtures, remove fixed canvas numbers under budget, improve efficiency --- .../__tests__/ref-store.test.ts | 10 + .../src/session-manager/ref-store.ts | 2 +- .../tools/__tests__/frame-geometry.test.ts | 53 +++- .../src/tools/__tests__/observation.test.ts | 50 +++- .../__tests__/visual-screenshot-plan.test.ts | 37 +++ apps/extension/src/tools/frame-geometry.ts | 236 ++++++++++++++---- apps/extension/src/tools/geometry.ts | 19 -- apps/extension/src/tools/observation.ts | 85 +++++-- apps/extension/src/tools/viewport-metrics.ts | 18 ++ .../src/tools/visual-screenshot-plan.ts | 52 ++++ .../src/tools/vom/__tests__/capture.test.ts | 16 +- .../tools/vom/__tests__/frame-capture.test.ts | 69 +++++ .../vom/__tests__/rendered-surfaces.test.ts | 78 +++++- apps/extension/src/tools/vom/capture.ts | 18 +- apps/extension/src/tools/vom/frame-capture.ts | 85 ++++--- .../tools/vom/rendered-surfaces/cluster.ts | 198 ++++++++++++--- .../src/tools/vom/rendered-surfaces/index.ts | 6 +- .../tools/vom/rendered-surfaces/project.ts | 16 +- .../src/tools/vom/rendered-surfaces/types.ts | 9 +- .../test-fixtures/rendered-surfaces/README.md | 5 + .../rendered-surfaces/sparklines.html | 28 +++ packages/vom/src/__tests__/render.test.ts | 28 ++- packages/vom/src/render.ts | 85 +++---- packages/vom/src/types.ts | 4 + 24 files changed, 960 insertions(+), 247 deletions(-) create mode 100644 apps/extension/src/tools/__tests__/visual-screenshot-plan.test.ts create mode 100644 apps/extension/src/tools/viewport-metrics.ts create mode 100644 apps/extension/src/tools/visual-screenshot-plan.ts create mode 100644 apps/extension/test-fixtures/rendered-surfaces/sparklines.html diff --git a/apps/extension/src/session-manager/__tests__/ref-store.test.ts b/apps/extension/src/session-manager/__tests__/ref-store.test.ts index dfccd38..2358926 100644 --- a/apps/extension/src/session-manager/__tests__/ref-store.test.ts +++ b/apps/extension/src/session-manager/__tests__/ref-store.test.ts @@ -77,4 +77,14 @@ describe("RefStore", () => { expect(s.resolveEntry("e1")?.capabilities).toEqual(["screenshot"]); expect(s.resolveEntry("e2")?.capabilities).toEqual(["screenshot"]); }); + + it("copies caller-owned capability arrays", () => { + const s = new RefStore(); + const capabilities: Array<"interact" | "screenshot"> = ["screenshot"]; + s.set("e1", 42, { tabId: 7, capabilities }); + + capabilities.push("interact"); + + expect(s.resolveEntry("e1")?.capabilities).toEqual(["screenshot"]); + }); }); diff --git a/apps/extension/src/session-manager/ref-store.ts b/apps/extension/src/session-manager/ref-store.ts index af7a872..eb3253b 100644 --- a/apps/extension/src/session-manager/ref-store.ts +++ b/apps/extension/src/session-manager/ref-store.ts @@ -45,7 +45,7 @@ function capabilitiesFor( capabilities: RefCapability[] | undefined, ): RefCapability[] { if (kind === "surface") return ["screenshot"]; - return capabilities ?? ["interact", "screenshot"]; + return capabilities ? [...capabilities] : ["interact", "screenshot"]; } export class RefStore { diff --git a/apps/extension/src/tools/__tests__/frame-geometry.test.ts b/apps/extension/src/tools/__tests__/frame-geometry.test.ts index 9116f6a..241ac92 100644 --- a/apps/extension/src/tools/__tests__/frame-geometry.test.ts +++ b/apps/extension/src/tools/__tests__/frame-geometry.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it, vi } from "vitest"; -import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry"; +import { + createFrameGeometryContext, + resolveFrameProjection, + resolveNodeGeometry, +} from "../frame-geometry"; import { clipPolygon, polygonArea, @@ -37,6 +41,53 @@ describe("frame geometry projection", () => { expect(polygonBounds(projected.flat())).toEqual({ x: 224, y: 346, width: 200, height: 80 }); }); + it("projects nested same-target frame-local bounds through authoritative content quads", async () => { + const boxCalls: number[] = []; + const cdp: CdpRunner = { + send: vi.fn(async (_tabId, method, params) => { + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1_000, clientHeight: 800 } }; + } + if (method === "Page.createIsolatedWorld") return { executionContextId: 91 }; + if (method === "Runtime.evaluate") { + return { result: { value: { width: 100, height: 80 } } }; + } + if (method === "DOM.getBoxModel") { + const backendNodeId = (params as { backendNodeId?: number }).backendNodeId ?? 0; + boxCalls.push(backendNodeId); + return backendNodeId === 10 + ? { model: { content: [30, 100.5, 330, 100.5, 330, 300.5, 30, 300.5] } } + : { model: { content: [45, 120.5, 145, 120.5, 145, 200.5, 45, 200.5] } }; + } + throw new Error(`unexpected ${method}`); + }) as CdpRunner["send"], + }; + const geometry = createFrameGeometryContext(cdp, { + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 10, + target: { tabId: 4 }, + }, + { + frameId: "nested", + parentFrameId: "child", + ownerBackendNodeId: 20, + target: { tabId: 4 }, + }, + ], + }); + + await expect( + geometry.projectFrameLocalRect("nested", { x: 12, y: 12, w: 40, h: 20 }), + ).resolves.toEqual({ x: 57, y: 132.5, width: 40, height: 20 }); + await geometry.projectFrameLocalRect("nested", { x: 1, y: 1, w: 2, h: 2 }); + expect(boxCalls.sort()).toEqual([10, 20]); + }); + it("uses a projective mapping for perspective-transformed iframe quads", () => { const quad: Quad = [ { x: 0, y: 0 }, diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index 2850ae3..f5cdcfd 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -330,6 +330,7 @@ describe("handleScreenshot", () => { tabId: 7, kind: "surface", capabilities: ["screenshot"], + visibleRect: { x: 0, y: 0, w: 4096, h: 4096 }, }); const { cdp, sent } = makeFakeCdp({ "Page.getLayoutMetrics": () => ({ @@ -359,6 +360,49 @@ describe("handleScreenshot", () => { expect(clip.clip?.scale).toBeCloseTo(Math.sqrt(4_000_000 / (4096 * 4096))); }); + it("retries once when actual surface PNG dimensions exceed the pixel budget", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + ctx.refStore.set("e5", 999, { + tabId: 7, + kind: "surface", + visibleRect: { x: 0, y: 0, w: 1_000, h: 1_000 }, + }); + let captures = 0; + const { cdp, sent } = makeFakeCdp({ + "Runtime.evaluate": () => ({ result: { value: 1 } }), + "Page.getLayoutMetrics": () => ({ + cssLayoutViewport: { clientWidth: 1_000, clientHeight: 1_000 }, + }), + "DOM.getContentQuads": () => ({ + quads: [[0, 0, 1_000, 0, 1_000, 1_000, 0, 1_000]], + }), + "Page.captureScreenshot": () => ({ + data: + ++captures === 1 + ? pngWithIhdrDimensions(4_000, 4_000) + : pngWithIhdrDimensions(1_998, 1_998), + }), + }); + + const res = await handleScreenshot( + sm, + { session_id: "aa11", ref: "@e5", tab_id: 7 }, + makeScreenshotDeps({ + cdp, + get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab), + query: vi.fn(), + captureVisibleTab: vi.fn(), + }), + ); + + if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(res).toMatchObject({ width: 1_998, height: 1_998 }); + const calls = sent.filter((call) => call.method === "Page.captureScreenshot"); + expect(calls).toHaveLength(2); + expect((calls[1]?.params as { clip?: { scale?: number } }).clip?.scale).toBeLessThan(0.5); + }); + it("crops a visual surface screenshot to its observation-time visible region", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); @@ -462,7 +506,11 @@ describe("handleScreenshot", () => { it("fails closed when a ref capture is not a valid PNG", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const ctx = await sm.start("aa11"); - ctx.refStore.set("e5", 999, { tabId: 7, kind: "surface" }); + ctx.refStore.set("e5", 999, { + tabId: 7, + kind: "surface", + visibleRect: { x: 0, y: 0, w: 50, h: 50 }, + }); const { cdp } = makeFakeCdp({ "DOM.getContentQuads": () => ({ quads: [[0, 0, 50, 0, 50, 50, 0, 50]] }), "Page.captureScreenshot": () => ({ data: "bm90LWEtcG5n" }), diff --git a/apps/extension/src/tools/__tests__/visual-screenshot-plan.test.ts b/apps/extension/src/tools/__tests__/visual-screenshot-plan.test.ts new file mode 100644 index 0000000..0988e8a --- /dev/null +++ b/apps/extension/src/tools/__tests__/visual-screenshot-plan.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { + correctedVisualScreenshotScale, + planVisualScreenshot, + screenshotFitsPixelBudget, +} from "../visual-screenshot-plan"; + +const budget = { maxEdge: 2_048, maxPixels: 4_000_000 }; + +describe("visual screenshot pixel planning", () => { + it.each([1, 1.25, 1.5, 2, 3])("accounts for raster scale %s", (rasterScaleEstimate) => { + const plan = planVisualScreenshot({ + cssRect: { width: 2_000, height: 1_000 }, + rasterScaleEstimate, + budget, + }); + const width = 2_000 * rasterScaleEstimate * plan.clipScale; + const height = 1_000 * rasterScaleEstimate * plan.clipScale; + + expect(Math.max(width, height)).toBeLessThanOrEqual(budget.maxEdge); + expect(width * height).toBeLessThanOrEqual(budget.maxPixels); + }); + + it("corrects a scale from actual PNG dimensions", () => { + const corrected = correctedVisualScreenshotScale({ + previousClipScale: 1, + actualWidth: 4_000, + actualHeight: 2_000, + budget, + }); + + expect(corrected).not.toBeNull(); + expect(corrected as number).toBeLessThan(2_048 / 4_000); + expect(screenshotFitsPixelBudget({ width: 2_000, height: 1_000 }, budget)).toBe(true); + expect(screenshotFitsPixelBudget({ width: 4_000, height: 2_000 }, budget)).toBe(false); + }); +}); diff --git a/apps/extension/src/tools/frame-geometry.ts b/apps/extension/src/tools/frame-geometry.ts index 0c788c9..b0df1c4 100644 --- a/apps/extension/src/tools/frame-geometry.ts +++ b/apps/extension/src/tools/frame-geometry.ts @@ -1,4 +1,9 @@ -import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph"; +import { + type CdpFrame, + type CdpFrameGraph, + type CdpTarget, + cdpTargetKey, +} from "@/browser-driver/frame-graph"; import type { RpcError } from "@/transport/types"; import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry"; import { @@ -10,6 +15,7 @@ import { parseCdpQuad, polygonArea, polygonCentroid, + projectRectToViewport, projectRegionToViewport, type Quad, type Region, @@ -84,6 +90,41 @@ async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise 0 && height > 0 ? { width, height } : null; } +async function sameTargetFrameViewport(cdp: CdpRunner, frame: CdpFrame): Promise { + const world = await sendToCdpTarget<{ executionContextId?: number }>( + cdp, + frame.target, + "Page.createIsolatedWorld", + { + frameId: frame.frameId, + worldName: "bsk-frame-geometry", + grantUniveralAccess: false, + }, + ); + if (world.executionContextId === undefined) return null; + const result = await sendToCdpTarget<{ result?: { value?: unknown } }>( + cdp, + frame.target, + "Runtime.evaluate", + { + expression: "({width:window.innerWidth,height:window.innerHeight})", + contextId: world.executionContextId, + returnByValue: true, + }, + ); + const value = result.result?.value as { width?: unknown; height?: unknown } | undefined; + const width = value?.width; + const height = value?.height; + return typeof width === "number" && + Number.isFinite(width) && + width > 0 && + typeof height === "number" && + Number.isFinite(height) && + height > 0 + ? { width, height } + : null; +} + async function ownerContentQuad( cdp: CdpRunner, parent: CdpFrame, @@ -98,26 +139,150 @@ async function ownerContentQuad( return parseCdpQuad(result.model?.content); } -async function sameTargetFrameClips( +export interface FrameGeometryContext { + resolveFrameProjection(frameId: string): Promise; + projectFrameLocalRect( + frameId: string, + rect: { x: number; y: number; w: number; h: number }, + ): Promise; +} + +export function createFrameGeometryContext( cdp: CdpRunner, - byId: Map, - frame: CdpFrame, - targetRoot: CdpFrame, -): Promise { - const clips: Polygon[] = []; - let current = frame; - const seen = new Set(); - while (current.frameId !== targetRoot.frameId) { - if (seen.has(current.frameId) || !current.parentFrameId) return null; - seen.add(current.frameId); - const parent = byId.get(current.parentFrameId); - if (!parent || current.ownerBackendNodeId === undefined) return null; - const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId); - if (!clip) return null; - clips.push(clip); - current = parent; - } - return clips; + graph: CdpFrameGraph, +): FrameGeometryContext { + const byId = frameMap(graph); + const targetViewports = new Map>(); + const frameViewports = new Map>(); + const ownerQuads = new Map>(); + const projections = new Map>(); + const localProjections = new Map>(); + + const cachedTargetViewport = (target: CdpTarget): Promise => { + const key = cdpTargetKey(target); + let value = targetViewports.get(key); + if (!value) { + value = targetViewport(cdp, target); + targetViewports.set(key, value); + } + return value; + }; + const cachedFrameViewport = (frame: CdpFrame): Promise => { + let value = frameViewports.get(frame.frameId); + if (!value) { + const root = targetRootFrame(byId, frame); + value = + root?.frameId === frame.frameId + ? cachedTargetViewport(frame.target) + : sameTargetFrameViewport(cdp, frame); + frameViewports.set(frame.frameId, value); + } + return value; + }; + const cachedOwnerQuad = (parent: CdpFrame, backendNodeId: number): Promise => { + const key = `${cdpTargetKey(parent.target)}:${backendNodeId}`; + let value = ownerQuads.get(key); + if (!value) { + value = ownerContentQuad(cdp, parent, backendNodeId); + ownerQuads.set(key, value); + } + return value; + }; + const cachedSameTargetClips = async ( + frame: CdpFrame, + root: CdpFrame, + ): Promise => { + const clips: Polygon[] = []; + let current = frame; + const seen = new Set(); + while (current.frameId !== root.frameId) { + if (seen.has(current.frameId) || !current.parentFrameId) return null; + seen.add(current.frameId); + const parent = byId.get(current.parentFrameId); + if (!parent || current.ownerBackendNodeId === undefined) return null; + const clip = await cachedOwnerQuad(parent, current.ownerBackendNodeId); + if (!clip) return null; + clips.push(clip); + current = parent; + } + return clips; + }; + + const buildTargetProjection = async (frameId: string): Promise => { + const frame = byId.get(frameId); + if (!frame) return null; + let root = targetRootFrame(byId, frame); + if (!root) return null; + const sourceViewport = await cachedTargetViewport(root.target); + if (!sourceViewport) return null; + const sourceClips = await cachedSameTargetClips(frame, root); + if (!sourceClips) return null; + const edges: ProjectiveEdge[] = []; + while (root.parentFrameId) { + const parent = byId.get(root.parentFrameId); + if (!parent || root.ownerBackendNodeId === undefined) return null; + const destinationQuad = await cachedOwnerQuad(parent, root.ownerBackendNodeId); + if (!destinationQuad) return null; + const source = await cachedTargetViewport(root.target); + if (!source) return null; + const parentRoot = targetRootFrame(byId, parent); + if (!parentRoot) return null; + const destinationClips = await cachedSameTargetClips(parent, parentRoot); + if (!destinationClips) return null; + edges.push({ sourceViewport: source, destinationQuad, destinationClips }); + root = parentRoot; + } + const topViewport = await cachedTargetViewport(root.target); + return topViewport ? { sourceClips, edges, topViewport } : null; + }; + + const resolveFrameProjectionCached = (frameId: string): Promise => { + let value = projections.get(frameId); + if (!value) { + value = buildTargetProjection(frameId); + projections.set(frameId, value); + } + return value; + }; + + const buildLocalProjection = async (frameId: string): Promise => { + const frame = byId.get(frameId); + if (!frame) return null; + const root = targetRootFrame(byId, frame); + if (!root) return null; + const targetProjection = await resolveFrameProjectionCached(root.frameId); + if (!targetProjection) return null; + if (root.frameId === frame.frameId) return targetProjection; + if (!frame.parentFrameId || frame.ownerBackendNodeId === undefined) return null; + const parent = byId.get(frame.parentFrameId); + if (!parent) return null; + const sourceViewport = await cachedFrameViewport(frame); + const destinationQuad = await cachedOwnerQuad(parent, frame.ownerBackendNodeId); + const destinationClips = await cachedSameTargetClips(frame, root); + if (!sourceViewport || !destinationQuad || !destinationClips) return null; + return { + sourceClips: [], + edges: [{ sourceViewport, destinationQuad, destinationClips }, ...targetProjection.edges], + topViewport: targetProjection.topViewport, + }; + }; + + const localProjection = (frameId: string): Promise => { + let value = localProjections.get(frameId); + if (!value) { + value = buildLocalProjection(frameId); + localProjections.set(frameId, value); + } + return value; + }; + + return { + resolveFrameProjection: resolveFrameProjectionCached, + async projectFrameLocalRect(frameId, rect) { + const projection = await localProjection(frameId); + return projection ? projectRectToViewport(rect, projection) : null; + }, + }; } export async function resolveFrameProjection( @@ -125,36 +290,7 @@ export async function resolveFrameProjection( graph: CdpFrameGraph, frameId: string, ): Promise { - const byId = frameMap(graph); - const frame = byId.get(frameId); - if (!frame) return null; - let targetRoot = targetRootFrame(byId, frame); - if (!targetRoot) return null; - const sourceViewport = await targetViewport(cdp, targetRoot.target); - if (!sourceViewport) return null; - const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot); - if (!sourceClips) return null; - - const edges: ProjectiveEdge[] = []; - while (targetRoot.parentFrameId) { - const parent = byId.get(targetRoot.parentFrameId); - if (!parent || targetRoot.ownerBackendNodeId === undefined) return null; - const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId); - if (!destinationQuad) return null; - const source = - edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target); - if (!source) return null; - const parentTargetRoot = targetRootFrame(byId, parent); - if (!parentTargetRoot) return null; - const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot); - if (!destinationClips) return null; - edges.push({ sourceViewport: source, destinationQuad, destinationClips }); - targetRoot = parentTargetRoot; - } - - const topViewport = - edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target); - return topViewport ? { sourceClips, edges, topViewport } : null; + return createFrameGeometryContext(cdp, graph).resolveFrameProjection(frameId); } async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise { diff --git a/apps/extension/src/tools/geometry.ts b/apps/extension/src/tools/geometry.ts index fea8782..247ca02 100644 --- a/apps/extension/src/tools/geometry.ts +++ b/apps/extension/src/tools/geometry.ts @@ -245,22 +245,3 @@ export function projectRectToViewport( if (!rect) return null; return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection)); } - -export function childFrameProjection( - parent: GeometryProjection, - ownerRectInParent: { x: number; y: number; w: number; h: number }, -): GeometryProjection { - const destinationQuad = rectPolygon(ownerRectInParent); - return { - sourceClips: [], - edges: [ - { - sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h }, - destinationQuad, - destinationClips: parent.sourceClips, - }, - ...parent.edges, - ], - topViewport: parent.topViewport, - }; -} diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index e37531d..a22522d 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -48,6 +48,13 @@ import { type ToolEffect, } from "./shared"; import { resolveSnapshotRef } from "./snapshot-ref"; +import { topLevelScreenshotRasterScale } from "./viewport-metrics"; +import { + correctedVisualScreenshotScale, + planVisualScreenshot, + type ScreenshotPixelBudget, + screenshotFitsPixelBudget, +} from "./visual-screenshot-plan"; import { type CapturedNode, type CapturedSurfaceProbe, @@ -101,6 +108,10 @@ export const normaliseRef = sharedNormaliseRef; const VISUAL_SURFACE_SCREENSHOT_MAX_EDGE = 2_048; const VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS = 4_000_000; +const VISUAL_SURFACE_SCREENSHOT_BUDGET: ScreenshotPixelBudget = { + maxEdge: VISUAL_SURFACE_SCREENSHOT_MAX_EDGE, + maxPixels: VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS, +}; /** * Strip the `data:image/...;base64,` prefix from a Chrome @@ -225,36 +236,52 @@ async function captureElementScreenshot( })() : liveRect; if (!rect) return { code: "permission_denied", message: "surface is no longer visible" }; - const scale = visualSurface - ? Math.min( - 1, - VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.width, - VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.height, - Math.sqrt(VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS / (rect.width * rect.height)), - ) + const rasterScaleEstimate = visualSurface ? await topLevelScreenshotRasterScale(cdp, tabId) : 1; + let scale = visualSurface + ? planVisualScreenshot({ + cssRect: rect, + rasterScaleEstimate, + budget: VISUAL_SURFACE_SCREENSHOT_BUDGET, + }).clipScale : 1; try { - const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", { - format: "png", - clip: { - x: rect.x, - y: rect.y, - width: rect.width, - height: rect.height, - scale, - }, - }); - if (signal?.aborted) return cancelled("screenshot"); - const image_base64 = shot.data ?? ""; - if (!image_base64) { - return { code: "cdp_failed", message: "Page.captureScreenshot returned no data" }; + for (let attempt = 0; attempt < (visualSurface ? 2 : 1); attempt += 1) { + const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", { + format: "png", + clip: { + x: rect.x, + y: rect.y, + width: rect.width, + height: rect.height, + scale, + }, + }); + if (signal?.aborted) return cancelled("screenshot"); + const image_base64 = shot.data ?? ""; + if (!image_base64) { + return { code: "cdp_failed", message: "Page.captureScreenshot returned no data" }; + } + const dims = parsePngDimensions(image_base64); + if (!dims) { + return { code: "cdp_failed", message: "Page.captureScreenshot returned invalid PNG data" }; + } + if (!visualSurface || screenshotFitsPixelBudget(dims, VISUAL_SURFACE_SCREENSHOT_BUDGET)) { + return { image_base64, width: dims.width, height: dims.height }; + } + const corrected = correctedVisualScreenshotScale({ + previousClipScale: scale, + actualWidth: dims.width, + actualHeight: dims.height, + budget: VISUAL_SURFACE_SCREENSHOT_BUDGET, + }); + if (attempt === 1 || corrected === null) break; + scale = corrected; } - const dims = parsePngDimensions(image_base64); - if (!dims) { - return { code: "cdp_failed", message: "Page.captureScreenshot returned invalid PNG data" }; - } - return { image_base64, width: dims.width, height: dims.height }; + return { + code: "cdp_failed", + message: "Page.captureScreenshot exceeded the visual surface pixel budget", + }; } catch (err) { return { code: "cdp_failed", @@ -351,6 +378,12 @@ export async function handleScreenshot( } const node = resolveSnapshotRef(ctx, ref, target.tabId, "screenshot"); if (isRpcError(node)) return node; + if (node.kind === "surface" && !node.visibleRect) { + return { + code: "permission_denied", + message: `surface ref ${ref} has no observation-time visible bounds`, + }; + } if (signal?.aborted) return cancelled("screenshot"); deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url); diff --git a/apps/extension/src/tools/viewport-metrics.ts b/apps/extension/src/tools/viewport-metrics.ts new file mode 100644 index 0000000..3a6ca3c --- /dev/null +++ b/apps/extension/src/tools/viewport-metrics.ts @@ -0,0 +1,18 @@ +import type { CdpRunner } from "./shared"; + +/** Estimate the raster/CSS ratio used by top-level Page.captureScreenshot. */ +export async function topLevelScreenshotRasterScale( + cdp: CdpRunner, + tabId: number, +): Promise { + try { + const reply = await cdp.send<{ result?: { value?: unknown } }>(tabId, "Runtime.evaluate", { + expression: "window.devicePixelRatio", + returnByValue: true, + }); + const value = reply.result?.value; + return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 1; + } catch { + return 1; + } +} diff --git a/apps/extension/src/tools/visual-screenshot-plan.ts b/apps/extension/src/tools/visual-screenshot-plan.ts new file mode 100644 index 0000000..bb7a747 --- /dev/null +++ b/apps/extension/src/tools/visual-screenshot-plan.ts @@ -0,0 +1,52 @@ +export interface ScreenshotPixelBudget { + maxEdge: number; + maxPixels: number; +} + +export interface VisualScreenshotPlan { + clipScale: number; +} + +export function planVisualScreenshot(input: { + cssRect: { width: number; height: number }; + rasterScaleEstimate: number; + budget: ScreenshotPixelBudget; +}): VisualScreenshotPlan { + const rasterScale = + Number.isFinite(input.rasterScaleEstimate) && input.rasterScaleEstimate > 0 + ? input.rasterScaleEstimate + : 1; + const { width, height } = input.cssRect; + const scale = Math.min( + 1, + input.budget.maxEdge / (width * rasterScale), + input.budget.maxEdge / (height * rasterScale), + Math.sqrt(input.budget.maxPixels / (width * height * rasterScale * rasterScale)), + ); + return { clipScale: Math.max(Number.EPSILON, scale) }; +} + +export function correctedVisualScreenshotScale(input: { + previousClipScale: number; + actualWidth: number; + actualHeight: number; + budget: ScreenshotPixelBudget; +}): number | null { + const correction = Math.min( + input.budget.maxEdge / input.actualWidth, + input.budget.maxEdge / input.actualHeight, + Math.sqrt(input.budget.maxPixels / (input.actualWidth * input.actualHeight)), + ); + if (!Number.isFinite(correction) || correction >= 1 || correction <= 0) return null; + return Math.max(Number.EPSILON, input.previousClipScale * correction * 0.999); +} + +export function screenshotFitsPixelBudget( + dimensions: { width: number; height: number }, + budget: ScreenshotPixelBudget, +): boolean { + return ( + Math.max(dimensions.width, dimensions.height) <= budget.maxEdge && + dimensions.width * dimensions.height <= budget.maxPixels + ); +} diff --git a/apps/extension/src/tools/vom/__tests__/capture.test.ts b/apps/extension/src/tools/vom/__tests__/capture.test.ts index 97d1b37..c47898d 100644 --- a/apps/extension/src/tools/vom/__tests__/capture.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture.test.ts @@ -1011,7 +1011,7 @@ describe("captureViewModel", () => { expect(input?.attrs.type).toBe("text"); expect(input?.ownerFrameBackendNodeId).toBe(13); expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 }); - expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 }); + expect(input?.rect).toBeNull(); expect(rootFrameId).toBe("main-frame"); expect(frameNodes?.get("child-frame")).toBe(subNodes); expect(frameOwnerBackendNodeIds?.get("child-frame")).toBe(13); @@ -1183,13 +1183,13 @@ describe("captureViewModel", () => { const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31); expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 }); - expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 }); + expect(nestedIframe?.rect).toBeNull(); expect(input?.ownerFrameBackendNodeId).toBe(23); expect(input?.localRect).toEqual({ x: 1, y: 2, w: 40, h: 20 }); - expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 }); + expect(input?.rect).toBeNull(); }); - it("keeps same-origin iframe canvas discovery in top-level CSS coordinates at dpr 1 and 2", async () => { + it("keeps same-origin iframe canvas frame-local until frame geometry is resolved", async () => { const captureAtDpr = async (dpr: number) => { const S = ["html", "body", "iframe", "canvas", "static", "auto"]; const i = (s: string) => S.indexOf(s); @@ -1270,13 +1270,7 @@ describe("captureViewModel", () => { const atDpr1 = await captureAtDpr(1); const atDpr2 = await captureAtDpr(2); - expect(atDpr1).toEqual([ - expect.objectContaining({ - backendNodeId: 21, - frameId: "child", - visibleRect: { x: 120, y: 180, w: 120, h: 60 }, - }), - ]); + expect(atDpr1).toEqual([]); expect(atDpr2).toEqual(atDpr1); }); diff --git a/apps/extension/src/tools/vom/__tests__/frame-capture.test.ts b/apps/extension/src/tools/vom/__tests__/frame-capture.test.ts index 6bbd224..7450426 100644 --- a/apps/extension/src/tools/vom/__tests__/frame-capture.test.ts +++ b/apps/extension/src/tools/vom/__tests__/frame-capture.test.ts @@ -50,6 +50,75 @@ function childSnapshot(frameId: string, backendNodeId: number) { } describe("captureFrameData", () => { + it("projects same-origin frame-local rects through the iframe content box", async () => { + const owner = ownerNode(10, 50); + const canvas: CapturedNode = { + backendNodeId: 101, + parentBackendNodeId: 100, + frameId: "child", + ownerFrameBackendNodeId: 10, + tag: "canvas", + attrs: {}, + rect: null, + localRect: { x: 12, y: 12, w: 224, h: 94 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }; + const captured: CapturedViewModel = { + nodes: [owner], + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map([[10, [canvas]]]), + frameNodes: new Map([ + ["main", [owner]], + ["child", [canvas]], + ]), + frameOwnerBackendNodeIds: new Map([["child", 10]]), + frameParentIds: new Map([["child", "main"]]), + rootFrameId: "main", + excludedBackendNodeIds: new Set(), + }; + const cdp: CdpRunner = { + send: vi.fn(async (_tabId, method) => { + if (method === "Accessibility.enable") return {}; + if (method === "Accessibility.getFullAXTree") return { nodes: [] }; + if (method === "Page.getLayoutMetrics") { + return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } }; + } + if (method === "DOM.getBoxModel") { + return { model: { content: [52, 102, 352, 102, 352, 302, 52, 302] } }; + } + if (method === "Page.createIsolatedWorld") return { executionContextId: 91 }; + if (method === "Runtime.evaluate") { + return { result: { value: { width: 300, height: 200 } } }; + } + throw new Error(`unexpected ${method}`); + }) as CdpRunner["send"], + getFrameGraph: vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 10, + target: { tabId: 4 }, + }, + ], + })), + }; + + await captureFrameData(cdp, 4, captured); + + expect(captured.frameNodes?.get("child")?.[0]?.rect).toEqual({ + x: 64, + y: 114, + w: 224, + h: 94, + }); + expect(captured.iframeNodes.get(10)?.[0]?.rect).toEqual({ x: 64, y: 114, w: 224, h: 94 }); + }); + it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => { const leftOwner = ownerNode(10, 50); const rightOwner = ownerNode(20, 500); diff --git a/apps/extension/src/tools/vom/__tests__/rendered-surfaces.test.ts b/apps/extension/src/tools/vom/__tests__/rendered-surfaces.test.ts index 03edf59..03f1cb9 100644 --- a/apps/extension/src/tools/vom/__tests__/rendered-surfaces.test.ts +++ b/apps/extension/src/tools/vom/__tests__/rendered-surfaces.test.ts @@ -183,7 +183,7 @@ describe("rendered surface clustering and projection", () => { ], { width: 1280, height: 720 }, ); - const groups = clusterRenderedSurfaces(surfaces); + const clustered = clusterRenderedSurfaces(surfaces); const scene: VomScene = { viewport: { width: 1280, height: 720 }, nodes: [ @@ -202,14 +202,14 @@ describe("rendered surface clustering and projection", () => { ], }; - expect(groups).toHaveLength(1); - expect(groups[0]?.members).toHaveLength(2); - expect(groups[0]?.label).toBeUndefined(); + expect(clustered.groups).toHaveLength(1); + expect(clustered.groups[0]?.memberCount).toBe(2); + expect(clustered.groups[0]?.label).toBeUndefined(); expect( projectRenderedSurfaces( scene, [document([body, ...surfaces.map((s) => domNode(s.backendNodeId))])], - groups, + clustered, ), ).toMatchObject({ visualSurfaces: [expect.objectContaining({ parentId: 10, backendNodeId: 3, memberCount: 2 })], @@ -217,7 +217,7 @@ describe("rendered surface clustering and projection", () => { }); it("does not merge a contained canvas with a larger independent canvas", () => { - const groups = clusterRenderedSurfaces([ + const clustered = clusterRenderedSurfaces([ ...discoverRenderedSurfaces( [ document([ @@ -233,11 +233,34 @@ describe("rendered surface clustering and projection", () => { ), ]); - expect(groups).toHaveLength(2); + expect(clustered.groups).toHaveLength(2); + }); + + it("keeps the existing inclusive 90 percent stack threshold", () => { + const clustered = clusterRenderedSurfaces([ + { + renderingKind: "canvas", + frameId: "main", + backendNodeId: 2, + parentBackendNodeId: 1, + visibleRect: { x: 0, y: 0, w: 100, h: 100 }, + paintOrder: 1, + }, + { + renderingKind: "canvas", + frameId: "main", + backendNodeId: 3, + parentBackendNodeId: 1, + visibleRect: { x: 5, y: 0, w: 90, h: 100 }, + paintOrder: 2, + }, + ]); + + expect(clustered.groups).toEqual([expect.objectContaining({ memberCount: 2 })]); }); it("does not merge coincident canvases owned by different containers", () => { - const groups = clusterRenderedSurfaces( + const clustered = clusterRenderedSurfaces( discoverRenderedSurfaces( [ document([ @@ -251,6 +274,43 @@ describe("rendered surface clustering and projection", () => { ), ); - expect(groups).toHaveLength(2); + expect(clustered.groups).toHaveLength(2); + }); + + it("keeps adjacent map tiles and twenty sparklines as independent surfaces", () => { + const surfaces = Array.from({ length: 20 }, (_, index) => + domNode(100 + index, { + parentBackendNodeId: 1, + rect: { x: index * 12, y: 20, w: 10, h: 8 }, + }), + ); + const clustered = clusterRenderedSurfaces( + discoverRenderedSurfaces( + [document([domNode(1, { tag: "main", rect: null, localRect: null }), ...surfaces])], + { width: 1280, height: 720 }, + ), + ); + + expect(clustered.groups).toHaveLength(20); + expect(clustered.groups.every((group) => group.memberCount === 1)).toBe(true); + expect(clustered.truncated).toBe(false); + }); + + it("clusters a large coincident stack without retaining every member", () => { + const surfaces = Array.from({ length: 10_000 }, (_, index) => ({ + renderingKind: "canvas" as const, + frameId: "main", + backendNodeId: index + 1, + parentBackendNodeId: 1, + visibleRect: { x: 10, y: 20, w: 800, h: 500 }, + paintOrder: index, + })); + + const clustered = clusterRenderedSurfaces(surfaces); + + expect(clustered.groups).toEqual([ + expect.objectContaining({ memberCount: 10_000, representative: surfaces[9_999] }), + ]); + expect(clustered.truncated).toBe(false); }); }); diff --git a/apps/extension/src/tools/vom/capture.ts b/apps/extension/src/tools/vom/capture.ts index cc3ba09..d43c655 100644 --- a/apps/extension/src/tools/vom/capture.ts +++ b/apps/extension/src/tools/vom/capture.ts @@ -7,7 +7,7 @@ import type { Rect, Viewport } from "@browser-skill/vom"; import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy"; import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge"; -import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry"; +import { type GeometryProjection, projectRectToViewport } from "../geometry"; import type { CdpRunner } from "../shared"; import { clearHover, ProbeBudget, waitForHover } from "./hover-perception"; @@ -1014,7 +1014,6 @@ function parseChildFrameDocuments( strings: string[], dpr: number, parentDocIndex: number, - parentNodes: CapturedNode[], parentContext: FrameContext, visited = new Set(), ): ParseFrameDocumentsResult { @@ -1029,19 +1028,17 @@ function parseChildFrameDocuments( } const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? []; - const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node])); - for (const [nodeArrayIdx, childDocIndex] of cdi) { if (visited.has(childDocIndex)) continue; const childDoc = documents[childDocIndex]; if (!childDoc) continue; const iframeBackendId = parentBackendIds[nodeArrayIdx]; if (iframeBackendId === undefined) continue; - const iframeNode = parentNodeByBackendId.get(iframeBackendId); - const projection = - parentContext.projection && iframeNode?.localRect - ? childFrameProjection(parentContext.projection, iframeNode.localRect) - : null; + // DOMSnapshot reports the iframe owner's border-box bounds, but child + // document coordinates start at the content-box origin. Preserve the + // frame-local rectangles here and let frame-capture normalize them with + // authoritative content quads once the frame graph is available. + const projection = null; const childContext: FrameContext = { frameId: snapshotFrameId(childDoc, strings), @@ -1065,7 +1062,6 @@ function parseChildFrameDocuments( strings, dpr, childDocIndex, - parsed.nodes, childContext, nextVisited, ); @@ -1143,7 +1139,7 @@ export async function captureViewModel( const nodes = mainParsed.nodes; const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds); - const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext); + const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, topContext); const iframeNodes = frameParsed.iframeNodes; for (const id of frameParsed.excludedBackendNodeIds) { excludedBackendNodeIds.add(id); diff --git a/apps/extension/src/tools/vom/frame-capture.ts b/apps/extension/src/tools/vom/frame-capture.ts index d643a7b..6211243 100644 --- a/apps/extension/src/tools/vom/frame-capture.ts +++ b/apps/extension/src/tools/vom/frame-capture.ts @@ -1,6 +1,5 @@ import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph"; -import { resolveFrameProjection } from "../frame-geometry"; -import { type GeometryProjection, projectRectToViewport } from "../geometry"; +import { createFrameGeometryContext } from "../frame-geometry"; import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared"; import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture"; import { @@ -31,21 +30,6 @@ async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise ({ - ...node, - rect: projection - ? (() => { - const bounds = projectRectToViewport(node.rect, projection); - return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null; - })() - : null, - })); -} - async function captureMissingFrameDocuments( cdp: CdpRunner, tabId: number, @@ -63,16 +47,6 @@ async function captureMissingFrameDocuments( const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, { signal, }); - let projection: GeometryProjection | null = null; - try { - projection = await resolveFrameProjection(cdp, graph, frame.frameId); - } catch (err) { - if (isAbortError(err)) throw err; - console.debug("[bsk observation] child frame geometry projection failed", { - frameId: frame.frameId, - err, - }); - } const rootFrameId = child.rootFrameId ?? frame.frameId; const childFrames = child.frameNodes ?? new Map(); if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes); @@ -83,7 +57,7 @@ async function captureMissingFrameDocuments( ); } for (const [childFrameId, nodes] of childFrames) { - captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection)); + captured.frameNodes.set(childFrameId, nodes); } if (frame.ownerBackendNodeId !== undefined) { captured.iframeNodes.set( @@ -109,6 +83,56 @@ async function captureMissingFrameDocuments( } } +async function normalizeFrameLocalRects( + cdp: CdpRunner, + graph: CdpFrameGraph, + captured: CapturedViewModel, + signal?: AbortSignal, +): Promise { + if (!captured.frameNodes) return; + const geometry = createFrameGeometryContext(cdp, graph); + const frameById = new Map(graph.frames.map((frame) => [frame.frameId, frame])); + await Promise.all( + [...captured.frameNodes].map(async ([frameId, nodes]) => { + if (frameId === graph.rootFrameId || !frameById.has(frameId)) return; + try { + const normalized = await Promise.all( + nodes.map(async (node) => { + if (!node.localRect) return { ...node, rect: null }; + const bounds = await geometry.projectFrameLocalRect(frameId, node.localRect); + return { + ...node, + rect: bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null, + }; + }), + ); + throwIfAborted(signal); + captured.frameNodes?.set(frameId, normalized); + const ownerBackendNodeId = + frameById.get(frameId)?.ownerBackendNodeId ?? + captured.frameOwnerBackendNodeIds?.get(frameId); + if (ownerBackendNodeId !== undefined) { + captured.iframeNodes.set(ownerBackendNodeId, normalized); + } + } catch (err) { + if (isAbortError(err)) throw err; + console.debug("[bsk observation] frame-local geometry normalization failed", { + frameId, + err, + }); + const unavailable = nodes.map((node) => ({ ...node, rect: null })); + captured.frameNodes?.set(frameId, unavailable); + const ownerBackendNodeId = + frameById.get(frameId)?.ownerBackendNodeId ?? + captured.frameOwnerBackendNodeIds?.get(frameId); + if (ownerBackendNodeId !== undefined) { + captured.iframeNodes.set(ownerBackendNodeId, unavailable); + } + } + }), + ); +} + async function captureAxTrees( cdp: CdpRunner, tabId: number, @@ -174,7 +198,10 @@ export async function captureFrameData( ): Promise[]> { const graph = await discoverFrameGraph(cdp, tabId); const frames = graph?.frames ?? []; - if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal); + if (graph) { + await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal); + await normalizeFrameLocalRects(cdp, graph, captured, signal); + } throwIfAborted(signal); const batches = await captureAxTrees(cdp, tabId, frames, captured, signal); return buildFrameDocuments(graph, batches, captured); diff --git a/apps/extension/src/tools/vom/rendered-surfaces/cluster.ts b/apps/extension/src/tools/vom/rendered-surfaces/cluster.ts index 32db4a4..8cf34b5 100644 --- a/apps/extension/src/tools/vom/rendered-surfaces/cluster.ts +++ b/apps/extension/src/tools/vom/rendered-surfaces/cluster.ts @@ -1,8 +1,17 @@ import type { Rect } from "@browser-skill/vom"; -import type { RenderedSurface, RenderedSurfaceGroup } from "./types"; +import type { ClusteredRenderedSurfaces, RenderedSurface, RenderedSurfaceGroup } from "./types"; const STACK_IOU_RATIO = 0.9; const STACK_SIZE_RATIO = 0.9; +const SIZE_BAND_BASE = 1 / STACK_SIZE_RATIO; +const POSITION_CELL_RATIO = 0.1; +const MAX_CLUSTER_COMPARISONS = 1_000_000; +const MAX_CLUSTER_SURFACES = 50_000; + +interface IndexedGroup extends RenderedSurfaceGroup { + creationIndex: number; + indexKey: string; +} function area(rect: Rect): number { return Math.max(0, rect.w) * Math.max(0, rect.h); @@ -14,9 +23,7 @@ function intersectionArea(a: Rect, b: Rect): number { return width * height; } -function sameVisualStack(a: RenderedSurface, b: RenderedSurface): boolean { - if (a.frameId !== b.frameId) return false; - if (a.parentBackendNodeId !== b.parentBackendNodeId) return false; +function stackScore(a: RenderedSurface, b: RenderedSurface): number | null { const aArea = area(a.visibleRect); const bArea = area(b.visibleRect); const intersection = intersectionArea(a.visibleRect, b.visibleRect); @@ -25,41 +32,158 @@ function sameVisualStack(a: RenderedSurface, b: RenderedSurface): boolean { Math.min(a.visibleRect.w, b.visibleRect.w) / Math.max(a.visibleRect.w, b.visibleRect.w); const heightRatio = Math.min(a.visibleRect.h, b.visibleRect.h) / Math.max(a.visibleRect.h, b.visibleRect.h); - return ( - union > 0 && - intersection / union >= STACK_IOU_RATIO && - widthRatio >= STACK_SIZE_RATIO && - heightRatio >= STACK_SIZE_RATIO - ); + const iou = union > 0 ? intersection / union : 0; + return iou >= STACK_IOU_RATIO && widthRatio >= STACK_SIZE_RATIO && heightRatio >= STACK_SIZE_RATIO + ? iou + : null; } -function toGroup(members: RenderedSurface[]): RenderedSurfaceGroup { - const representative = [...members].sort( - (a, b) => area(b.visibleRect) - area(a.visibleRect) || b.paintOrder - a.paintOrder, - )[0]; +function sizeBand(value: number): number { + return Math.floor(Math.log(Math.max(Number.EPSILON, value)) / Math.log(SIZE_BAND_BASE)); +} + +function bandSize(band: number): number { + return SIZE_BAND_BASE ** band; +} + +function spatialKey(rect: Rect): string { + const widthBand = sizeBand(rect.w); + const heightBand = sizeBand(rect.h); + const cellWidth = bandSize(widthBand) * POSITION_CELL_RATIO; + const cellHeight = bandSize(heightBand) * POSITION_CELL_RATIO; + const centerX = rect.x + rect.w / 2; + const centerY = rect.y + rect.h / 2; + return `${widthBand}:${heightBand}:${Math.floor(centerX / cellWidth)}:${Math.floor(centerY / cellHeight)}`; +} + +function candidateKeys(rect: Rect): string[] { + const keys: string[] = []; + const centerX = rect.x + rect.w / 2; + const centerY = rect.y + rect.h / 2; + const widthBand = sizeBand(rect.w); + const heightBand = sizeBand(rect.h); + // A >=0.9 size ratio crosses at most one logarithmic band. IoU >=0.9 + // likewise keeps centers within one 10%-of-size position cell. + for (let dw = -1; dw <= 1; dw += 1) { + for (let dh = -1; dh <= 1; dh += 1) { + const candidateWidthBand = widthBand + dw; + const candidateHeightBand = heightBand + dh; + const cellWidth = bandSize(candidateWidthBand) * POSITION_CELL_RATIO; + const cellHeight = bandSize(candidateHeightBand) * POSITION_CELL_RATIO; + const cellX = Math.floor(centerX / cellWidth); + const cellY = Math.floor(centerY / cellHeight); + for (let dx = -1; dx <= 1; dx += 1) { + for (let dy = -1; dy <= 1; dy += 1) { + keys.push(`${candidateWidthBand}:${candidateHeightBand}:${cellX + dx}:${cellY + dy}`); + } + } + } + } + return keys; +} + +function preferredRepresentative(a: RenderedSurface, b: RenderedSurface): RenderedSurface { + const aArea = area(a.visibleRect); + const bArea = area(b.visibleRect); + return bArea > aArea || (bArea === aArea && b.paintOrder > a.paintOrder) ? b : a; +} + +function partitionKey(surface: RenderedSurface): string { + return `${surface.frameId}\u0000${surface.parentBackendNodeId ?? "root"}`; +} + +export function clusterRenderedSurfaces(surfaces: RenderedSurface[]): ClusteredRenderedSurfaces { + const partitions = new Map>>(); + const groups: IndexedGroup[] = []; + let comparisons = 0; + let processed = 0; + let truncated = false; + + for (const surface of surfaces) { + if (processed >= MAX_CLUSTER_SURFACES || comparisons >= MAX_CLUSTER_COMPARISONS) { + truncated = true; + break; + } + const key = partitionKey(surface); + let index = partitions.get(key); + if (!index) { + index = new Map(); + partitions.set(key, index); + } + const candidates = new Set(); + for (const candidateKey of candidateKeys(surface.visibleRect)) { + for (const candidate of index.get(candidateKey) ?? []) candidates.add(candidate); + } + + let best: { group: IndexedGroup; score: number; areaDelta: number } | null = null; + for (const group of candidates) { + comparisons += 1; + if (comparisons > MAX_CLUSTER_COMPARISONS) { + truncated = true; + break; + } + const score = stackScore(group.representative, surface); + if (score === null) continue; + const areaDelta = Math.abs( + area(group.representative.visibleRect) - area(surface.visibleRect), + ); + if ( + !best || + score > best.score || + (score === best.score && areaDelta < best.areaDelta) || + (score === best.score && + areaDelta === best.areaDelta && + group.creationIndex < best.group.creationIndex) + ) { + best = { group, score, areaDelta }; + } + } + if (truncated) break; + + if (best) { + const group = best.group; + const representative = preferredRepresentative(group.representative, surface); + group.memberCount += 1; + group.label ??= surface.label; + if (representative !== group.representative) { + index.get(group.indexKey)?.delete(group); + group.representative = representative; + group.indexKey = spatialKey(representative.visibleRect); + const indexed = index.get(group.indexKey) ?? new Set(); + indexed.add(group); + index.set(group.indexKey, indexed); + } + processed += 1; + continue; + } + + const indexKey = spatialKey(surface.visibleRect); + const group: IndexedGroup = { + frameId: surface.frameId, + parentBackendNodeId: surface.parentBackendNodeId, + representative: surface, + ...(surface.label ? { label: surface.label } : {}), + memberCount: 1, + creationIndex: groups.length, + indexKey, + }; + groups.push(group); + const indexed = index.get(indexKey) ?? new Set(); + indexed.add(group); + index.set(indexKey, indexed); + processed += 1; + } + return { - frameId: representative.frameId, - members, - representative, - label: members.map((surface) => surface.label).find((label) => label !== undefined), + groups: groups + .map(({ creationIndex: _creationIndex, indexKey: _indexKey, ...group }) => group) + .sort( + (a, b) => + a.frameId.localeCompare(b.frameId) || + a.representative.visibleRect.y - b.representative.visibleRect.y || + a.representative.visibleRect.x - b.representative.visibleRect.x, + ), + truncated, + ...(truncated ? { omittedCount: surfaces.length - processed } : {}), }; } - -export function clusterRenderedSurfaces(surfaces: RenderedSurface[]): RenderedSurfaceGroup[] { - const groups: RenderedSurface[][] = []; - for (const surface of surfaces) { - const matching = groups.find((group) => - group.some((member) => sameVisualStack(member, surface)), - ); - if (matching) matching.push(surface); - else groups.push([surface]); - } - return groups - .map(toGroup) - .sort( - (a, b) => - a.frameId.localeCompare(b.frameId) || - a.representative.visibleRect.y - b.representative.visibleRect.y || - a.representative.visibleRect.x - b.representative.visibleRect.x, - ); -} diff --git a/apps/extension/src/tools/vom/rendered-surfaces/index.ts b/apps/extension/src/tools/vom/rendered-surfaces/index.ts index 6fd1ec9..d1c1873 100644 --- a/apps/extension/src/tools/vom/rendered-surfaces/index.ts +++ b/apps/extension/src/tools/vom/rendered-surfaces/index.ts @@ -1,4 +1,8 @@ export { clusterRenderedSurfaces } from "./cluster"; export { discoverRenderedSurfaces } from "./discovery"; export { projectRenderedSurfaces } from "./project"; -export type { RenderedSurface, RenderedSurfaceGroup } from "./types"; +export type { + ClusteredRenderedSurfaces, + RenderedSurface, + RenderedSurfaceGroup, +} from "./types"; diff --git a/apps/extension/src/tools/vom/rendered-surfaces/project.ts b/apps/extension/src/tools/vom/rendered-surfaces/project.ts index c219457..8f94882 100644 --- a/apps/extension/src/tools/vom/rendered-surfaces/project.ts +++ b/apps/extension/src/tools/vom/rendered-surfaces/project.ts @@ -1,7 +1,7 @@ import type { VomScene, VomVisualSurface } from "@browser-skill/vom"; import type { CapturedFrameDocument } from "../frame-capture"; import type { SemanticAxNode } from "../semantic-graph"; -import type { RenderedSurfaceGroup } from "./types"; +import type { ClusteredRenderedSurfaces, RenderedSurfaceGroup } from "./types"; function frameBackendKey(frameId: string, backendNodeId: number): string { return `${frameId}\u0000${backendNodeId}`; @@ -75,12 +75,12 @@ function buildProjectionIndex( export function projectRenderedSurfaces( scene: VomScene, documents: CapturedFrameDocument[], - groups: RenderedSurfaceGroup[], + clustered: ClusteredRenderedSurfaces, ): VomScene { const index = buildProjectionIndex(scene, documents); const visualSurfaces: VomVisualSurface[] = []; - for (const group of groups) { + for (const group of clustered.groups) { const parentId = nearestSceneParent(index, group); visualSurfaces.push({ parentId, @@ -89,12 +89,20 @@ export function projectRenderedSurfaces( renderingKind: "canvas", visibleRect: group.representative.visibleRect, ...(group.label ? { label: group.label } : {}), - memberCount: group.members.length, + memberCount: group.memberCount, }); } return { ...scene, ...(visualSurfaces.length > 0 ? { visualSurfaces } : {}), + ...(clustered.truncated + ? { + visualSurfacesTruncated: true, + ...(clustered.omittedCount !== undefined + ? { omittedVisualSurfaceCount: clustered.omittedCount } + : {}), + } + : {}), }; } diff --git a/apps/extension/src/tools/vom/rendered-surfaces/types.ts b/apps/extension/src/tools/vom/rendered-surfaces/types.ts index 947d26b..5d1b8e1 100644 --- a/apps/extension/src/tools/vom/rendered-surfaces/types.ts +++ b/apps/extension/src/tools/vom/rendered-surfaces/types.ts @@ -12,7 +12,14 @@ export interface RenderedSurface { export interface RenderedSurfaceGroup { frameId: string; - members: RenderedSurface[]; + parentBackendNodeId: number | null; representative: RenderedSurface; label?: string; + memberCount: number; +} + +export interface ClusteredRenderedSurfaces { + groups: RenderedSurfaceGroup[]; + truncated: boolean; + omittedCount?: number; } diff --git a/apps/extension/test-fixtures/rendered-surfaces/README.md b/apps/extension/test-fixtures/rendered-surfaces/README.md index d0f10a0..67925f6 100644 --- a/apps/extension/test-fixtures/rendered-surfaces/README.md +++ b/apps/extension/test-fixtures/rendered-surfaces/README.md @@ -95,6 +95,11 @@ Use the normal DOM buttons and re-observe after each action: - After **Restore size**: present with a fresh ref. - After **Remove canvas**: absent. +### `sparklines.html` + +Expected Surface labels `fixture-sparkline-01` through `fixture-sparkline-20` must all be present. +The adjacent, non-overlapping canvases remain twenty independent screenshot-only Surface refs. + ## Invariants - `snapshot` must not emit any of these Surface markers. diff --git a/apps/extension/test-fixtures/rendered-surfaces/sparklines.html b/apps/extension/test-fixtures/rendered-surfaces/sparklines.html new file mode 100644 index 0000000..686c3d3 --- /dev/null +++ b/apps/extension/test-fixtures/rendered-surfaces/sparklines.html @@ -0,0 +1,28 @@ + + + + + + Rendered Surface sparkline matrix + + + +

Twenty independent sparklines

+
+ + + diff --git a/packages/vom/src/__tests__/render.test.ts b/packages/vom/src/__tests__/render.test.ts index b843bbc..42b085f 100644 --- a/packages/vom/src/__tests__/render.test.ts +++ b/packages/vom/src/__tests__/render.test.ts @@ -117,7 +117,7 @@ describe("renderVom single-layer page", () => { ); }); - it("bounds visual output and prefers explicitly labeled surfaces", () => { + it("uses the appendix token budget and prefers explicitly labeled surfaces", () => { const input = scene([node({ id: 1, role: "RootWebArea", frameId: "main" })]); input.visualSurfaces = Array.from({ length: 12 }, (_, index) => ({ parentId: 1, @@ -132,10 +132,30 @@ describe("renderVom single-layer page", () => { const out = renderVom(input); const surfaceRefs = out.refs.filter((ref) => ref.kind === "surface"); - expect(surfaceRefs).toHaveLength(8); + expect(surfaceRefs).toHaveLength(12); expect(surfaceRefs[0]).toMatchObject({ backendNodeId: 111, name: "Important status" }); - expect(out.text).toContain("… 4 additional visual surfaces omitted"); - expect(out.truncated).toBe(true); + expect(out.truncated).toBe(false); + }); + + it("renders twenty independent sparklines without changing DOM refs", () => { + const input = scene([ + node({ id: 1, role: "RootWebArea", frameId: "main" }), + node({ id: 2, parentId: 1, role: "button", name: "Submit", tag: "button" }), + ]); + input.visualSurfaces = Array.from({ length: 20 }, (_, index) => ({ + parentId: 1, + backendNodeId: 100 + index, + frameId: "main", + renderingKind: "canvas" as const, + visibleRect: { x: index * 12, y: 20, w: 10, h: 8 }, + memberCount: 1, + })); + + const out = renderVom(input); + + expect(out.refs[0]).toMatchObject({ ref: "e1", backendNodeId: 2, kind: "dom" }); + expect(out.refs.filter((ref) => ref.kind === "surface")).toHaveLength(20); + expect(out.truncated).toBe(false); }); it("does not let a visual surface exhaust the budget before later DOM content", () => { diff --git a/packages/vom/src/render.ts b/packages/vom/src/render.ts index d4d3601..df94a32 100644 --- a/packages/vom/src/render.ts +++ b/packages/vom/src/render.ts @@ -88,7 +88,8 @@ const MAX_CUSTOM_RECOVERY_AREA = 180_000; const MAX_RECOVERABLE_SINGLE_INTENT_AREA = 360_000; const MAX_HANDLE_CONTEXT_ITEMS = 3; const MAX_SURFACE_ITEMS = 12; -const MAX_VISUAL_SURFACES = 8; +const DEFAULT_VISUAL_APPENDIX_TOKENS = 2_048; +const MAX_VISUAL_SURFACE_REFS = 512; const MAX_SCOPE_LINES = 40; const RECOVERY_HANDLER_ATTRS = ["onclick", "onmousedown", "onkeydown", "onkeyup", "onkeypress"]; const NATIVE_TAGS = new Set(["button", "input", "select", "textarea"]); @@ -796,6 +797,16 @@ interface RenderState { surfaceMap: Map; scopeMap: Map; visualSurfaces: VomVisualSurface[]; + visualSurfacesTruncated: boolean; + omittedVisualSurfaceCount?: number; +} + +function tryAppendLine(state: RenderState, line: string, tokenLimit = state.maxTokens): boolean { + const nextTokens = state.tokens + estimateTokens(line); + if (nextTokens > tokenLimit) return false; + state.lines.push(line); + state.tokens = nextTokens; + return true; } function visualSurfaceArea(surface: VomVisualSurface): number { @@ -834,41 +845,35 @@ function selectVisualSurfaces( if (surface.parentId !== null && !nodesById.has(surface.parentId)) continue; eligibleCount += 1; - const insertionIndex = selected.findIndex( - (candidate) => compareVisualSurfaces(surface, candidate) < 0, - ); - if (insertionIndex < 0) { - if (selected.length < MAX_VISUAL_SURFACES) selected.push(surface); - continue; - } - selected.splice(insertionIndex, 0, surface); - if (selected.length > MAX_VISUAL_SURFACES) selected.pop(); + selected.push(surface); } - + selected.sort(compareVisualSurfaces); return { selected, eligibleCount }; } function emitVisualAppendix(state: RenderState): void { if (state.stopped) return; const { selected, eligibleCount } = selectVisualSurfaces(state.visualSurfaces, state.nodesById); - if (eligibleCount === 0) return; + if (eligibleCount === 0 && !state.visualSurfacesTruncated) return; const header = "[visual surfaces]"; + const appendixStartTokens = state.tokens; + const appendixTokenLimit = Number.isFinite(state.maxTokens) + ? state.maxTokens + : appendixStartTokens + DEFAULT_VISUAL_APPENDIX_TOKENS; let headerEmitted = false; let emitted = 0; for (const surface of selected) { + if (emitted >= MAX_VISUAL_SURFACE_REFS) break; const ref = `e${state.nextRef}`; const line = visualSurfaceLine(surface, ref); - const nextTokens = - state.tokens + estimateTokens(line) + (headerEmitted ? 0 : estimateTokens(header)); - if (nextTokens > state.maxTokens) break; + const requiredTokens = estimateTokens(line) + (headerEmitted ? 0 : estimateTokens(header)); + if (state.tokens + requiredTokens > appendixTokenLimit) break; if (!headerEmitted) { - state.lines.push(header); - state.tokens += estimateTokens(header); + if (!tryAppendLine(state, header, appendixTokenLimit)) break; headerEmitted = true; } - state.lines.push(line); - state.tokens += estimateTokens(line); + if (!tryAppendLine(state, line, appendixTokenLimit)) break; const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`; state.refs.push({ ref, @@ -885,16 +890,15 @@ function emitVisualAppendix(state: RenderState): void { emitted += 1; } - const omitted = eligibleCount - emitted; - if (omitted > 0) { + const locallyOmitted = eligibleCount - emitted; + const upstreamOmitted = state.omittedVisualSurfaceCount ?? 0; + const omitted = locallyOmitted + upstreamOmitted; + if (omitted > 0 || state.visualSurfacesTruncated) { state.truncated = true; if (headerEmitted) { - const summary = ` … ${omitted} additional visual surfaces omitted`; - const nextTokens = state.tokens + estimateTokens(summary); - if (nextTokens <= state.maxTokens) { - state.lines.push(summary); - state.tokens = nextTokens; - } + const count = + omitted > 0 ? `${state.visualSurfacesTruncated ? "at least " : ""}${omitted}` : "some"; + tryAppendLine(state, ` … ${count} additional visual surfaces omitted`, appendixTokenLimit); } } } @@ -903,28 +907,22 @@ function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: Ren if (state.stopped || scope.lines.length === 0) return; const indent = " ".repeat(depth); const header = `${indent}[§ active: ${scope.label}]`; - const headerTokens = estimateTokens(header); - if (state.tokens + headerTokens > state.maxTokens) { + if (!tryAppendLine(state, header)) { state.truncated = true; state.stopped = true; return; } - state.lines.push(header); - state.tokens += headerTokens; for (const text of scope.lines.slice(0, MAX_SCOPE_LINES)) { if (state.stopped) return; const clean = cleaned(text); if (!clean) continue; const line = `${indent} ${clean}`; - const nextTokens = state.tokens + estimateTokens(line); - if (nextTokens > state.maxTokens) { + if (!tryAppendLine(state, line)) { state.truncated = true; state.stopped = true; return; } - state.lines.push(line); - state.tokens = nextTokens; } } @@ -959,15 +957,11 @@ function renderTree(children: Map, state: RenderState) const context = ref ? handleContext(node, state) : []; const surface = state.surfaceMap.get(node.id); const line = renderNodeLine(node, depth, ref, context, surface, state.redactValues); - const nextTokens = state.tokens + estimateTokens(line); - if (nextTokens > state.maxTokens) { + if (!tryAppendLine(state, line)) { state.truncated = true; state.stopped = true; break; } - - state.lines.push(line); - state.tokens = nextTokens; if (ref) { const name = cleaned(node.name); state.refs.push({ @@ -1001,6 +995,8 @@ function renderNodes( surfaces: CondSurface[] = [], activeScopeBlocks: ActiveScopeBlock[] = [], visualSurfaces: VomVisualSurface[] = [], + visualSurfacesTruncated = false, + omittedVisualSurfaceCount?: number, ): RenderState { const children = buildChildren(nodes); const state: RenderState = { @@ -1021,6 +1017,8 @@ function renderNodes( surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])), scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])), visualSurfaces, + visualSurfacesTruncated, + omittedVisualSurfaceCount, }; renderTree(children, state); @@ -1241,10 +1239,11 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp scene.surfaces, scene.activeScopeBlocks, scene.visualSurfaces?.filter((surface) => visibleSurface(surface.parentId)), + scene.visualSurfacesTruncated, + scene.omittedVisualSurfaceCount, ); const occlusionLine = renderPageOcclusionLine(hiddenCount); - state.lines.push(occlusionLine); - state.tokens += estimateTokens(occlusionLine); + if (!tryAppendLine(state, occlusionLine)) state.truncated = true; emitVisualAppendix(state); return { @@ -1273,6 +1272,8 @@ export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult scene.surfaces, scene.activeScopeBlocks, scene.visualSurfaces, + scene.visualSurfacesTruncated, + scene.omittedVisualSurfaceCount, ); emitVisualAppendix(state); diff --git a/packages/vom/src/types.ts b/packages/vom/src/types.ts index b76f33d..e87c5a8 100644 --- a/packages/vom/src/types.ts +++ b/packages/vom/src/types.ts @@ -63,6 +63,10 @@ export interface VomScene { activeScopeBlocks?: ActiveScopeBlock[]; /** Non-semantic rendered regions discovered alongside the AX/DOM tree. */ visualSurfaces?: VomVisualSurface[]; + /** Upstream surface discovery/aggregation stopped at a resource guardrail. */ + visualSurfacesTruncated?: boolean; + /** Exact omitted count when upstream could determine it. */ + omittedVisualSurfaceCount?: number; } export interface VomOptions {