mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix(vom): add fixtures, remove fixed canvas numbers under budget, improve efficiency
This commit is contained in:
Ljy-0827
@@ -77,4 +77,14 @@ describe("RefStore", () => {
|
||||
expect(s.resolveEntry("e1")?.capabilities).toEqual(["screenshot"]);
|
||||
expect(s.resolveEntry("e2")?.capabilities).toEqual(["screenshot"]);
|
||||
});
|
||||
|
||||
it("copies caller-owned capability arrays", () => {
|
||||
const s = new RefStore();
|
||||
const capabilities: Array<"interact" | "screenshot"> = ["screenshot"];
|
||||
s.set("e1", 42, { tabId: 7, capabilities });
|
||||
|
||||
capabilities.push("interact");
|
||||
|
||||
expect(s.resolveEntry("e1")?.capabilities).toEqual(["screenshot"]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -45,7 +45,7 @@ function capabilitiesFor(
|
||||
capabilities: RefCapability[] | undefined,
|
||||
): RefCapability[] {
|
||||
if (kind === "surface") return ["screenshot"];
|
||||
return capabilities ?? ["interact", "screenshot"];
|
||||
return capabilities ? [...capabilities] : ["interact", "screenshot"];
|
||||
}
|
||||
|
||||
export class RefStore {
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry";
|
||||
import {
|
||||
createFrameGeometryContext,
|
||||
resolveFrameProjection,
|
||||
resolveNodeGeometry,
|
||||
} from "../frame-geometry";
|
||||
import {
|
||||
clipPolygon,
|
||||
polygonArea,
|
||||
@@ -37,6 +41,53 @@ describe("frame geometry projection", () => {
|
||||
expect(polygonBounds(projected.flat())).toEqual({ x: 224, y: 346, width: 200, height: 80 });
|
||||
});
|
||||
|
||||
it("projects nested same-target frame-local bounds through authoritative content quads", async () => {
|
||||
const boxCalls: number[] = [];
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method, params) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1_000, clientHeight: 800 } };
|
||||
}
|
||||
if (method === "Page.createIsolatedWorld") return { executionContextId: 91 };
|
||||
if (method === "Runtime.evaluate") {
|
||||
return { result: { value: { width: 100, height: 80 } } };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
const backendNodeId = (params as { backendNodeId?: number }).backendNodeId ?? 0;
|
||||
boxCalls.push(backendNodeId);
|
||||
return backendNodeId === 10
|
||||
? { model: { content: [30, 100.5, 330, 100.5, 330, 300.5, 30, 300.5] } }
|
||||
: { model: { content: [45, 120.5, 145, 120.5, 145, 200.5, 45, 200.5] } };
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
};
|
||||
const geometry = createFrameGeometryContext(cdp, {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4 },
|
||||
},
|
||||
{
|
||||
frameId: "nested",
|
||||
parentFrameId: "child",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 4 },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
await expect(
|
||||
geometry.projectFrameLocalRect("nested", { x: 12, y: 12, w: 40, h: 20 }),
|
||||
).resolves.toEqual({ x: 57, y: 132.5, width: 40, height: 20 });
|
||||
await geometry.projectFrameLocalRect("nested", { x: 1, y: 1, w: 2, h: 2 });
|
||||
expect(boxCalls.sort()).toEqual([10, 20]);
|
||||
});
|
||||
|
||||
it("uses a projective mapping for perspective-transformed iframe quads", () => {
|
||||
const quad: Quad = [
|
||||
{ x: 0, y: 0 },
|
||||
|
||||
@@ -330,6 +330,7 @@ describe("handleScreenshot", () => {
|
||||
tabId: 7,
|
||||
kind: "surface",
|
||||
capabilities: ["screenshot"],
|
||||
visibleRect: { x: 0, y: 0, w: 4096, h: 4096 },
|
||||
});
|
||||
const { cdp, sent } = makeFakeCdp({
|
||||
"Page.getLayoutMetrics": () => ({
|
||||
@@ -359,6 +360,49 @@ describe("handleScreenshot", () => {
|
||||
expect(clip.clip?.scale).toBeCloseTo(Math.sqrt(4_000_000 / (4096 * 4096)));
|
||||
});
|
||||
|
||||
it("retries once when actual surface PNG dimensions exceed the pixel budget", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e5", 999, {
|
||||
tabId: 7,
|
||||
kind: "surface",
|
||||
visibleRect: { x: 0, y: 0, w: 1_000, h: 1_000 },
|
||||
});
|
||||
let captures = 0;
|
||||
const { cdp, sent } = makeFakeCdp({
|
||||
"Runtime.evaluate": () => ({ result: { value: 1 } }),
|
||||
"Page.getLayoutMetrics": () => ({
|
||||
cssLayoutViewport: { clientWidth: 1_000, clientHeight: 1_000 },
|
||||
}),
|
||||
"DOM.getContentQuads": () => ({
|
||||
quads: [[0, 0, 1_000, 0, 1_000, 1_000, 0, 1_000]],
|
||||
}),
|
||||
"Page.captureScreenshot": () => ({
|
||||
data:
|
||||
++captures === 1
|
||||
? pngWithIhdrDimensions(4_000, 4_000)
|
||||
: pngWithIhdrDimensions(1_998, 1_998),
|
||||
}),
|
||||
});
|
||||
|
||||
const res = await handleScreenshot(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
|
||||
makeScreenshotDeps({
|
||||
cdp,
|
||||
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
|
||||
query: vi.fn(),
|
||||
captureVisibleTab: vi.fn(),
|
||||
}),
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res).toMatchObject({ width: 1_998, height: 1_998 });
|
||||
const calls = sent.filter((call) => call.method === "Page.captureScreenshot");
|
||||
expect(calls).toHaveLength(2);
|
||||
expect((calls[1]?.params as { clip?: { scale?: number } }).clip?.scale).toBeLessThan(0.5);
|
||||
});
|
||||
|
||||
it("crops a visual surface screenshot to its observation-time visible region", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
@@ -462,7 +506,11 @@ describe("handleScreenshot", () => {
|
||||
it("fails closed when a ref capture is not a valid PNG", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e5", 999, { tabId: 7, kind: "surface" });
|
||||
ctx.refStore.set("e5", 999, {
|
||||
tabId: 7,
|
||||
kind: "surface",
|
||||
visibleRect: { x: 0, y: 0, w: 50, h: 50 },
|
||||
});
|
||||
const { cdp } = makeFakeCdp({
|
||||
"DOM.getContentQuads": () => ({ quads: [[0, 0, 50, 0, 50, 50, 0, 50]] }),
|
||||
"Page.captureScreenshot": () => ({ data: "bm90LWEtcG5n" }),
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
correctedVisualScreenshotScale,
|
||||
planVisualScreenshot,
|
||||
screenshotFitsPixelBudget,
|
||||
} from "../visual-screenshot-plan";
|
||||
|
||||
const budget = { maxEdge: 2_048, maxPixels: 4_000_000 };
|
||||
|
||||
describe("visual screenshot pixel planning", () => {
|
||||
it.each([1, 1.25, 1.5, 2, 3])("accounts for raster scale %s", (rasterScaleEstimate) => {
|
||||
const plan = planVisualScreenshot({
|
||||
cssRect: { width: 2_000, height: 1_000 },
|
||||
rasterScaleEstimate,
|
||||
budget,
|
||||
});
|
||||
const width = 2_000 * rasterScaleEstimate * plan.clipScale;
|
||||
const height = 1_000 * rasterScaleEstimate * plan.clipScale;
|
||||
|
||||
expect(Math.max(width, height)).toBeLessThanOrEqual(budget.maxEdge);
|
||||
expect(width * height).toBeLessThanOrEqual(budget.maxPixels);
|
||||
});
|
||||
|
||||
it("corrects a scale from actual PNG dimensions", () => {
|
||||
const corrected = correctedVisualScreenshotScale({
|
||||
previousClipScale: 1,
|
||||
actualWidth: 4_000,
|
||||
actualHeight: 2_000,
|
||||
budget,
|
||||
});
|
||||
|
||||
expect(corrected).not.toBeNull();
|
||||
expect(corrected as number).toBeLessThan(2_048 / 4_000);
|
||||
expect(screenshotFitsPixelBudget({ width: 2_000, height: 1_000 }, budget)).toBe(true);
|
||||
expect(screenshotFitsPixelBudget({ width: 4_000, height: 2_000 }, budget)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,9 @@
|
||||
import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import {
|
||||
type CdpFrame,
|
||||
type CdpFrameGraph,
|
||||
type CdpTarget,
|
||||
cdpTargetKey,
|
||||
} from "@/browser-driver/frame-graph";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry";
|
||||
import {
|
||||
@@ -10,6 +15,7 @@ import {
|
||||
parseCdpQuad,
|
||||
polygonArea,
|
||||
polygonCentroid,
|
||||
projectRectToViewport,
|
||||
projectRegionToViewport,
|
||||
type Quad,
|
||||
type Region,
|
||||
@@ -84,6 +90,41 @@ async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise<Size |
|
||||
return width > 0 && height > 0 ? { width, height } : null;
|
||||
}
|
||||
|
||||
async function sameTargetFrameViewport(cdp: CdpRunner, frame: CdpFrame): Promise<Size | null> {
|
||||
const world = await sendToCdpTarget<{ executionContextId?: number }>(
|
||||
cdp,
|
||||
frame.target,
|
||||
"Page.createIsolatedWorld",
|
||||
{
|
||||
frameId: frame.frameId,
|
||||
worldName: "bsk-frame-geometry",
|
||||
grantUniveralAccess: false,
|
||||
},
|
||||
);
|
||||
if (world.executionContextId === undefined) return null;
|
||||
const result = await sendToCdpTarget<{ result?: { value?: unknown } }>(
|
||||
cdp,
|
||||
frame.target,
|
||||
"Runtime.evaluate",
|
||||
{
|
||||
expression: "({width:window.innerWidth,height:window.innerHeight})",
|
||||
contextId: world.executionContextId,
|
||||
returnByValue: true,
|
||||
},
|
||||
);
|
||||
const value = result.result?.value as { width?: unknown; height?: unknown } | undefined;
|
||||
const width = value?.width;
|
||||
const height = value?.height;
|
||||
return typeof width === "number" &&
|
||||
Number.isFinite(width) &&
|
||||
width > 0 &&
|
||||
typeof height === "number" &&
|
||||
Number.isFinite(height) &&
|
||||
height > 0
|
||||
? { width, height }
|
||||
: null;
|
||||
}
|
||||
|
||||
async function ownerContentQuad(
|
||||
cdp: CdpRunner,
|
||||
parent: CdpFrame,
|
||||
@@ -98,26 +139,150 @@ async function ownerContentQuad(
|
||||
return parseCdpQuad(result.model?.content);
|
||||
}
|
||||
|
||||
async function sameTargetFrameClips(
|
||||
export interface FrameGeometryContext {
|
||||
resolveFrameProjection(frameId: string): Promise<GeometryProjection | null>;
|
||||
projectFrameLocalRect(
|
||||
frameId: string,
|
||||
rect: { x: number; y: number; w: number; h: number },
|
||||
): Promise<ViewportRect | null>;
|
||||
}
|
||||
|
||||
export function createFrameGeometryContext(
|
||||
cdp: CdpRunner,
|
||||
byId: Map<string, CdpFrame>,
|
||||
frame: CdpFrame,
|
||||
targetRoot: CdpFrame,
|
||||
): Promise<Polygon[] | null> {
|
||||
const clips: Polygon[] = [];
|
||||
let current = frame;
|
||||
const seen = new Set<string>();
|
||||
while (current.frameId !== targetRoot.frameId) {
|
||||
if (seen.has(current.frameId) || !current.parentFrameId) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent || current.ownerBackendNodeId === undefined) return null;
|
||||
const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId);
|
||||
if (!clip) return null;
|
||||
clips.push(clip);
|
||||
current = parent;
|
||||
}
|
||||
return clips;
|
||||
graph: CdpFrameGraph,
|
||||
): FrameGeometryContext {
|
||||
const byId = frameMap(graph);
|
||||
const targetViewports = new Map<string, Promise<Size | null>>();
|
||||
const frameViewports = new Map<string, Promise<Size | null>>();
|
||||
const ownerQuads = new Map<string, Promise<Quad | null>>();
|
||||
const projections = new Map<string, Promise<GeometryProjection | null>>();
|
||||
const localProjections = new Map<string, Promise<GeometryProjection | null>>();
|
||||
|
||||
const cachedTargetViewport = (target: CdpTarget): Promise<Size | null> => {
|
||||
const key = cdpTargetKey(target);
|
||||
let value = targetViewports.get(key);
|
||||
if (!value) {
|
||||
value = targetViewport(cdp, target);
|
||||
targetViewports.set(key, value);
|
||||
}
|
||||
return value;
|
||||
};
|
||||
const cachedFrameViewport = (frame: CdpFrame): Promise<Size | null> => {
|
||||
let value = frameViewports.get(frame.frameId);
|
||||
if (!value) {
|
||||
const root = targetRootFrame(byId, frame);
|
||||
value =
|
||||
root?.frameId === frame.frameId
|
||||
? cachedTargetViewport(frame.target)
|
||||
: sameTargetFrameViewport(cdp, frame);
|
||||
frameViewports.set(frame.frameId, value);
|
||||
}
|
||||
return value;
|
||||
};
|
||||
const cachedOwnerQuad = (parent: CdpFrame, backendNodeId: number): Promise<Quad | null> => {
|
||||
const key = `${cdpTargetKey(parent.target)}:${backendNodeId}`;
|
||||
let value = ownerQuads.get(key);
|
||||
if (!value) {
|
||||
value = ownerContentQuad(cdp, parent, backendNodeId);
|
||||
ownerQuads.set(key, value);
|
||||
}
|
||||
return value;
|
||||
};
|
||||
const cachedSameTargetClips = async (
|
||||
frame: CdpFrame,
|
||||
root: CdpFrame,
|
||||
): Promise<Polygon[] | null> => {
|
||||
const clips: Polygon[] = [];
|
||||
let current = frame;
|
||||
const seen = new Set<string>();
|
||||
while (current.frameId !== root.frameId) {
|
||||
if (seen.has(current.frameId) || !current.parentFrameId) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent || current.ownerBackendNodeId === undefined) return null;
|
||||
const clip = await cachedOwnerQuad(parent, current.ownerBackendNodeId);
|
||||
if (!clip) return null;
|
||||
clips.push(clip);
|
||||
current = parent;
|
||||
}
|
||||
return clips;
|
||||
};
|
||||
|
||||
const buildTargetProjection = async (frameId: string): Promise<GeometryProjection | null> => {
|
||||
const frame = byId.get(frameId);
|
||||
if (!frame) return null;
|
||||
let root = targetRootFrame(byId, frame);
|
||||
if (!root) return null;
|
||||
const sourceViewport = await cachedTargetViewport(root.target);
|
||||
if (!sourceViewport) return null;
|
||||
const sourceClips = await cachedSameTargetClips(frame, root);
|
||||
if (!sourceClips) return null;
|
||||
const edges: ProjectiveEdge[] = [];
|
||||
while (root.parentFrameId) {
|
||||
const parent = byId.get(root.parentFrameId);
|
||||
if (!parent || root.ownerBackendNodeId === undefined) return null;
|
||||
const destinationQuad = await cachedOwnerQuad(parent, root.ownerBackendNodeId);
|
||||
if (!destinationQuad) return null;
|
||||
const source = await cachedTargetViewport(root.target);
|
||||
if (!source) return null;
|
||||
const parentRoot = targetRootFrame(byId, parent);
|
||||
if (!parentRoot) return null;
|
||||
const destinationClips = await cachedSameTargetClips(parent, parentRoot);
|
||||
if (!destinationClips) return null;
|
||||
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
|
||||
root = parentRoot;
|
||||
}
|
||||
const topViewport = await cachedTargetViewport(root.target);
|
||||
return topViewport ? { sourceClips, edges, topViewport } : null;
|
||||
};
|
||||
|
||||
const resolveFrameProjectionCached = (frameId: string): Promise<GeometryProjection | null> => {
|
||||
let value = projections.get(frameId);
|
||||
if (!value) {
|
||||
value = buildTargetProjection(frameId);
|
||||
projections.set(frameId, value);
|
||||
}
|
||||
return value;
|
||||
};
|
||||
|
||||
const buildLocalProjection = async (frameId: string): Promise<GeometryProjection | null> => {
|
||||
const frame = byId.get(frameId);
|
||||
if (!frame) return null;
|
||||
const root = targetRootFrame(byId, frame);
|
||||
if (!root) return null;
|
||||
const targetProjection = await resolveFrameProjectionCached(root.frameId);
|
||||
if (!targetProjection) return null;
|
||||
if (root.frameId === frame.frameId) return targetProjection;
|
||||
if (!frame.parentFrameId || frame.ownerBackendNodeId === undefined) return null;
|
||||
const parent = byId.get(frame.parentFrameId);
|
||||
if (!parent) return null;
|
||||
const sourceViewport = await cachedFrameViewport(frame);
|
||||
const destinationQuad = await cachedOwnerQuad(parent, frame.ownerBackendNodeId);
|
||||
const destinationClips = await cachedSameTargetClips(frame, root);
|
||||
if (!sourceViewport || !destinationQuad || !destinationClips) return null;
|
||||
return {
|
||||
sourceClips: [],
|
||||
edges: [{ sourceViewport, destinationQuad, destinationClips }, ...targetProjection.edges],
|
||||
topViewport: targetProjection.topViewport,
|
||||
};
|
||||
};
|
||||
|
||||
const localProjection = (frameId: string): Promise<GeometryProjection | null> => {
|
||||
let value = localProjections.get(frameId);
|
||||
if (!value) {
|
||||
value = buildLocalProjection(frameId);
|
||||
localProjections.set(frameId, value);
|
||||
}
|
||||
return value;
|
||||
};
|
||||
|
||||
return {
|
||||
resolveFrameProjection: resolveFrameProjectionCached,
|
||||
async projectFrameLocalRect(frameId, rect) {
|
||||
const projection = await localProjection(frameId);
|
||||
return projection ? projectRectToViewport(rect, projection) : null;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function resolveFrameProjection(
|
||||
@@ -125,36 +290,7 @@ export async function resolveFrameProjection(
|
||||
graph: CdpFrameGraph,
|
||||
frameId: string,
|
||||
): Promise<GeometryProjection | null> {
|
||||
const byId = frameMap(graph);
|
||||
const frame = byId.get(frameId);
|
||||
if (!frame) return null;
|
||||
let targetRoot = targetRootFrame(byId, frame);
|
||||
if (!targetRoot) return null;
|
||||
const sourceViewport = await targetViewport(cdp, targetRoot.target);
|
||||
if (!sourceViewport) return null;
|
||||
const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot);
|
||||
if (!sourceClips) return null;
|
||||
|
||||
const edges: ProjectiveEdge[] = [];
|
||||
while (targetRoot.parentFrameId) {
|
||||
const parent = byId.get(targetRoot.parentFrameId);
|
||||
if (!parent || targetRoot.ownerBackendNodeId === undefined) return null;
|
||||
const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId);
|
||||
if (!destinationQuad) return null;
|
||||
const source =
|
||||
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
||||
if (!source) return null;
|
||||
const parentTargetRoot = targetRootFrame(byId, parent);
|
||||
if (!parentTargetRoot) return null;
|
||||
const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot);
|
||||
if (!destinationClips) return null;
|
||||
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
|
||||
targetRoot = parentTargetRoot;
|
||||
}
|
||||
|
||||
const topViewport =
|
||||
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
||||
return topViewport ? { sourceClips, edges, topViewport } : null;
|
||||
return createFrameGeometryContext(cdp, graph).resolveFrameProjection(frameId);
|
||||
}
|
||||
|
||||
async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
|
||||
|
||||
@@ -245,22 +245,3 @@ export function projectRectToViewport(
|
||||
if (!rect) return null;
|
||||
return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection));
|
||||
}
|
||||
|
||||
export function childFrameProjection(
|
||||
parent: GeometryProjection,
|
||||
ownerRectInParent: { x: number; y: number; w: number; h: number },
|
||||
): GeometryProjection {
|
||||
const destinationQuad = rectPolygon(ownerRectInParent);
|
||||
return {
|
||||
sourceClips: [],
|
||||
edges: [
|
||||
{
|
||||
sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
|
||||
destinationQuad,
|
||||
destinationClips: parent.sourceClips,
|
||||
},
|
||||
...parent.edges,
|
||||
],
|
||||
topViewport: parent.topViewport,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -48,6 +48,13 @@ import {
|
||||
type ToolEffect,
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import { topLevelScreenshotRasterScale } from "./viewport-metrics";
|
||||
import {
|
||||
correctedVisualScreenshotScale,
|
||||
planVisualScreenshot,
|
||||
type ScreenshotPixelBudget,
|
||||
screenshotFitsPixelBudget,
|
||||
} from "./visual-screenshot-plan";
|
||||
import {
|
||||
type CapturedNode,
|
||||
type CapturedSurfaceProbe,
|
||||
@@ -101,6 +108,10 @@ export const normaliseRef = sharedNormaliseRef;
|
||||
|
||||
const VISUAL_SURFACE_SCREENSHOT_MAX_EDGE = 2_048;
|
||||
const VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS = 4_000_000;
|
||||
const VISUAL_SURFACE_SCREENSHOT_BUDGET: ScreenshotPixelBudget = {
|
||||
maxEdge: VISUAL_SURFACE_SCREENSHOT_MAX_EDGE,
|
||||
maxPixels: VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS,
|
||||
};
|
||||
|
||||
/**
|
||||
* Strip the `data:image/...;base64,` prefix from a Chrome
|
||||
@@ -225,36 +236,52 @@ async function captureElementScreenshot(
|
||||
})()
|
||||
: liveRect;
|
||||
if (!rect) return { code: "permission_denied", message: "surface is no longer visible" };
|
||||
const scale = visualSurface
|
||||
? Math.min(
|
||||
1,
|
||||
VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.width,
|
||||
VISUAL_SURFACE_SCREENSHOT_MAX_EDGE / rect.height,
|
||||
Math.sqrt(VISUAL_SURFACE_SCREENSHOT_MAX_PIXELS / (rect.width * rect.height)),
|
||||
)
|
||||
const rasterScaleEstimate = visualSurface ? await topLevelScreenshotRasterScale(cdp, tabId) : 1;
|
||||
let scale = visualSurface
|
||||
? planVisualScreenshot({
|
||||
cssRect: rect,
|
||||
rasterScaleEstimate,
|
||||
budget: VISUAL_SURFACE_SCREENSHOT_BUDGET,
|
||||
}).clipScale
|
||||
: 1;
|
||||
|
||||
try {
|
||||
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
|
||||
format: "png",
|
||||
clip: {
|
||||
x: rect.x,
|
||||
y: rect.y,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
scale,
|
||||
},
|
||||
});
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const image_base64 = shot.data ?? "";
|
||||
if (!image_base64) {
|
||||
return { code: "cdp_failed", message: "Page.captureScreenshot returned no data" };
|
||||
for (let attempt = 0; attempt < (visualSurface ? 2 : 1); attempt += 1) {
|
||||
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
|
||||
format: "png",
|
||||
clip: {
|
||||
x: rect.x,
|
||||
y: rect.y,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
scale,
|
||||
},
|
||||
});
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const image_base64 = shot.data ?? "";
|
||||
if (!image_base64) {
|
||||
return { code: "cdp_failed", message: "Page.captureScreenshot returned no data" };
|
||||
}
|
||||
const dims = parsePngDimensions(image_base64);
|
||||
if (!dims) {
|
||||
return { code: "cdp_failed", message: "Page.captureScreenshot returned invalid PNG data" };
|
||||
}
|
||||
if (!visualSurface || screenshotFitsPixelBudget(dims, VISUAL_SURFACE_SCREENSHOT_BUDGET)) {
|
||||
return { image_base64, width: dims.width, height: dims.height };
|
||||
}
|
||||
const corrected = correctedVisualScreenshotScale({
|
||||
previousClipScale: scale,
|
||||
actualWidth: dims.width,
|
||||
actualHeight: dims.height,
|
||||
budget: VISUAL_SURFACE_SCREENSHOT_BUDGET,
|
||||
});
|
||||
if (attempt === 1 || corrected === null) break;
|
||||
scale = corrected;
|
||||
}
|
||||
const dims = parsePngDimensions(image_base64);
|
||||
if (!dims) {
|
||||
return { code: "cdp_failed", message: "Page.captureScreenshot returned invalid PNG data" };
|
||||
}
|
||||
return { image_base64, width: dims.width, height: dims.height };
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: "Page.captureScreenshot exceeded the visual surface pixel budget",
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -351,6 +378,12 @@ export async function handleScreenshot(
|
||||
}
|
||||
const node = resolveSnapshotRef(ctx, ref, target.tabId, "screenshot");
|
||||
if (isRpcError(node)) return node;
|
||||
if (node.kind === "surface" && !node.visibleRect) {
|
||||
return {
|
||||
code: "permission_denied",
|
||||
message: `surface ref ${ref} has no observation-time visible bounds`,
|
||||
};
|
||||
}
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import type { CdpRunner } from "./shared";
|
||||
|
||||
/** Estimate the raster/CSS ratio used by top-level Page.captureScreenshot. */
|
||||
export async function topLevelScreenshotRasterScale(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
): Promise<number> {
|
||||
try {
|
||||
const reply = await cdp.send<{ result?: { value?: unknown } }>(tabId, "Runtime.evaluate", {
|
||||
expression: "window.devicePixelRatio",
|
||||
returnByValue: true,
|
||||
});
|
||||
const value = reply.result?.value;
|
||||
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 1;
|
||||
} catch {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
export interface ScreenshotPixelBudget {
|
||||
maxEdge: number;
|
||||
maxPixels: number;
|
||||
}
|
||||
|
||||
export interface VisualScreenshotPlan {
|
||||
clipScale: number;
|
||||
}
|
||||
|
||||
export function planVisualScreenshot(input: {
|
||||
cssRect: { width: number; height: number };
|
||||
rasterScaleEstimate: number;
|
||||
budget: ScreenshotPixelBudget;
|
||||
}): VisualScreenshotPlan {
|
||||
const rasterScale =
|
||||
Number.isFinite(input.rasterScaleEstimate) && input.rasterScaleEstimate > 0
|
||||
? input.rasterScaleEstimate
|
||||
: 1;
|
||||
const { width, height } = input.cssRect;
|
||||
const scale = Math.min(
|
||||
1,
|
||||
input.budget.maxEdge / (width * rasterScale),
|
||||
input.budget.maxEdge / (height * rasterScale),
|
||||
Math.sqrt(input.budget.maxPixels / (width * height * rasterScale * rasterScale)),
|
||||
);
|
||||
return { clipScale: Math.max(Number.EPSILON, scale) };
|
||||
}
|
||||
|
||||
export function correctedVisualScreenshotScale(input: {
|
||||
previousClipScale: number;
|
||||
actualWidth: number;
|
||||
actualHeight: number;
|
||||
budget: ScreenshotPixelBudget;
|
||||
}): number | null {
|
||||
const correction = Math.min(
|
||||
input.budget.maxEdge / input.actualWidth,
|
||||
input.budget.maxEdge / input.actualHeight,
|
||||
Math.sqrt(input.budget.maxPixels / (input.actualWidth * input.actualHeight)),
|
||||
);
|
||||
if (!Number.isFinite(correction) || correction >= 1 || correction <= 0) return null;
|
||||
return Math.max(Number.EPSILON, input.previousClipScale * correction * 0.999);
|
||||
}
|
||||
|
||||
export function screenshotFitsPixelBudget(
|
||||
dimensions: { width: number; height: number },
|
||||
budget: ScreenshotPixelBudget,
|
||||
): boolean {
|
||||
return (
|
||||
Math.max(dimensions.width, dimensions.height) <= budget.maxEdge &&
|
||||
dimensions.width * dimensions.height <= budget.maxPixels
|
||||
);
|
||||
}
|
||||
@@ -1011,7 +1011,7 @@ describe("captureViewModel", () => {
|
||||
expect(input?.attrs.type).toBe("text");
|
||||
expect(input?.ownerFrameBackendNodeId).toBe(13);
|
||||
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
|
||||
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
|
||||
expect(input?.rect).toBeNull();
|
||||
expect(rootFrameId).toBe("main-frame");
|
||||
expect(frameNodes?.get("child-frame")).toBe(subNodes);
|
||||
expect(frameOwnerBackendNodeIds?.get("child-frame")).toBe(13);
|
||||
@@ -1183,13 +1183,13 @@ describe("captureViewModel", () => {
|
||||
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31);
|
||||
|
||||
expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 });
|
||||
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
|
||||
expect(nestedIframe?.rect).toBeNull();
|
||||
expect(input?.ownerFrameBackendNodeId).toBe(23);
|
||||
expect(input?.localRect).toEqual({ x: 1, y: 2, w: 40, h: 20 });
|
||||
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
|
||||
expect(input?.rect).toBeNull();
|
||||
});
|
||||
|
||||
it("keeps same-origin iframe canvas discovery in top-level CSS coordinates at dpr 1 and 2", async () => {
|
||||
it("keeps same-origin iframe canvas frame-local until frame geometry is resolved", async () => {
|
||||
const captureAtDpr = async (dpr: number) => {
|
||||
const S = ["html", "body", "iframe", "canvas", "static", "auto"];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
@@ -1270,13 +1270,7 @@ describe("captureViewModel", () => {
|
||||
const atDpr1 = await captureAtDpr(1);
|
||||
const atDpr2 = await captureAtDpr(2);
|
||||
|
||||
expect(atDpr1).toEqual([
|
||||
expect.objectContaining({
|
||||
backendNodeId: 21,
|
||||
frameId: "child",
|
||||
visibleRect: { x: 120, y: 180, w: 120, h: 60 },
|
||||
}),
|
||||
]);
|
||||
expect(atDpr1).toEqual([]);
|
||||
expect(atDpr2).toEqual(atDpr1);
|
||||
});
|
||||
|
||||
|
||||
@@ -50,6 +50,75 @@ function childSnapshot(frameId: string, backendNodeId: number) {
|
||||
}
|
||||
|
||||
describe("captureFrameData", () => {
|
||||
it("projects same-origin frame-local rects through the iframe content box", async () => {
|
||||
const owner = ownerNode(10, 50);
|
||||
const canvas: CapturedNode = {
|
||||
backendNodeId: 101,
|
||||
parentBackendNodeId: 100,
|
||||
frameId: "child",
|
||||
ownerFrameBackendNodeId: 10,
|
||||
tag: "canvas",
|
||||
attrs: {},
|
||||
rect: null,
|
||||
localRect: { x: 12, y: 12, w: 224, h: 94 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
const captured: CapturedViewModel = {
|
||||
nodes: [owner],
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map([[10, [canvas]]]),
|
||||
frameNodes: new Map([
|
||||
["main", [owner]],
|
||||
["child", [canvas]],
|
||||
]),
|
||||
frameOwnerBackendNodeIds: new Map([["child", 10]]),
|
||||
frameParentIds: new Map([["child", "main"]]),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [52, 102, 352, 102, 352, 302, 52, 302] } };
|
||||
}
|
||||
if (method === "Page.createIsolatedWorld") return { executionContextId: 91 };
|
||||
if (method === "Runtime.evaluate") {
|
||||
return { result: { value: { width: 300, height: 200 } } };
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4 },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
await captureFrameData(cdp, 4, captured);
|
||||
|
||||
expect(captured.frameNodes?.get("child")?.[0]?.rect).toEqual({
|
||||
x: 64,
|
||||
y: 114,
|
||||
w: 224,
|
||||
h: 94,
|
||||
});
|
||||
expect(captured.iframeNodes.get(10)?.[0]?.rect).toEqual({ x: 64, y: 114, w: 224, h: 94 });
|
||||
});
|
||||
|
||||
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
|
||||
const leftOwner = ownerNode(10, 50);
|
||||
const rightOwner = ownerNode(20, 500);
|
||||
|
||||
@@ -183,7 +183,7 @@ describe("rendered surface clustering and projection", () => {
|
||||
],
|
||||
{ width: 1280, height: 720 },
|
||||
);
|
||||
const groups = clusterRenderedSurfaces(surfaces);
|
||||
const clustered = clusterRenderedSurfaces(surfaces);
|
||||
const scene: VomScene = {
|
||||
viewport: { width: 1280, height: 720 },
|
||||
nodes: [
|
||||
@@ -202,14 +202,14 @@ describe("rendered surface clustering and projection", () => {
|
||||
],
|
||||
};
|
||||
|
||||
expect(groups).toHaveLength(1);
|
||||
expect(groups[0]?.members).toHaveLength(2);
|
||||
expect(groups[0]?.label).toBeUndefined();
|
||||
expect(clustered.groups).toHaveLength(1);
|
||||
expect(clustered.groups[0]?.memberCount).toBe(2);
|
||||
expect(clustered.groups[0]?.label).toBeUndefined();
|
||||
expect(
|
||||
projectRenderedSurfaces(
|
||||
scene,
|
||||
[document([body, ...surfaces.map((s) => domNode(s.backendNodeId))])],
|
||||
groups,
|
||||
clustered,
|
||||
),
|
||||
).toMatchObject({
|
||||
visualSurfaces: [expect.objectContaining({ parentId: 10, backendNodeId: 3, memberCount: 2 })],
|
||||
@@ -217,7 +217,7 @@ describe("rendered surface clustering and projection", () => {
|
||||
});
|
||||
|
||||
it("does not merge a contained canvas with a larger independent canvas", () => {
|
||||
const groups = clusterRenderedSurfaces([
|
||||
const clustered = clusterRenderedSurfaces([
|
||||
...discoverRenderedSurfaces(
|
||||
[
|
||||
document([
|
||||
@@ -233,11 +233,34 @@ describe("rendered surface clustering and projection", () => {
|
||||
),
|
||||
]);
|
||||
|
||||
expect(groups).toHaveLength(2);
|
||||
expect(clustered.groups).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("keeps the existing inclusive 90 percent stack threshold", () => {
|
||||
const clustered = clusterRenderedSurfaces([
|
||||
{
|
||||
renderingKind: "canvas",
|
||||
frameId: "main",
|
||||
backendNodeId: 2,
|
||||
parentBackendNodeId: 1,
|
||||
visibleRect: { x: 0, y: 0, w: 100, h: 100 },
|
||||
paintOrder: 1,
|
||||
},
|
||||
{
|
||||
renderingKind: "canvas",
|
||||
frameId: "main",
|
||||
backendNodeId: 3,
|
||||
parentBackendNodeId: 1,
|
||||
visibleRect: { x: 5, y: 0, w: 90, h: 100 },
|
||||
paintOrder: 2,
|
||||
},
|
||||
]);
|
||||
|
||||
expect(clustered.groups).toEqual([expect.objectContaining({ memberCount: 2 })]);
|
||||
});
|
||||
|
||||
it("does not merge coincident canvases owned by different containers", () => {
|
||||
const groups = clusterRenderedSurfaces(
|
||||
const clustered = clusterRenderedSurfaces(
|
||||
discoverRenderedSurfaces(
|
||||
[
|
||||
document([
|
||||
@@ -251,6 +274,43 @@ describe("rendered surface clustering and projection", () => {
|
||||
),
|
||||
);
|
||||
|
||||
expect(groups).toHaveLength(2);
|
||||
expect(clustered.groups).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("keeps adjacent map tiles and twenty sparklines as independent surfaces", () => {
|
||||
const surfaces = Array.from({ length: 20 }, (_, index) =>
|
||||
domNode(100 + index, {
|
||||
parentBackendNodeId: 1,
|
||||
rect: { x: index * 12, y: 20, w: 10, h: 8 },
|
||||
}),
|
||||
);
|
||||
const clustered = clusterRenderedSurfaces(
|
||||
discoverRenderedSurfaces(
|
||||
[document([domNode(1, { tag: "main", rect: null, localRect: null }), ...surfaces])],
|
||||
{ width: 1280, height: 720 },
|
||||
),
|
||||
);
|
||||
|
||||
expect(clustered.groups).toHaveLength(20);
|
||||
expect(clustered.groups.every((group) => group.memberCount === 1)).toBe(true);
|
||||
expect(clustered.truncated).toBe(false);
|
||||
});
|
||||
|
||||
it("clusters a large coincident stack without retaining every member", () => {
|
||||
const surfaces = Array.from({ length: 10_000 }, (_, index) => ({
|
||||
renderingKind: "canvas" as const,
|
||||
frameId: "main",
|
||||
backendNodeId: index + 1,
|
||||
parentBackendNodeId: 1,
|
||||
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
paintOrder: index,
|
||||
}));
|
||||
|
||||
const clustered = clusterRenderedSurfaces(surfaces);
|
||||
|
||||
expect(clustered.groups).toEqual([
|
||||
expect.objectContaining({ memberCount: 10_000, representative: surfaces[9_999] }),
|
||||
]);
|
||||
expect(clustered.truncated).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import { type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
|
||||
|
||||
@@ -1014,7 +1014,6 @@ function parseChildFrameDocuments(
|
||||
strings: string[],
|
||||
dpr: number,
|
||||
parentDocIndex: number,
|
||||
parentNodes: CapturedNode[],
|
||||
parentContext: FrameContext,
|
||||
visited = new Set<number>(),
|
||||
): ParseFrameDocumentsResult {
|
||||
@@ -1029,19 +1028,17 @@ function parseChildFrameDocuments(
|
||||
}
|
||||
|
||||
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
|
||||
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
|
||||
|
||||
for (const [nodeArrayIdx, childDocIndex] of cdi) {
|
||||
if (visited.has(childDocIndex)) continue;
|
||||
const childDoc = documents[childDocIndex];
|
||||
if (!childDoc) continue;
|
||||
const iframeBackendId = parentBackendIds[nodeArrayIdx];
|
||||
if (iframeBackendId === undefined) continue;
|
||||
const iframeNode = parentNodeByBackendId.get(iframeBackendId);
|
||||
const projection =
|
||||
parentContext.projection && iframeNode?.localRect
|
||||
? childFrameProjection(parentContext.projection, iframeNode.localRect)
|
||||
: null;
|
||||
// DOMSnapshot reports the iframe owner's border-box bounds, but child
|
||||
// document coordinates start at the content-box origin. Preserve the
|
||||
// frame-local rectangles here and let frame-capture normalize them with
|
||||
// authoritative content quads once the frame graph is available.
|
||||
const projection = null;
|
||||
|
||||
const childContext: FrameContext = {
|
||||
frameId: snapshotFrameId(childDoc, strings),
|
||||
@@ -1065,7 +1062,6 @@ function parseChildFrameDocuments(
|
||||
strings,
|
||||
dpr,
|
||||
childDocIndex,
|
||||
parsed.nodes,
|
||||
childContext,
|
||||
nextVisited,
|
||||
);
|
||||
@@ -1143,7 +1139,7 @@ export async function captureViewModel(
|
||||
const nodes = mainParsed.nodes;
|
||||
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
|
||||
|
||||
const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext);
|
||||
const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, topContext);
|
||||
const iframeNodes = frameParsed.iframeNodes;
|
||||
for (const id of frameParsed.excludedBackendNodeIds) {
|
||||
excludedBackendNodeIds.add(id);
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import { resolveFrameProjection } from "../frame-geometry";
|
||||
import { type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import { createFrameGeometryContext } from "../frame-geometry";
|
||||
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
|
||||
import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
|
||||
import {
|
||||
@@ -31,21 +30,6 @@ async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFra
|
||||
}
|
||||
}
|
||||
|
||||
function transformFrameNodes(
|
||||
nodes: CapturedNode[],
|
||||
projection: GeometryProjection | null,
|
||||
): CapturedNode[] {
|
||||
return nodes.map((node) => ({
|
||||
...node,
|
||||
rect: projection
|
||||
? (() => {
|
||||
const bounds = projectRectToViewport(node.rect, projection);
|
||||
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||
})()
|
||||
: null,
|
||||
}));
|
||||
}
|
||||
|
||||
async function captureMissingFrameDocuments(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
@@ -63,16 +47,6 @@ async function captureMissingFrameDocuments(
|
||||
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
|
||||
signal,
|
||||
});
|
||||
let projection: GeometryProjection | null = null;
|
||||
try {
|
||||
projection = await resolveFrameProjection(cdp, graph, frame.frameId);
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
console.debug("[bsk observation] child frame geometry projection failed", {
|
||||
frameId: frame.frameId,
|
||||
err,
|
||||
});
|
||||
}
|
||||
const rootFrameId = child.rootFrameId ?? frame.frameId;
|
||||
const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
|
||||
if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
|
||||
@@ -83,7 +57,7 @@ async function captureMissingFrameDocuments(
|
||||
);
|
||||
}
|
||||
for (const [childFrameId, nodes] of childFrames) {
|
||||
captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
|
||||
captured.frameNodes.set(childFrameId, nodes);
|
||||
}
|
||||
if (frame.ownerBackendNodeId !== undefined) {
|
||||
captured.iframeNodes.set(
|
||||
@@ -109,6 +83,56 @@ async function captureMissingFrameDocuments(
|
||||
}
|
||||
}
|
||||
|
||||
async function normalizeFrameLocalRects(
|
||||
cdp: CdpRunner,
|
||||
graph: CdpFrameGraph,
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
if (!captured.frameNodes) return;
|
||||
const geometry = createFrameGeometryContext(cdp, graph);
|
||||
const frameById = new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
||||
await Promise.all(
|
||||
[...captured.frameNodes].map(async ([frameId, nodes]) => {
|
||||
if (frameId === graph.rootFrameId || !frameById.has(frameId)) return;
|
||||
try {
|
||||
const normalized = await Promise.all(
|
||||
nodes.map(async (node) => {
|
||||
if (!node.localRect) return { ...node, rect: null };
|
||||
const bounds = await geometry.projectFrameLocalRect(frameId, node.localRect);
|
||||
return {
|
||||
...node,
|
||||
rect: bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null,
|
||||
};
|
||||
}),
|
||||
);
|
||||
throwIfAborted(signal);
|
||||
captured.frameNodes?.set(frameId, normalized);
|
||||
const ownerBackendNodeId =
|
||||
frameById.get(frameId)?.ownerBackendNodeId ??
|
||||
captured.frameOwnerBackendNodeIds?.get(frameId);
|
||||
if (ownerBackendNodeId !== undefined) {
|
||||
captured.iframeNodes.set(ownerBackendNodeId, normalized);
|
||||
}
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
console.debug("[bsk observation] frame-local geometry normalization failed", {
|
||||
frameId,
|
||||
err,
|
||||
});
|
||||
const unavailable = nodes.map((node) => ({ ...node, rect: null }));
|
||||
captured.frameNodes?.set(frameId, unavailable);
|
||||
const ownerBackendNodeId =
|
||||
frameById.get(frameId)?.ownerBackendNodeId ??
|
||||
captured.frameOwnerBackendNodeIds?.get(frameId);
|
||||
if (ownerBackendNodeId !== undefined) {
|
||||
captured.iframeNodes.set(ownerBackendNodeId, unavailable);
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function captureAxTrees<T extends FrameAxNode>(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
@@ -174,7 +198,10 @@ export async function captureFrameData<T extends FrameAxNode>(
|
||||
): Promise<CapturedFrameDocument<T>[]> {
|
||||
const graph = await discoverFrameGraph(cdp, tabId);
|
||||
const frames = graph?.frames ?? [];
|
||||
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal);
|
||||
if (graph) {
|
||||
await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal);
|
||||
await normalizeFrameLocalRects(cdp, graph, captured, signal);
|
||||
}
|
||||
throwIfAborted(signal);
|
||||
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
|
||||
return buildFrameDocuments(graph, batches, captured);
|
||||
|
||||
@@ -1,8 +1,17 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
import type { RenderedSurface, RenderedSurfaceGroup } from "./types";
|
||||
import type { ClusteredRenderedSurfaces, RenderedSurface, RenderedSurfaceGroup } from "./types";
|
||||
|
||||
const STACK_IOU_RATIO = 0.9;
|
||||
const STACK_SIZE_RATIO = 0.9;
|
||||
const SIZE_BAND_BASE = 1 / STACK_SIZE_RATIO;
|
||||
const POSITION_CELL_RATIO = 0.1;
|
||||
const MAX_CLUSTER_COMPARISONS = 1_000_000;
|
||||
const MAX_CLUSTER_SURFACES = 50_000;
|
||||
|
||||
interface IndexedGroup extends RenderedSurfaceGroup {
|
||||
creationIndex: number;
|
||||
indexKey: string;
|
||||
}
|
||||
|
||||
function area(rect: Rect): number {
|
||||
return Math.max(0, rect.w) * Math.max(0, rect.h);
|
||||
@@ -14,9 +23,7 @@ function intersectionArea(a: Rect, b: Rect): number {
|
||||
return width * height;
|
||||
}
|
||||
|
||||
function sameVisualStack(a: RenderedSurface, b: RenderedSurface): boolean {
|
||||
if (a.frameId !== b.frameId) return false;
|
||||
if (a.parentBackendNodeId !== b.parentBackendNodeId) return false;
|
||||
function stackScore(a: RenderedSurface, b: RenderedSurface): number | null {
|
||||
const aArea = area(a.visibleRect);
|
||||
const bArea = area(b.visibleRect);
|
||||
const intersection = intersectionArea(a.visibleRect, b.visibleRect);
|
||||
@@ -25,41 +32,158 @@ function sameVisualStack(a: RenderedSurface, b: RenderedSurface): boolean {
|
||||
Math.min(a.visibleRect.w, b.visibleRect.w) / Math.max(a.visibleRect.w, b.visibleRect.w);
|
||||
const heightRatio =
|
||||
Math.min(a.visibleRect.h, b.visibleRect.h) / Math.max(a.visibleRect.h, b.visibleRect.h);
|
||||
return (
|
||||
union > 0 &&
|
||||
intersection / union >= STACK_IOU_RATIO &&
|
||||
widthRatio >= STACK_SIZE_RATIO &&
|
||||
heightRatio >= STACK_SIZE_RATIO
|
||||
);
|
||||
const iou = union > 0 ? intersection / union : 0;
|
||||
return iou >= STACK_IOU_RATIO && widthRatio >= STACK_SIZE_RATIO && heightRatio >= STACK_SIZE_RATIO
|
||||
? iou
|
||||
: null;
|
||||
}
|
||||
|
||||
function toGroup(members: RenderedSurface[]): RenderedSurfaceGroup {
|
||||
const representative = [...members].sort(
|
||||
(a, b) => area(b.visibleRect) - area(a.visibleRect) || b.paintOrder - a.paintOrder,
|
||||
)[0];
|
||||
function sizeBand(value: number): number {
|
||||
return Math.floor(Math.log(Math.max(Number.EPSILON, value)) / Math.log(SIZE_BAND_BASE));
|
||||
}
|
||||
|
||||
function bandSize(band: number): number {
|
||||
return SIZE_BAND_BASE ** band;
|
||||
}
|
||||
|
||||
function spatialKey(rect: Rect): string {
|
||||
const widthBand = sizeBand(rect.w);
|
||||
const heightBand = sizeBand(rect.h);
|
||||
const cellWidth = bandSize(widthBand) * POSITION_CELL_RATIO;
|
||||
const cellHeight = bandSize(heightBand) * POSITION_CELL_RATIO;
|
||||
const centerX = rect.x + rect.w / 2;
|
||||
const centerY = rect.y + rect.h / 2;
|
||||
return `${widthBand}:${heightBand}:${Math.floor(centerX / cellWidth)}:${Math.floor(centerY / cellHeight)}`;
|
||||
}
|
||||
|
||||
function candidateKeys(rect: Rect): string[] {
|
||||
const keys: string[] = [];
|
||||
const centerX = rect.x + rect.w / 2;
|
||||
const centerY = rect.y + rect.h / 2;
|
||||
const widthBand = sizeBand(rect.w);
|
||||
const heightBand = sizeBand(rect.h);
|
||||
// A >=0.9 size ratio crosses at most one logarithmic band. IoU >=0.9
|
||||
// likewise keeps centers within one 10%-of-size position cell.
|
||||
for (let dw = -1; dw <= 1; dw += 1) {
|
||||
for (let dh = -1; dh <= 1; dh += 1) {
|
||||
const candidateWidthBand = widthBand + dw;
|
||||
const candidateHeightBand = heightBand + dh;
|
||||
const cellWidth = bandSize(candidateWidthBand) * POSITION_CELL_RATIO;
|
||||
const cellHeight = bandSize(candidateHeightBand) * POSITION_CELL_RATIO;
|
||||
const cellX = Math.floor(centerX / cellWidth);
|
||||
const cellY = Math.floor(centerY / cellHeight);
|
||||
for (let dx = -1; dx <= 1; dx += 1) {
|
||||
for (let dy = -1; dy <= 1; dy += 1) {
|
||||
keys.push(`${candidateWidthBand}:${candidateHeightBand}:${cellX + dx}:${cellY + dy}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
function preferredRepresentative(a: RenderedSurface, b: RenderedSurface): RenderedSurface {
|
||||
const aArea = area(a.visibleRect);
|
||||
const bArea = area(b.visibleRect);
|
||||
return bArea > aArea || (bArea === aArea && b.paintOrder > a.paintOrder) ? b : a;
|
||||
}
|
||||
|
||||
function partitionKey(surface: RenderedSurface): string {
|
||||
return `${surface.frameId}\u0000${surface.parentBackendNodeId ?? "root"}`;
|
||||
}
|
||||
|
||||
export function clusterRenderedSurfaces(surfaces: RenderedSurface[]): ClusteredRenderedSurfaces {
|
||||
const partitions = new Map<string, Map<string, Set<IndexedGroup>>>();
|
||||
const groups: IndexedGroup[] = [];
|
||||
let comparisons = 0;
|
||||
let processed = 0;
|
||||
let truncated = false;
|
||||
|
||||
for (const surface of surfaces) {
|
||||
if (processed >= MAX_CLUSTER_SURFACES || comparisons >= MAX_CLUSTER_COMPARISONS) {
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
const key = partitionKey(surface);
|
||||
let index = partitions.get(key);
|
||||
if (!index) {
|
||||
index = new Map();
|
||||
partitions.set(key, index);
|
||||
}
|
||||
const candidates = new Set<IndexedGroup>();
|
||||
for (const candidateKey of candidateKeys(surface.visibleRect)) {
|
||||
for (const candidate of index.get(candidateKey) ?? []) candidates.add(candidate);
|
||||
}
|
||||
|
||||
let best: { group: IndexedGroup; score: number; areaDelta: number } | null = null;
|
||||
for (const group of candidates) {
|
||||
comparisons += 1;
|
||||
if (comparisons > MAX_CLUSTER_COMPARISONS) {
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
const score = stackScore(group.representative, surface);
|
||||
if (score === null) continue;
|
||||
const areaDelta = Math.abs(
|
||||
area(group.representative.visibleRect) - area(surface.visibleRect),
|
||||
);
|
||||
if (
|
||||
!best ||
|
||||
score > best.score ||
|
||||
(score === best.score && areaDelta < best.areaDelta) ||
|
||||
(score === best.score &&
|
||||
areaDelta === best.areaDelta &&
|
||||
group.creationIndex < best.group.creationIndex)
|
||||
) {
|
||||
best = { group, score, areaDelta };
|
||||
}
|
||||
}
|
||||
if (truncated) break;
|
||||
|
||||
if (best) {
|
||||
const group = best.group;
|
||||
const representative = preferredRepresentative(group.representative, surface);
|
||||
group.memberCount += 1;
|
||||
group.label ??= surface.label;
|
||||
if (representative !== group.representative) {
|
||||
index.get(group.indexKey)?.delete(group);
|
||||
group.representative = representative;
|
||||
group.indexKey = spatialKey(representative.visibleRect);
|
||||
const indexed = index.get(group.indexKey) ?? new Set<IndexedGroup>();
|
||||
indexed.add(group);
|
||||
index.set(group.indexKey, indexed);
|
||||
}
|
||||
processed += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
const indexKey = spatialKey(surface.visibleRect);
|
||||
const group: IndexedGroup = {
|
||||
frameId: surface.frameId,
|
||||
parentBackendNodeId: surface.parentBackendNodeId,
|
||||
representative: surface,
|
||||
...(surface.label ? { label: surface.label } : {}),
|
||||
memberCount: 1,
|
||||
creationIndex: groups.length,
|
||||
indexKey,
|
||||
};
|
||||
groups.push(group);
|
||||
const indexed = index.get(indexKey) ?? new Set<IndexedGroup>();
|
||||
indexed.add(group);
|
||||
index.set(indexKey, indexed);
|
||||
processed += 1;
|
||||
}
|
||||
|
||||
return {
|
||||
frameId: representative.frameId,
|
||||
members,
|
||||
representative,
|
||||
label: members.map((surface) => surface.label).find((label) => label !== undefined),
|
||||
groups: groups
|
||||
.map(({ creationIndex: _creationIndex, indexKey: _indexKey, ...group }) => group)
|
||||
.sort(
|
||||
(a, b) =>
|
||||
a.frameId.localeCompare(b.frameId) ||
|
||||
a.representative.visibleRect.y - b.representative.visibleRect.y ||
|
||||
a.representative.visibleRect.x - b.representative.visibleRect.x,
|
||||
),
|
||||
truncated,
|
||||
...(truncated ? { omittedCount: surfaces.length - processed } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function clusterRenderedSurfaces(surfaces: RenderedSurface[]): RenderedSurfaceGroup[] {
|
||||
const groups: RenderedSurface[][] = [];
|
||||
for (const surface of surfaces) {
|
||||
const matching = groups.find((group) =>
|
||||
group.some((member) => sameVisualStack(member, surface)),
|
||||
);
|
||||
if (matching) matching.push(surface);
|
||||
else groups.push([surface]);
|
||||
}
|
||||
return groups
|
||||
.map(toGroup)
|
||||
.sort(
|
||||
(a, b) =>
|
||||
a.frameId.localeCompare(b.frameId) ||
|
||||
a.representative.visibleRect.y - b.representative.visibleRect.y ||
|
||||
a.representative.visibleRect.x - b.representative.visibleRect.x,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
export { clusterRenderedSurfaces } from "./cluster";
|
||||
export { discoverRenderedSurfaces } from "./discovery";
|
||||
export { projectRenderedSurfaces } from "./project";
|
||||
export type { RenderedSurface, RenderedSurfaceGroup } from "./types";
|
||||
export type {
|
||||
ClusteredRenderedSurfaces,
|
||||
RenderedSurface,
|
||||
RenderedSurfaceGroup,
|
||||
} from "./types";
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { VomScene, VomVisualSurface } from "@browser-skill/vom";
|
||||
import type { CapturedFrameDocument } from "../frame-capture";
|
||||
import type { SemanticAxNode } from "../semantic-graph";
|
||||
import type { RenderedSurfaceGroup } from "./types";
|
||||
import type { ClusteredRenderedSurfaces, RenderedSurfaceGroup } from "./types";
|
||||
|
||||
function frameBackendKey(frameId: string, backendNodeId: number): string {
|
||||
return `${frameId}\u0000${backendNodeId}`;
|
||||
@@ -75,12 +75,12 @@ function buildProjectionIndex(
|
||||
export function projectRenderedSurfaces(
|
||||
scene: VomScene,
|
||||
documents: CapturedFrameDocument<SemanticAxNode>[],
|
||||
groups: RenderedSurfaceGroup[],
|
||||
clustered: ClusteredRenderedSurfaces,
|
||||
): VomScene {
|
||||
const index = buildProjectionIndex(scene, documents);
|
||||
const visualSurfaces: VomVisualSurface[] = [];
|
||||
|
||||
for (const group of groups) {
|
||||
for (const group of clustered.groups) {
|
||||
const parentId = nearestSceneParent(index, group);
|
||||
visualSurfaces.push({
|
||||
parentId,
|
||||
@@ -89,12 +89,20 @@ export function projectRenderedSurfaces(
|
||||
renderingKind: "canvas",
|
||||
visibleRect: group.representative.visibleRect,
|
||||
...(group.label ? { label: group.label } : {}),
|
||||
memberCount: group.members.length,
|
||||
memberCount: group.memberCount,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
...scene,
|
||||
...(visualSurfaces.length > 0 ? { visualSurfaces } : {}),
|
||||
...(clustered.truncated
|
||||
? {
|
||||
visualSurfacesTruncated: true,
|
||||
...(clustered.omittedCount !== undefined
|
||||
? { omittedVisualSurfaceCount: clustered.omittedCount }
|
||||
: {}),
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -12,7 +12,14 @@ export interface RenderedSurface {
|
||||
|
||||
export interface RenderedSurfaceGroup {
|
||||
frameId: string;
|
||||
members: RenderedSurface[];
|
||||
parentBackendNodeId: number | null;
|
||||
representative: RenderedSurface;
|
||||
label?: string;
|
||||
memberCount: number;
|
||||
}
|
||||
|
||||
export interface ClusteredRenderedSurfaces {
|
||||
groups: RenderedSurfaceGroup[];
|
||||
truncated: boolean;
|
||||
omittedCount?: number;
|
||||
}
|
||||
|
||||
@@ -95,6 +95,11 @@ Use the normal DOM buttons and re-observe after each action:
|
||||
- After **Restore size**: present with a fresh ref.
|
||||
- After **Remove canvas**: absent.
|
||||
|
||||
### `sparklines.html`
|
||||
|
||||
Expected Surface labels `fixture-sparkline-01` through `fixture-sparkline-20` must all be present.
|
||||
The adjacent, non-overlapping canvases remain twenty independent screenshot-only Surface refs.
|
||||
|
||||
## Invariants
|
||||
|
||||
- `snapshot` must not emit any of these Surface markers.
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Rendered Surface sparkline matrix</title>
|
||||
<style>
|
||||
html, body { margin: 0; font: 14px system-ui, sans-serif; }
|
||||
body { padding: 16px; }
|
||||
main { display: grid; grid-template-columns: repeat(5, 120px); gap: 12px; }
|
||||
canvas { display: block; width: 100px; height: 30px; background: #eaf2f8; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Twenty independent sparklines</h1>
|
||||
<main id="sparklines"></main>
|
||||
<script>
|
||||
const host = document.querySelector("#sparklines");
|
||||
for (let index = 1; index <= 20; index += 1) {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = 100;
|
||||
canvas.height = 30;
|
||||
canvas.setAttribute("aria-label", `fixture-sparkline-${String(index).padStart(2, "0")}`);
|
||||
host.append(canvas);
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -117,7 +117,7 @@ describe("renderVom single-layer page", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("bounds visual output and prefers explicitly labeled surfaces", () => {
|
||||
it("uses the appendix token budget and prefers explicitly labeled surfaces", () => {
|
||||
const input = scene([node({ id: 1, role: "RootWebArea", frameId: "main" })]);
|
||||
input.visualSurfaces = Array.from({ length: 12 }, (_, index) => ({
|
||||
parentId: 1,
|
||||
@@ -132,10 +132,30 @@ describe("renderVom single-layer page", () => {
|
||||
const out = renderVom(input);
|
||||
const surfaceRefs = out.refs.filter((ref) => ref.kind === "surface");
|
||||
|
||||
expect(surfaceRefs).toHaveLength(8);
|
||||
expect(surfaceRefs).toHaveLength(12);
|
||||
expect(surfaceRefs[0]).toMatchObject({ backendNodeId: 111, name: "Important status" });
|
||||
expect(out.text).toContain("… 4 additional visual surfaces omitted");
|
||||
expect(out.truncated).toBe(true);
|
||||
expect(out.truncated).toBe(false);
|
||||
});
|
||||
|
||||
it("renders twenty independent sparklines without changing DOM refs", () => {
|
||||
const input = scene([
|
||||
node({ id: 1, role: "RootWebArea", frameId: "main" }),
|
||||
node({ id: 2, parentId: 1, role: "button", name: "Submit", tag: "button" }),
|
||||
]);
|
||||
input.visualSurfaces = Array.from({ length: 20 }, (_, index) => ({
|
||||
parentId: 1,
|
||||
backendNodeId: 100 + index,
|
||||
frameId: "main",
|
||||
renderingKind: "canvas" as const,
|
||||
visibleRect: { x: index * 12, y: 20, w: 10, h: 8 },
|
||||
memberCount: 1,
|
||||
}));
|
||||
|
||||
const out = renderVom(input);
|
||||
|
||||
expect(out.refs[0]).toMatchObject({ ref: "e1", backendNodeId: 2, kind: "dom" });
|
||||
expect(out.refs.filter((ref) => ref.kind === "surface")).toHaveLength(20);
|
||||
expect(out.truncated).toBe(false);
|
||||
});
|
||||
|
||||
it("does not let a visual surface exhaust the budget before later DOM content", () => {
|
||||
|
||||
+43
-42
@@ -88,7 +88,8 @@ const MAX_CUSTOM_RECOVERY_AREA = 180_000;
|
||||
const MAX_RECOVERABLE_SINGLE_INTENT_AREA = 360_000;
|
||||
const MAX_HANDLE_CONTEXT_ITEMS = 3;
|
||||
const MAX_SURFACE_ITEMS = 12;
|
||||
const MAX_VISUAL_SURFACES = 8;
|
||||
const DEFAULT_VISUAL_APPENDIX_TOKENS = 2_048;
|
||||
const MAX_VISUAL_SURFACE_REFS = 512;
|
||||
const MAX_SCOPE_LINES = 40;
|
||||
const RECOVERY_HANDLER_ATTRS = ["onclick", "onmousedown", "onkeydown", "onkeyup", "onkeypress"];
|
||||
const NATIVE_TAGS = new Set(["button", "input", "select", "textarea"]);
|
||||
@@ -796,6 +797,16 @@ interface RenderState {
|
||||
surfaceMap: Map<number, CondSurface>;
|
||||
scopeMap: Map<number, ActiveScopeBlock>;
|
||||
visualSurfaces: VomVisualSurface[];
|
||||
visualSurfacesTruncated: boolean;
|
||||
omittedVisualSurfaceCount?: number;
|
||||
}
|
||||
|
||||
function tryAppendLine(state: RenderState, line: string, tokenLimit = state.maxTokens): boolean {
|
||||
const nextTokens = state.tokens + estimateTokens(line);
|
||||
if (nextTokens > tokenLimit) return false;
|
||||
state.lines.push(line);
|
||||
state.tokens = nextTokens;
|
||||
return true;
|
||||
}
|
||||
|
||||
function visualSurfaceArea(surface: VomVisualSurface): number {
|
||||
@@ -834,41 +845,35 @@ function selectVisualSurfaces(
|
||||
if (surface.parentId !== null && !nodesById.has(surface.parentId)) continue;
|
||||
eligibleCount += 1;
|
||||
|
||||
const insertionIndex = selected.findIndex(
|
||||
(candidate) => compareVisualSurfaces(surface, candidate) < 0,
|
||||
);
|
||||
if (insertionIndex < 0) {
|
||||
if (selected.length < MAX_VISUAL_SURFACES) selected.push(surface);
|
||||
continue;
|
||||
}
|
||||
selected.splice(insertionIndex, 0, surface);
|
||||
if (selected.length > MAX_VISUAL_SURFACES) selected.pop();
|
||||
selected.push(surface);
|
||||
}
|
||||
|
||||
selected.sort(compareVisualSurfaces);
|
||||
return { selected, eligibleCount };
|
||||
}
|
||||
|
||||
function emitVisualAppendix(state: RenderState): void {
|
||||
if (state.stopped) return;
|
||||
const { selected, eligibleCount } = selectVisualSurfaces(state.visualSurfaces, state.nodesById);
|
||||
if (eligibleCount === 0) return;
|
||||
if (eligibleCount === 0 && !state.visualSurfacesTruncated) return;
|
||||
|
||||
const header = "[visual surfaces]";
|
||||
const appendixStartTokens = state.tokens;
|
||||
const appendixTokenLimit = Number.isFinite(state.maxTokens)
|
||||
? state.maxTokens
|
||||
: appendixStartTokens + DEFAULT_VISUAL_APPENDIX_TOKENS;
|
||||
let headerEmitted = false;
|
||||
let emitted = 0;
|
||||
for (const surface of selected) {
|
||||
if (emitted >= MAX_VISUAL_SURFACE_REFS) break;
|
||||
const ref = `e${state.nextRef}`;
|
||||
const line = visualSurfaceLine(surface, ref);
|
||||
const nextTokens =
|
||||
state.tokens + estimateTokens(line) + (headerEmitted ? 0 : estimateTokens(header));
|
||||
if (nextTokens > state.maxTokens) break;
|
||||
const requiredTokens = estimateTokens(line) + (headerEmitted ? 0 : estimateTokens(header));
|
||||
if (state.tokens + requiredTokens > appendixTokenLimit) break;
|
||||
if (!headerEmitted) {
|
||||
state.lines.push(header);
|
||||
state.tokens += estimateTokens(header);
|
||||
if (!tryAppendLine(state, header, appendixTokenLimit)) break;
|
||||
headerEmitted = true;
|
||||
}
|
||||
state.lines.push(line);
|
||||
state.tokens += estimateTokens(line);
|
||||
if (!tryAppendLine(state, line, appendixTokenLimit)) break;
|
||||
const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`;
|
||||
state.refs.push({
|
||||
ref,
|
||||
@@ -885,16 +890,15 @@ function emitVisualAppendix(state: RenderState): void {
|
||||
emitted += 1;
|
||||
}
|
||||
|
||||
const omitted = eligibleCount - emitted;
|
||||
if (omitted > 0) {
|
||||
const locallyOmitted = eligibleCount - emitted;
|
||||
const upstreamOmitted = state.omittedVisualSurfaceCount ?? 0;
|
||||
const omitted = locallyOmitted + upstreamOmitted;
|
||||
if (omitted > 0 || state.visualSurfacesTruncated) {
|
||||
state.truncated = true;
|
||||
if (headerEmitted) {
|
||||
const summary = ` … ${omitted} additional visual surfaces omitted`;
|
||||
const nextTokens = state.tokens + estimateTokens(summary);
|
||||
if (nextTokens <= state.maxTokens) {
|
||||
state.lines.push(summary);
|
||||
state.tokens = nextTokens;
|
||||
}
|
||||
const count =
|
||||
omitted > 0 ? `${state.visualSurfacesTruncated ? "at least " : ""}${omitted}` : "some";
|
||||
tryAppendLine(state, ` … ${count} additional visual surfaces omitted`, appendixTokenLimit);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -903,28 +907,22 @@ function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: Ren
|
||||
if (state.stopped || scope.lines.length === 0) return;
|
||||
const indent = " ".repeat(depth);
|
||||
const header = `${indent}[§ active: ${scope.label}]`;
|
||||
const headerTokens = estimateTokens(header);
|
||||
if (state.tokens + headerTokens > state.maxTokens) {
|
||||
if (!tryAppendLine(state, header)) {
|
||||
state.truncated = true;
|
||||
state.stopped = true;
|
||||
return;
|
||||
}
|
||||
state.lines.push(header);
|
||||
state.tokens += headerTokens;
|
||||
|
||||
for (const text of scope.lines.slice(0, MAX_SCOPE_LINES)) {
|
||||
if (state.stopped) return;
|
||||
const clean = cleaned(text);
|
||||
if (!clean) continue;
|
||||
const line = `${indent} ${clean}`;
|
||||
const nextTokens = state.tokens + estimateTokens(line);
|
||||
if (nextTokens > state.maxTokens) {
|
||||
if (!tryAppendLine(state, line)) {
|
||||
state.truncated = true;
|
||||
state.stopped = true;
|
||||
return;
|
||||
}
|
||||
state.lines.push(line);
|
||||
state.tokens = nextTokens;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -959,15 +957,11 @@ function renderTree(children: Map<number | null, VomNode[]>, state: RenderState)
|
||||
const context = ref ? handleContext(node, state) : [];
|
||||
const surface = state.surfaceMap.get(node.id);
|
||||
const line = renderNodeLine(node, depth, ref, context, surface, state.redactValues);
|
||||
const nextTokens = state.tokens + estimateTokens(line);
|
||||
if (nextTokens > state.maxTokens) {
|
||||
if (!tryAppendLine(state, line)) {
|
||||
state.truncated = true;
|
||||
state.stopped = true;
|
||||
break;
|
||||
}
|
||||
|
||||
state.lines.push(line);
|
||||
state.tokens = nextTokens;
|
||||
if (ref) {
|
||||
const name = cleaned(node.name);
|
||||
state.refs.push({
|
||||
@@ -1001,6 +995,8 @@ function renderNodes(
|
||||
surfaces: CondSurface[] = [],
|
||||
activeScopeBlocks: ActiveScopeBlock[] = [],
|
||||
visualSurfaces: VomVisualSurface[] = [],
|
||||
visualSurfacesTruncated = false,
|
||||
omittedVisualSurfaceCount?: number,
|
||||
): RenderState {
|
||||
const children = buildChildren(nodes);
|
||||
const state: RenderState = {
|
||||
@@ -1021,6 +1017,8 @@ function renderNodes(
|
||||
surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])),
|
||||
scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])),
|
||||
visualSurfaces,
|
||||
visualSurfacesTruncated,
|
||||
omittedVisualSurfaceCount,
|
||||
};
|
||||
|
||||
renderTree(children, state);
|
||||
@@ -1241,10 +1239,11 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
|
||||
scene.surfaces,
|
||||
scene.activeScopeBlocks,
|
||||
scene.visualSurfaces?.filter((surface) => visibleSurface(surface.parentId)),
|
||||
scene.visualSurfacesTruncated,
|
||||
scene.omittedVisualSurfaceCount,
|
||||
);
|
||||
const occlusionLine = renderPageOcclusionLine(hiddenCount);
|
||||
state.lines.push(occlusionLine);
|
||||
state.tokens += estimateTokens(occlusionLine);
|
||||
if (!tryAppendLine(state, occlusionLine)) state.truncated = true;
|
||||
emitVisualAppendix(state);
|
||||
|
||||
return {
|
||||
@@ -1273,6 +1272,8 @@ export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult
|
||||
scene.surfaces,
|
||||
scene.activeScopeBlocks,
|
||||
scene.visualSurfaces,
|
||||
scene.visualSurfacesTruncated,
|
||||
scene.omittedVisualSurfaceCount,
|
||||
);
|
||||
emitVisualAppendix(state);
|
||||
|
||||
|
||||
@@ -63,6 +63,10 @@ export interface VomScene {
|
||||
activeScopeBlocks?: ActiveScopeBlock[];
|
||||
/** Non-semantic rendered regions discovered alongside the AX/DOM tree. */
|
||||
visualSurfaces?: VomVisualSurface[];
|
||||
/** Upstream surface discovery/aggregation stopped at a resource guardrail. */
|
||||
visualSurfacesTruncated?: boolean;
|
||||
/** Exact omitted count when upstream could determine it. */
|
||||
omittedVisualSurfaceCount?: number;
|
||||
}
|
||||
|
||||
export interface VomOptions {
|
||||
|
||||
Reference in New Issue
Block a user