mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix(vom): bug fix
This commit is contained in:
@@ -36,6 +36,14 @@ function fakeAgentWindow(ids: number[]) {
|
||||
const TINY_PNG =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABh6FO1AAAAABJRU5ErkJggg==";
|
||||
|
||||
function pngWithIhdrDimensions(width: number, height: number): string {
|
||||
const bytes = Uint8Array.from(atob(TINY_PNG), (character) => character.charCodeAt(0));
|
||||
const view = new DataView(bytes.buffer);
|
||||
view.setUint32(16, width);
|
||||
view.setUint32(20, height);
|
||||
return btoa(String.fromCharCode(...bytes));
|
||||
}
|
||||
|
||||
function makeScreenshotDeps(
|
||||
opts: {
|
||||
cdp?: CdpRunner;
|
||||
@@ -86,6 +94,7 @@ describe("stripDataUrlPrefix", () => {
|
||||
describe("parsePngDimensions", () => {
|
||||
it("parses width/height from the IHDR chunk", () => {
|
||||
expect(parsePngDimensions(TINY_PNG)).toEqual({ width: 1, height: 1 });
|
||||
expect(parsePngDimensions(pngWithIhdrDimensions(100, 80))).toEqual({ width: 100, height: 80 });
|
||||
});
|
||||
it("returns null on non-PNG input", () => {
|
||||
expect(parsePngDimensions("not-a-png-payload-just-random-base64-text-zzzzzzzzz")).toBeNull();
|
||||
@@ -120,6 +129,20 @@ describe("handleScreenshot", () => {
|
||||
expect(get).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("fails closed when a full-tab capture is not a valid PNG", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const res = await handleScreenshot(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
makeScreenshotDeps({
|
||||
captureVisibleTab: vi.fn(async () => "data:image/png;base64,bm90LWEtcG5n"),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(res).toMatchObject({ code: "cdp_failed", message: /invalid PNG data/ });
|
||||
});
|
||||
|
||||
it("returns not_found when Agent Window has no active tab", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
@@ -349,7 +372,7 @@ describe("handleScreenshot", () => {
|
||||
cssLayoutViewport: { clientWidth: 800, clientHeight: 600 },
|
||||
}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[0, 0, 200, 0, 200, 100, 0, 100]] }),
|
||||
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
|
||||
"Page.captureScreenshot": () => ({ data: pngWithIhdrDimensions(100, 80) }),
|
||||
});
|
||||
|
||||
const res = await handleScreenshot(
|
||||
@@ -368,6 +391,95 @@ describe("handleScreenshot", () => {
|
||||
clip?: { x: number; y: number; width: number; height: number };
|
||||
};
|
||||
expect(clip.clip).toMatchObject({ x: 25, y: 30, width: 50, height: 40 });
|
||||
expect(res).toMatchObject({ width: 100, height: 80 });
|
||||
});
|
||||
|
||||
it("routes OOPIF surface geometry through its child session and captures top coordinates", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e5", 999, {
|
||||
tabId: 7,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
kind: "surface",
|
||||
visibleRect: { x: 224, y: 346, w: 200, h: 80 },
|
||||
});
|
||||
const { cdp, sent } = makeFakeCdp({
|
||||
"DOM.getBoxModel": () => ({
|
||||
model: { content: [204, 306, 604, 306, 604, 506, 204, 506] },
|
||||
}),
|
||||
"Page.getLayoutMetrics": () => ({
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
|
||||
}),
|
||||
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
|
||||
});
|
||||
cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 7 } },
|
||||
{
|
||||
frameId: "child-frame",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 99,
|
||||
target: { tabId: 7, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const targetCalls: Array<{ sessionId?: string; method: string }> = [];
|
||||
cdp.sendToTarget = vi.fn(async (target, method) => {
|
||||
targetCalls.push({ sessionId: target.sessionId, method });
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
throw new Error(`unexpected child CDP call ${method}`);
|
||||
}) as CdpRunner["sendToTarget"];
|
||||
|
||||
const res = await handleScreenshot(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
|
||||
makeScreenshotDeps({
|
||||
cdp,
|
||||
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
|
||||
query: vi.fn(),
|
||||
captureVisibleTab: vi.fn(),
|
||||
}),
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(targetCalls).toEqual([
|
||||
{ sessionId: "child-session", method: "DOM.getContentQuads" },
|
||||
{ sessionId: "child-session", method: "Page.getLayoutMetrics" },
|
||||
]);
|
||||
const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
|
||||
clip?: { x: number; y: number; width: number; height: number };
|
||||
};
|
||||
expect(clip.clip).toMatchObject({ x: 224, y: 346, width: 200, height: 80 });
|
||||
});
|
||||
|
||||
it("fails closed when a ref capture is not a valid PNG", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e5", 999, { tabId: 7, kind: "surface" });
|
||||
const { cdp } = makeFakeCdp({
|
||||
"DOM.getContentQuads": () => ({ quads: [[0, 0, 50, 0, 50, 50, 0, 50]] }),
|
||||
"Page.captureScreenshot": () => ({ data: "bm90LWEtcG5n" }),
|
||||
});
|
||||
|
||||
const res = await handleScreenshot(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
|
||||
makeScreenshotDeps({
|
||||
cdp,
|
||||
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
|
||||
query: vi.fn(),
|
||||
captureVisibleTab: vi.fn(),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(res).toMatchObject({ code: "cdp_failed", message: /invalid PNG data/ });
|
||||
});
|
||||
|
||||
it("returns not_found for unknown ref", async () => {
|
||||
|
||||
@@ -116,8 +116,8 @@ export function stripDataUrlPrefix(dataUrl: string): string {
|
||||
|
||||
/**
|
||||
* Parse a PNG's IHDR chunk and return `(width, height)`. Returns
|
||||
* `null` on any malformed input so callers fall back to `0/0` instead
|
||||
* of throwing.
|
||||
* `null` on malformed input so callers can reject dimensions that are
|
||||
* not authoritative instead of guessing from a CSS-space rectangle.
|
||||
*
|
||||
* PNG layout: 8-byte signature, then a 4-byte length, 4-byte type
|
||||
* ("IHDR"), then the chunk data — width is bytes 16-19 BE, height is
|
||||
@@ -129,9 +129,9 @@ export function parsePngDimensions(base64: string): { width: number; height: num
|
||||
const head = base64.length > 64 ? base64.slice(0, 64) : base64;
|
||||
const bin = atob(head);
|
||||
if (bin.length < 24) return null;
|
||||
if (bin.charCodeAt(0) !== 0x89 || bin.charCodeAt(1) !== 0x50 || bin.charCodeAt(2) !== 0x4e) {
|
||||
return null;
|
||||
}
|
||||
const expectedSignature = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
|
||||
if (expectedSignature.some((byte, index) => bin.charCodeAt(index) !== byte)) return null;
|
||||
if (bin.slice(12, 16) !== "IHDR") return null;
|
||||
const u32 = (off: number) =>
|
||||
(bin.charCodeAt(off) << 24) |
|
||||
(bin.charCodeAt(off + 1) << 16) |
|
||||
@@ -250,10 +250,10 @@ async function captureElementScreenshot(
|
||||
if (!image_base64) {
|
||||
return { code: "cdp_failed", message: "Page.captureScreenshot returned no data" };
|
||||
}
|
||||
const dims = parsePngDimensions(image_base64) ?? {
|
||||
width: Math.round(rect.width),
|
||||
height: Math.round(rect.height),
|
||||
};
|
||||
const dims = parsePngDimensions(image_base64);
|
||||
if (!dims) {
|
||||
return { code: "cdp_failed", message: "Page.captureScreenshot returned invalid PNG data" };
|
||||
}
|
||||
return { image_base64, width: dims.width, height: dims.height };
|
||||
} catch (err) {
|
||||
return {
|
||||
@@ -402,7 +402,10 @@ export async function handleScreenshot(
|
||||
if (isRpcError(captured)) return captured;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const image_base64 = captured;
|
||||
const dims = parsePngDimensions(image_base64) ?? { width: 0, height: 0 };
|
||||
const dims = parsePngDimensions(image_base64);
|
||||
if (!dims) {
|
||||
return { code: "cdp_failed", message: "screenshot capture returned invalid PNG data" };
|
||||
}
|
||||
return withShotDialogs({
|
||||
image_base64,
|
||||
width: dims.width,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
|
||||
import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
|
||||
import { discoverRenderedSurfaces } from "../rendered-surfaces";
|
||||
|
||||
// Minimal but format-accurate captureSnapshot reply: a body with one
|
||||
// fixed full-screen overlay div carrying a password input.
|
||||
@@ -1188,6 +1189,97 @@ describe("captureViewModel", () => {
|
||||
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
|
||||
});
|
||||
|
||||
it("keeps same-origin iframe canvas discovery in top-level CSS coordinates at dpr 1 and 2", async () => {
|
||||
const captureAtDpr = async (dpr: number) => {
|
||||
const S = ["html", "body", "iframe", "canvas", "static", "auto"];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const scale = (values: number[]) => values.map((value) => value * dpr);
|
||||
const snapshot = {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "main",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("iframe")],
|
||||
backendNodeId: [10, 11, 13],
|
||||
attributes: [[], [], []],
|
||||
contentDocumentIndex: { index: [2], value: [1] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [scale([0, 0, 1000, 800]), scale([100, 150, 300, 200])],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
{
|
||||
frameId: "child",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("canvas")],
|
||||
backendNodeId: [20, 21],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [scale([0, 0, 300, 200]), scale([20, 30, 120, 60])],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const cdp = {
|
||||
send: vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
layoutViewport: { clientWidth: 1000 * dpr, clientHeight: 800 * dpr },
|
||||
};
|
||||
}
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const captured = await captureViewModel(cdp, 4);
|
||||
const childNodes = captured.frameNodes?.get("child") ?? [];
|
||||
return discoverRenderedSurfaces(
|
||||
[
|
||||
{
|
||||
frameId: "child",
|
||||
contextScopeId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 13,
|
||||
target: { tabId: 4 },
|
||||
domNodes: childNodes,
|
||||
axNodes: [],
|
||||
},
|
||||
],
|
||||
captured.viewport,
|
||||
);
|
||||
};
|
||||
|
||||
const atDpr1 = await captureAtDpr(1);
|
||||
const atDpr2 = await captureAtDpr(2);
|
||||
|
||||
expect(atDpr1).toEqual([
|
||||
expect.objectContaining({
|
||||
backendNodeId: 21,
|
||||
frameId: "child",
|
||||
visibleRect: { x: 120, y: 180, w: 120, h: 60 },
|
||||
}),
|
||||
]);
|
||||
expect(atDpr2).toEqual(atDpr1);
|
||||
});
|
||||
|
||||
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => {
|
||||
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
|
||||
@@ -47,8 +47,9 @@ describe("renderVom single-layer page", () => {
|
||||
];
|
||||
const out = renderVom(input);
|
||||
|
||||
expect(out.text).toContain("[visual surfaces]");
|
||||
expect(out.text).toContain(
|
||||
'@e1 surface "Data grid" [rendering=canvas; layers=2; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
|
||||
'@e1 surface "Data grid" [bounds=10,20,800,500; rendering=canvas; layers=2; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
|
||||
);
|
||||
expect(out.refs).toContainEqual(
|
||||
expect.objectContaining({
|
||||
@@ -77,7 +78,7 @@ describe("renderVom single-layer page", () => {
|
||||
const out = renderVom(input);
|
||||
|
||||
expect(out.text).toContain(
|
||||
'@e1 surface "canvas visual surface" [rendering=canvas; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
|
||||
'@e1 surface "canvas visual surface" [bounds=10,20,800,500; rendering=canvas; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
|
||||
);
|
||||
expect(out.refs).toContainEqual(
|
||||
expect.objectContaining({
|
||||
@@ -89,6 +90,78 @@ describe("renderVom single-layer page", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("appends visual refs after DOM refs so canvas density cannot renumber controls", () => {
|
||||
const input = scene([
|
||||
node({ id: 1, role: "RootWebArea", frameId: "main" }),
|
||||
node({ id: 2, parentId: 1, role: "button", name: "Submit", tag: "button" }),
|
||||
]);
|
||||
input.visualSurfaces = [
|
||||
{
|
||||
parentId: 1,
|
||||
backendNodeId: 99,
|
||||
frameId: "main",
|
||||
renderingKind: "canvas",
|
||||
visibleRect: { x: 10, y: 20, w: 80, h: 50 },
|
||||
memberCount: 1,
|
||||
},
|
||||
];
|
||||
|
||||
const withoutVisuals = renderVom({ ...input, visualSurfaces: [] });
|
||||
const withVisuals = renderVom(input);
|
||||
|
||||
expect(withoutVisuals.refs[0]).toMatchObject({ ref: "e1", backendNodeId: 2, kind: "dom" });
|
||||
expect(withVisuals.refs[0]).toMatchObject({ ref: "e1", backendNodeId: 2, kind: "dom" });
|
||||
expect(withVisuals.refs[1]).toMatchObject({ ref: "e2", backendNodeId: 99, kind: "surface" });
|
||||
expect(withVisuals.text.indexOf('@e1 button "Submit"')).toBeLessThan(
|
||||
withVisuals.text.indexOf("[visual surfaces]"),
|
||||
);
|
||||
});
|
||||
|
||||
it("bounds visual output and prefers explicitly labeled surfaces", () => {
|
||||
const input = scene([node({ id: 1, role: "RootWebArea", frameId: "main" })]);
|
||||
input.visualSurfaces = Array.from({ length: 12 }, (_, index) => ({
|
||||
parentId: 1,
|
||||
backendNodeId: 100 + index,
|
||||
frameId: "main",
|
||||
renderingKind: "canvas" as const,
|
||||
visibleRect: { x: index * 10, y: 20, w: 8, h: 8 },
|
||||
...(index === 11 ? { label: "Important status" } : {}),
|
||||
memberCount: 1,
|
||||
}));
|
||||
|
||||
const out = renderVom(input);
|
||||
const surfaceRefs = out.refs.filter((ref) => ref.kind === "surface");
|
||||
|
||||
expect(surfaceRefs).toHaveLength(8);
|
||||
expect(surfaceRefs[0]).toMatchObject({ backendNodeId: 111, name: "Important status" });
|
||||
expect(out.text).toContain("… 4 additional visual surfaces omitted");
|
||||
expect(out.truncated).toBe(true);
|
||||
});
|
||||
|
||||
it("does not let a visual surface exhaust the budget before later DOM content", () => {
|
||||
const input = scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Doc", frameId: "main" }),
|
||||
node({ id: 2, parentId: 1, role: "button", name: "First", tag: "button" }),
|
||||
node({ id: 3, parentId: 1, role: "button", name: "Later", tag: "button" }),
|
||||
]);
|
||||
input.visualSurfaces = [
|
||||
{
|
||||
parentId: 1,
|
||||
backendNodeId: 99,
|
||||
frameId: "main",
|
||||
renderingKind: "canvas",
|
||||
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
|
||||
memberCount: 1,
|
||||
},
|
||||
];
|
||||
|
||||
const out = renderVom(input, { maxTokens: 30 });
|
||||
|
||||
expect(out.text).toContain('@e2 button "Later"');
|
||||
expect(out.refs.filter((ref) => ref.kind === "surface")).toHaveLength(0);
|
||||
expect(out.truncated).toBe(true);
|
||||
});
|
||||
|
||||
it("does not derive handle context across frame scopes", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
|
||||
+91
-43
@@ -88,6 +88,7 @@ const MAX_CUSTOM_RECOVERY_AREA = 180_000;
|
||||
const MAX_RECOVERABLE_SINGLE_INTENT_AREA = 360_000;
|
||||
const MAX_HANDLE_CONTEXT_ITEMS = 3;
|
||||
const MAX_SURFACE_ITEMS = 12;
|
||||
const MAX_VISUAL_SURFACES = 8;
|
||||
const MAX_SCOPE_LINES = 40;
|
||||
const RECOVERY_HANDLER_ATTRS = ["onclick", "onmousedown", "onkeydown", "onkeyup", "onkeypress"];
|
||||
const NATIVE_TAGS = new Set(["button", "input", "select", "textarea"]);
|
||||
@@ -794,40 +795,81 @@ interface RenderState {
|
||||
duplicateRefNames: Set<string>;
|
||||
surfaceMap: Map<number, CondSurface>;
|
||||
scopeMap: Map<number, ActiveScopeBlock>;
|
||||
visualSurfaces: Map<number | null, VomVisualSurface[]>;
|
||||
visualSurfaces: VomVisualSurface[];
|
||||
}
|
||||
|
||||
function groupedByParent<T extends { parentId: number | null }>(
|
||||
items: T[],
|
||||
): Map<number | null, T[]> {
|
||||
const grouped = new Map<number | null, T[]>();
|
||||
for (const item of items) {
|
||||
const siblings = grouped.get(item.parentId) ?? [];
|
||||
siblings.push(item);
|
||||
grouped.set(item.parentId, siblings);
|
||||
}
|
||||
return grouped;
|
||||
function visualSurfaceArea(surface: VomVisualSurface): number {
|
||||
return Math.max(0, surface.visibleRect.w) * Math.max(0, surface.visibleRect.h);
|
||||
}
|
||||
|
||||
function emitVisualEntries(parentId: number | null, depth: number, state: RenderState): void {
|
||||
if (state.stopped) return;
|
||||
for (const surface of state.visualSurfaces.get(parentId) ?? []) {
|
||||
if (depth > state.maxDepth) {
|
||||
state.truncated = true;
|
||||
function compareVisualSurfaces(a: VomVisualSurface, b: VomVisualSurface): number {
|
||||
const aLabeled = cleaned(a.label) !== undefined;
|
||||
const bLabeled = cleaned(b.label) !== undefined;
|
||||
return (
|
||||
Number(bLabeled) - Number(aLabeled) ||
|
||||
visualSurfaceArea(b) - visualSurfaceArea(a) ||
|
||||
a.frameId.localeCompare(b.frameId) ||
|
||||
a.visibleRect.y - b.visibleRect.y ||
|
||||
a.visibleRect.x - b.visibleRect.x ||
|
||||
a.backendNodeId - b.backendNodeId
|
||||
);
|
||||
}
|
||||
|
||||
function visualSurfaceLine(surface: VomVisualSurface, ref: string): string {
|
||||
const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`;
|
||||
const layers = surface.memberCount > 1 ? `; layers=${surface.memberCount}` : "";
|
||||
const { x, y, w, h } = surface.visibleRect;
|
||||
const bounds = [x, y, w, h].map(Math.round).join(",");
|
||||
return ` @${ref} surface ${JSON.stringify(label)} [bounds=${bounds}; rendering=${surface.renderingKind}${layers}; visual-only; requires=image-understanding; use: bsk screenshot --ref @${ref}]`;
|
||||
}
|
||||
|
||||
function selectVisualSurfaces(
|
||||
surfaces: readonly VomVisualSurface[],
|
||||
nodesById: ReadonlyMap<number, VomNode>,
|
||||
): { selected: VomVisualSurface[]; eligibleCount: number } {
|
||||
const selected: VomVisualSurface[] = [];
|
||||
let eligibleCount = 0;
|
||||
|
||||
for (const surface of surfaces) {
|
||||
if (surface.parentId !== null && !nodesById.has(surface.parentId)) continue;
|
||||
eligibleCount += 1;
|
||||
|
||||
const insertionIndex = selected.findIndex(
|
||||
(candidate) => compareVisualSurfaces(surface, candidate) < 0,
|
||||
);
|
||||
if (insertionIndex < 0) {
|
||||
if (selected.length < MAX_VISUAL_SURFACES) selected.push(surface);
|
||||
continue;
|
||||
}
|
||||
selected.splice(insertionIndex, 0, surface);
|
||||
if (selected.length > MAX_VISUAL_SURFACES) selected.pop();
|
||||
}
|
||||
|
||||
return { selected, eligibleCount };
|
||||
}
|
||||
|
||||
function emitVisualAppendix(state: RenderState): void {
|
||||
if (state.stopped) return;
|
||||
const { selected, eligibleCount } = selectVisualSurfaces(state.visualSurfaces, state.nodesById);
|
||||
if (eligibleCount === 0) return;
|
||||
|
||||
const header = "[visual surfaces]";
|
||||
let headerEmitted = false;
|
||||
let emitted = 0;
|
||||
for (const surface of selected) {
|
||||
const ref = `e${state.nextRef}`;
|
||||
const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`;
|
||||
const layers = surface.memberCount > 1 ? `; layers=${surface.memberCount}` : "";
|
||||
const line = `${" ".repeat(depth)}@${ref} surface ${JSON.stringify(label)} [rendering=${surface.renderingKind}${layers}; visual-only; requires=image-understanding; use: bsk screenshot --ref @${ref}]`;
|
||||
const nextTokens = state.tokens + estimateTokens(line);
|
||||
if (nextTokens > state.maxTokens) {
|
||||
state.truncated = true;
|
||||
state.stopped = true;
|
||||
return;
|
||||
const line = visualSurfaceLine(surface, ref);
|
||||
const nextTokens =
|
||||
state.tokens + estimateTokens(line) + (headerEmitted ? 0 : estimateTokens(header));
|
||||
if (nextTokens > state.maxTokens) break;
|
||||
if (!headerEmitted) {
|
||||
state.lines.push(header);
|
||||
state.tokens += estimateTokens(header);
|
||||
headerEmitted = true;
|
||||
}
|
||||
state.lines.push(line);
|
||||
state.tokens = nextTokens;
|
||||
state.tokens += estimateTokens(line);
|
||||
const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`;
|
||||
state.refs.push({
|
||||
ref,
|
||||
backendNodeId: surface.backendNodeId,
|
||||
@@ -840,6 +882,20 @@ function emitVisualEntries(parentId: number | null, depth: number, state: Render
|
||||
line: state.lines.length - 1,
|
||||
});
|
||||
state.nextRef += 1;
|
||||
emitted += 1;
|
||||
}
|
||||
|
||||
const omitted = eligibleCount - emitted;
|
||||
if (omitted > 0) {
|
||||
state.truncated = true;
|
||||
if (headerEmitted) {
|
||||
const summary = ` … ${omitted} additional visual surfaces omitted`;
|
||||
const nextTokens = state.tokens + estimateTokens(summary);
|
||||
if (nextTokens <= state.maxTokens) {
|
||||
state.lines.push(summary);
|
||||
state.tokens = nextTokens;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -872,34 +928,24 @@ function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: Ren
|
||||
}
|
||||
}
|
||||
|
||||
type RenderTask =
|
||||
| { kind: "node"; node: VomNode; depth: number }
|
||||
| { kind: "visual"; parentId: number | null; depth: number };
|
||||
|
||||
function pushRenderChildren(
|
||||
stack: RenderTask[],
|
||||
stack: Array<{ node: VomNode; depth: number }>,
|
||||
children: readonly VomNode[],
|
||||
depth: number,
|
||||
): void {
|
||||
for (let index = children.length - 1; index >= 0; index -= 1) {
|
||||
stack.push({ kind: "node", node: children[index], depth });
|
||||
stack.push({ node: children[index], depth });
|
||||
}
|
||||
}
|
||||
|
||||
function renderTree(children: Map<number | null, VomNode[]>, state: RenderState): void {
|
||||
const stack: RenderTask[] = [{ kind: "visual", parentId: null, depth: 1 }];
|
||||
const stack: Array<{ node: VomNode; depth: number }> = [];
|
||||
pushRenderChildren(stack, children.get(null) ?? [], 1);
|
||||
|
||||
while (stack.length > 0 && !state.stopped) {
|
||||
const task = stack.pop() as RenderTask;
|
||||
if (task.kind === "visual") {
|
||||
emitVisualEntries(task.parentId, task.depth, state);
|
||||
continue;
|
||||
}
|
||||
const { node, depth } = task;
|
||||
const { node, depth } = stack.pop() as { node: VomNode; depth: number };
|
||||
|
||||
if (!shouldRender(node)) {
|
||||
stack.push({ kind: "visual", parentId: node.id, depth });
|
||||
pushRenderChildren(stack, children.get(node.id) ?? [], depth);
|
||||
continue;
|
||||
}
|
||||
@@ -942,10 +988,8 @@ function renderTree(children: Map<number | null, VomNode[]>, state: RenderState)
|
||||
if (scope) emitActiveScopeBlock(scope, depth + 1, state);
|
||||
|
||||
if ((ref && shouldSkipRedundantRefChildren(node)) || shouldSkipRedundantChildren(node, state)) {
|
||||
emitVisualEntries(node.id, depth + 1, state);
|
||||
continue;
|
||||
}
|
||||
stack.push({ kind: "visual", parentId: node.id, depth: depth + 1 });
|
||||
pushRenderChildren(stack, children.get(node.id) ?? [], depth + 1);
|
||||
}
|
||||
}
|
||||
@@ -976,7 +1020,7 @@ function renderNodes(
|
||||
duplicateRefNames: duplicateReferenceNames(nodes),
|
||||
surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])),
|
||||
scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])),
|
||||
visualSurfaces: groupedByParent(visualSurfaces),
|
||||
visualSurfaces,
|
||||
};
|
||||
|
||||
renderTree(children, state);
|
||||
@@ -1198,7 +1242,10 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
|
||||
scene.activeScopeBlocks,
|
||||
scene.visualSurfaces?.filter((surface) => visibleSurface(surface.parentId)),
|
||||
);
|
||||
state.lines.push(renderPageOcclusionLine(hiddenCount));
|
||||
const occlusionLine = renderPageOcclusionLine(hiddenCount);
|
||||
state.lines.push(occlusionLine);
|
||||
state.tokens += estimateTokens(occlusionLine);
|
||||
emitVisualAppendix(state);
|
||||
|
||||
return {
|
||||
text: state.lines.join("\n"),
|
||||
@@ -1227,6 +1274,7 @@ export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult
|
||||
scene.activeScopeBlocks,
|
||||
scene.visualSurfaces,
|
||||
);
|
||||
emitVisualAppendix(state);
|
||||
|
||||
return {
|
||||
text: state.lines.join("\n"),
|
||||
|
||||
Reference in New Issue
Block a user