mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix(vom): bug fix
This commit is contained in:
@@ -36,6 +36,14 @@ function fakeAgentWindow(ids: number[]) {
|
|||||||
const TINY_PNG =
|
const TINY_PNG =
|
||||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABh6FO1AAAAABJRU5ErkJggg==";
|
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABh6FO1AAAAABJRU5ErkJggg==";
|
||||||
|
|
||||||
|
function pngWithIhdrDimensions(width: number, height: number): string {
|
||||||
|
const bytes = Uint8Array.from(atob(TINY_PNG), (character) => character.charCodeAt(0));
|
||||||
|
const view = new DataView(bytes.buffer);
|
||||||
|
view.setUint32(16, width);
|
||||||
|
view.setUint32(20, height);
|
||||||
|
return btoa(String.fromCharCode(...bytes));
|
||||||
|
}
|
||||||
|
|
||||||
function makeScreenshotDeps(
|
function makeScreenshotDeps(
|
||||||
opts: {
|
opts: {
|
||||||
cdp?: CdpRunner;
|
cdp?: CdpRunner;
|
||||||
@@ -86,6 +94,7 @@ describe("stripDataUrlPrefix", () => {
|
|||||||
describe("parsePngDimensions", () => {
|
describe("parsePngDimensions", () => {
|
||||||
it("parses width/height from the IHDR chunk", () => {
|
it("parses width/height from the IHDR chunk", () => {
|
||||||
expect(parsePngDimensions(TINY_PNG)).toEqual({ width: 1, height: 1 });
|
expect(parsePngDimensions(TINY_PNG)).toEqual({ width: 1, height: 1 });
|
||||||
|
expect(parsePngDimensions(pngWithIhdrDimensions(100, 80))).toEqual({ width: 100, height: 80 });
|
||||||
});
|
});
|
||||||
it("returns null on non-PNG input", () => {
|
it("returns null on non-PNG input", () => {
|
||||||
expect(parsePngDimensions("not-a-png-payload-just-random-base64-text-zzzzzzzzz")).toBeNull();
|
expect(parsePngDimensions("not-a-png-payload-just-random-base64-text-zzzzzzzzz")).toBeNull();
|
||||||
@@ -120,6 +129,20 @@ describe("handleScreenshot", () => {
|
|||||||
expect(get).not.toHaveBeenCalled();
|
expect(get).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("fails closed when a full-tab capture is not a valid PNG", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
await sm.start("aa11");
|
||||||
|
const res = await handleScreenshot(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11" },
|
||||||
|
makeScreenshotDeps({
|
||||||
|
captureVisibleTab: vi.fn(async () => "data:image/png;base64,bm90LWEtcG5n"),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(res).toMatchObject({ code: "cdp_failed", message: /invalid PNG data/ });
|
||||||
|
});
|
||||||
|
|
||||||
it("returns not_found when Agent Window has no active tab", async () => {
|
it("returns not_found when Agent Window has no active tab", async () => {
|
||||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
await sm.start("aa11");
|
await sm.start("aa11");
|
||||||
@@ -349,7 +372,7 @@ describe("handleScreenshot", () => {
|
|||||||
cssLayoutViewport: { clientWidth: 800, clientHeight: 600 },
|
cssLayoutViewport: { clientWidth: 800, clientHeight: 600 },
|
||||||
}),
|
}),
|
||||||
"DOM.getContentQuads": () => ({ quads: [[0, 0, 200, 0, 200, 100, 0, 100]] }),
|
"DOM.getContentQuads": () => ({ quads: [[0, 0, 200, 0, 200, 100, 0, 100]] }),
|
||||||
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
|
"Page.captureScreenshot": () => ({ data: pngWithIhdrDimensions(100, 80) }),
|
||||||
});
|
});
|
||||||
|
|
||||||
const res = await handleScreenshot(
|
const res = await handleScreenshot(
|
||||||
@@ -368,6 +391,95 @@ describe("handleScreenshot", () => {
|
|||||||
clip?: { x: number; y: number; width: number; height: number };
|
clip?: { x: number; y: number; width: number; height: number };
|
||||||
};
|
};
|
||||||
expect(clip.clip).toMatchObject({ x: 25, y: 30, width: 50, height: 40 });
|
expect(clip.clip).toMatchObject({ x: 25, y: 30, width: 50, height: 40 });
|
||||||
|
expect(res).toMatchObject({ width: 100, height: 80 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("routes OOPIF surface geometry through its child session and captures top coordinates", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.refStore.set("e5", 999, {
|
||||||
|
tabId: 7,
|
||||||
|
frameId: "child-frame",
|
||||||
|
cdpSessionId: "child-session",
|
||||||
|
kind: "surface",
|
||||||
|
visibleRect: { x: 224, y: 346, w: 200, h: 80 },
|
||||||
|
});
|
||||||
|
const { cdp, sent } = makeFakeCdp({
|
||||||
|
"DOM.getBoxModel": () => ({
|
||||||
|
model: { content: [204, 306, 604, 306, 604, 506, 204, 506] },
|
||||||
|
}),
|
||||||
|
"Page.getLayoutMetrics": () => ({
|
||||||
|
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
|
||||||
|
}),
|
||||||
|
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
|
||||||
|
});
|
||||||
|
cdp.getFrameGraph = vi.fn(async () => ({
|
||||||
|
rootFrameId: "main",
|
||||||
|
frames: [
|
||||||
|
{ frameId: "main", target: { tabId: 7 } },
|
||||||
|
{
|
||||||
|
frameId: "child-frame",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 99,
|
||||||
|
target: { tabId: 7, sessionId: "child-session" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}));
|
||||||
|
const targetCalls: Array<{ sessionId?: string; method: string }> = [];
|
||||||
|
cdp.sendToTarget = vi.fn(async (target, method) => {
|
||||||
|
targetCalls.push({ sessionId: target.sessionId, method });
|
||||||
|
if (method === "DOM.getContentQuads") {
|
||||||
|
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||||
|
}
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||||
|
}
|
||||||
|
throw new Error(`unexpected child CDP call ${method}`);
|
||||||
|
}) as CdpRunner["sendToTarget"];
|
||||||
|
|
||||||
|
const res = await handleScreenshot(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
|
||||||
|
makeScreenshotDeps({
|
||||||
|
cdp,
|
||||||
|
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
|
||||||
|
query: vi.fn(),
|
||||||
|
captureVisibleTab: vi.fn(),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||||
|
expect(targetCalls).toEqual([
|
||||||
|
{ sessionId: "child-session", method: "DOM.getContentQuads" },
|
||||||
|
{ sessionId: "child-session", method: "Page.getLayoutMetrics" },
|
||||||
|
]);
|
||||||
|
const clip = sent.find((call) => call.method === "Page.captureScreenshot")?.params as {
|
||||||
|
clip?: { x: number; y: number; width: number; height: number };
|
||||||
|
};
|
||||||
|
expect(clip.clip).toMatchObject({ x: 224, y: 346, width: 200, height: 80 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails closed when a ref capture is not a valid PNG", async () => {
|
||||||
|
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||||
|
const ctx = await sm.start("aa11");
|
||||||
|
ctx.refStore.set("e5", 999, { tabId: 7, kind: "surface" });
|
||||||
|
const { cdp } = makeFakeCdp({
|
||||||
|
"DOM.getContentQuads": () => ({ quads: [[0, 0, 50, 0, 50, 50, 0, 50]] }),
|
||||||
|
"Page.captureScreenshot": () => ({ data: "bm90LWEtcG5n" }),
|
||||||
|
});
|
||||||
|
|
||||||
|
const res = await handleScreenshot(
|
||||||
|
sm,
|
||||||
|
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
|
||||||
|
makeScreenshotDeps({
|
||||||
|
cdp,
|
||||||
|
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
|
||||||
|
query: vi.fn(),
|
||||||
|
captureVisibleTab: vi.fn(),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(res).toMatchObject({ code: "cdp_failed", message: /invalid PNG data/ });
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns not_found for unknown ref", async () => {
|
it("returns not_found for unknown ref", async () => {
|
||||||
|
|||||||
@@ -116,8 +116,8 @@ export function stripDataUrlPrefix(dataUrl: string): string {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Parse a PNG's IHDR chunk and return `(width, height)`. Returns
|
* Parse a PNG's IHDR chunk and return `(width, height)`. Returns
|
||||||
* `null` on any malformed input so callers fall back to `0/0` instead
|
* `null` on malformed input so callers can reject dimensions that are
|
||||||
* of throwing.
|
* not authoritative instead of guessing from a CSS-space rectangle.
|
||||||
*
|
*
|
||||||
* PNG layout: 8-byte signature, then a 4-byte length, 4-byte type
|
* PNG layout: 8-byte signature, then a 4-byte length, 4-byte type
|
||||||
* ("IHDR"), then the chunk data — width is bytes 16-19 BE, height is
|
* ("IHDR"), then the chunk data — width is bytes 16-19 BE, height is
|
||||||
@@ -129,9 +129,9 @@ export function parsePngDimensions(base64: string): { width: number; height: num
|
|||||||
const head = base64.length > 64 ? base64.slice(0, 64) : base64;
|
const head = base64.length > 64 ? base64.slice(0, 64) : base64;
|
||||||
const bin = atob(head);
|
const bin = atob(head);
|
||||||
if (bin.length < 24) return null;
|
if (bin.length < 24) return null;
|
||||||
if (bin.charCodeAt(0) !== 0x89 || bin.charCodeAt(1) !== 0x50 || bin.charCodeAt(2) !== 0x4e) {
|
const expectedSignature = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
|
||||||
return null;
|
if (expectedSignature.some((byte, index) => bin.charCodeAt(index) !== byte)) return null;
|
||||||
}
|
if (bin.slice(12, 16) !== "IHDR") return null;
|
||||||
const u32 = (off: number) =>
|
const u32 = (off: number) =>
|
||||||
(bin.charCodeAt(off) << 24) |
|
(bin.charCodeAt(off) << 24) |
|
||||||
(bin.charCodeAt(off + 1) << 16) |
|
(bin.charCodeAt(off + 1) << 16) |
|
||||||
@@ -250,10 +250,10 @@ async function captureElementScreenshot(
|
|||||||
if (!image_base64) {
|
if (!image_base64) {
|
||||||
return { code: "cdp_failed", message: "Page.captureScreenshot returned no data" };
|
return { code: "cdp_failed", message: "Page.captureScreenshot returned no data" };
|
||||||
}
|
}
|
||||||
const dims = parsePngDimensions(image_base64) ?? {
|
const dims = parsePngDimensions(image_base64);
|
||||||
width: Math.round(rect.width),
|
if (!dims) {
|
||||||
height: Math.round(rect.height),
|
return { code: "cdp_failed", message: "Page.captureScreenshot returned invalid PNG data" };
|
||||||
};
|
}
|
||||||
return { image_base64, width: dims.width, height: dims.height };
|
return { image_base64, width: dims.width, height: dims.height };
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
return {
|
return {
|
||||||
@@ -402,7 +402,10 @@ export async function handleScreenshot(
|
|||||||
if (isRpcError(captured)) return captured;
|
if (isRpcError(captured)) return captured;
|
||||||
if (signal?.aborted) return cancelled("screenshot");
|
if (signal?.aborted) return cancelled("screenshot");
|
||||||
const image_base64 = captured;
|
const image_base64 = captured;
|
||||||
const dims = parsePngDimensions(image_base64) ?? { width: 0, height: 0 };
|
const dims = parsePngDimensions(image_base64);
|
||||||
|
if (!dims) {
|
||||||
|
return { code: "cdp_failed", message: "screenshot capture returned invalid PNG data" };
|
||||||
|
}
|
||||||
return withShotDialogs({
|
return withShotDialogs({
|
||||||
image_base64,
|
image_base64,
|
||||||
width: dims.width,
|
width: dims.width,
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
import { describe, expect, it, vi } from "vitest";
|
||||||
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
|
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
|
||||||
import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
|
import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
|
||||||
|
import { discoverRenderedSurfaces } from "../rendered-surfaces";
|
||||||
|
|
||||||
// Minimal but format-accurate captureSnapshot reply: a body with one
|
// Minimal but format-accurate captureSnapshot reply: a body with one
|
||||||
// fixed full-screen overlay div carrying a password input.
|
// fixed full-screen overlay div carrying a password input.
|
||||||
@@ -1188,6 +1189,97 @@ describe("captureViewModel", () => {
|
|||||||
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
|
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("keeps same-origin iframe canvas discovery in top-level CSS coordinates at dpr 1 and 2", async () => {
|
||||||
|
const captureAtDpr = async (dpr: number) => {
|
||||||
|
const S = ["html", "body", "iframe", "canvas", "static", "auto"];
|
||||||
|
const i = (s: string) => S.indexOf(s);
|
||||||
|
const scale = (values: number[]) => values.map((value) => value * dpr);
|
||||||
|
const snapshot = {
|
||||||
|
strings: S,
|
||||||
|
documents: [
|
||||||
|
{
|
||||||
|
frameId: "main",
|
||||||
|
nodes: {
|
||||||
|
parentIndex: [-1, 0, 1],
|
||||||
|
nodeName: [i("html"), i("body"), i("iframe")],
|
||||||
|
backendNodeId: [10, 11, 13],
|
||||||
|
attributes: [[], [], []],
|
||||||
|
contentDocumentIndex: { index: [2], value: [1] },
|
||||||
|
},
|
||||||
|
layout: {
|
||||||
|
nodeIndex: [1, 2],
|
||||||
|
styles: [
|
||||||
|
[i("static"), i("auto")],
|
||||||
|
[i("static"), i("auto")],
|
||||||
|
],
|
||||||
|
bounds: [scale([0, 0, 1000, 800]), scale([100, 150, 300, 200])],
|
||||||
|
paintOrders: [0, 1],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
nodes: {
|
||||||
|
parentIndex: [-1, 0],
|
||||||
|
nodeName: [i("body"), i("canvas")],
|
||||||
|
backendNodeId: [20, 21],
|
||||||
|
attributes: [[], []],
|
||||||
|
},
|
||||||
|
layout: {
|
||||||
|
nodeIndex: [0, 1],
|
||||||
|
styles: [
|
||||||
|
[i("static"), i("auto")],
|
||||||
|
[i("static"), i("auto")],
|
||||||
|
],
|
||||||
|
bounds: [scale([0, 0, 300, 200]), scale([20, 30, 120, 60])],
|
||||||
|
paintOrders: [0, 1],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const cdp = {
|
||||||
|
send: vi.fn(async (_tabId: number, method: string) => {
|
||||||
|
if (method === "DOMSnapshot.enable") return {};
|
||||||
|
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||||
|
if (method === "Page.getLayoutMetrics") {
|
||||||
|
return {
|
||||||
|
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||||
|
layoutViewport: { clientWidth: 1000 * dpr, clientHeight: 800 * dpr },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
throw new Error(method);
|
||||||
|
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||||
|
};
|
||||||
|
const captured = await captureViewModel(cdp, 4);
|
||||||
|
const childNodes = captured.frameNodes?.get("child") ?? [];
|
||||||
|
return discoverRenderedSurfaces(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
frameId: "child",
|
||||||
|
contextScopeId: "child",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 13,
|
||||||
|
target: { tabId: 4 },
|
||||||
|
domNodes: childNodes,
|
||||||
|
axNodes: [],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
captured.viewport,
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
const atDpr1 = await captureAtDpr(1);
|
||||||
|
const atDpr2 = await captureAtDpr(2);
|
||||||
|
|
||||||
|
expect(atDpr1).toEqual([
|
||||||
|
expect.objectContaining({
|
||||||
|
backendNodeId: 21,
|
||||||
|
frameId: "child",
|
||||||
|
visibleRect: { x: 120, y: 180, w: 120, h: 60 },
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
expect(atDpr2).toEqual(atDpr1);
|
||||||
|
});
|
||||||
|
|
||||||
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => {
|
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => {
|
||||||
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
|
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
|
||||||
const i = (s: string) => S.indexOf(s);
|
const i = (s: string) => S.indexOf(s);
|
||||||
|
|||||||
@@ -47,8 +47,9 @@ describe("renderVom single-layer page", () => {
|
|||||||
];
|
];
|
||||||
const out = renderVom(input);
|
const out = renderVom(input);
|
||||||
|
|
||||||
|
expect(out.text).toContain("[visual surfaces]");
|
||||||
expect(out.text).toContain(
|
expect(out.text).toContain(
|
||||||
'@e1 surface "Data grid" [rendering=canvas; layers=2; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
|
'@e1 surface "Data grid" [bounds=10,20,800,500; rendering=canvas; layers=2; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
|
||||||
);
|
);
|
||||||
expect(out.refs).toContainEqual(
|
expect(out.refs).toContainEqual(
|
||||||
expect.objectContaining({
|
expect.objectContaining({
|
||||||
@@ -77,7 +78,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
const out = renderVom(input);
|
const out = renderVom(input);
|
||||||
|
|
||||||
expect(out.text).toContain(
|
expect(out.text).toContain(
|
||||||
'@e1 surface "canvas visual surface" [rendering=canvas; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
|
'@e1 surface "canvas visual surface" [bounds=10,20,800,500; rendering=canvas; visual-only; requires=image-understanding; use: bsk screenshot --ref @e1]',
|
||||||
);
|
);
|
||||||
expect(out.refs).toContainEqual(
|
expect(out.refs).toContainEqual(
|
||||||
expect.objectContaining({
|
expect.objectContaining({
|
||||||
@@ -89,6 +90,78 @@ describe("renderVom single-layer page", () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("appends visual refs after DOM refs so canvas density cannot renumber controls", () => {
|
||||||
|
const input = scene([
|
||||||
|
node({ id: 1, role: "RootWebArea", frameId: "main" }),
|
||||||
|
node({ id: 2, parentId: 1, role: "button", name: "Submit", tag: "button" }),
|
||||||
|
]);
|
||||||
|
input.visualSurfaces = [
|
||||||
|
{
|
||||||
|
parentId: 1,
|
||||||
|
backendNodeId: 99,
|
||||||
|
frameId: "main",
|
||||||
|
renderingKind: "canvas",
|
||||||
|
visibleRect: { x: 10, y: 20, w: 80, h: 50 },
|
||||||
|
memberCount: 1,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const withoutVisuals = renderVom({ ...input, visualSurfaces: [] });
|
||||||
|
const withVisuals = renderVom(input);
|
||||||
|
|
||||||
|
expect(withoutVisuals.refs[0]).toMatchObject({ ref: "e1", backendNodeId: 2, kind: "dom" });
|
||||||
|
expect(withVisuals.refs[0]).toMatchObject({ ref: "e1", backendNodeId: 2, kind: "dom" });
|
||||||
|
expect(withVisuals.refs[1]).toMatchObject({ ref: "e2", backendNodeId: 99, kind: "surface" });
|
||||||
|
expect(withVisuals.text.indexOf('@e1 button "Submit"')).toBeLessThan(
|
||||||
|
withVisuals.text.indexOf("[visual surfaces]"),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("bounds visual output and prefers explicitly labeled surfaces", () => {
|
||||||
|
const input = scene([node({ id: 1, role: "RootWebArea", frameId: "main" })]);
|
||||||
|
input.visualSurfaces = Array.from({ length: 12 }, (_, index) => ({
|
||||||
|
parentId: 1,
|
||||||
|
backendNodeId: 100 + index,
|
||||||
|
frameId: "main",
|
||||||
|
renderingKind: "canvas" as const,
|
||||||
|
visibleRect: { x: index * 10, y: 20, w: 8, h: 8 },
|
||||||
|
...(index === 11 ? { label: "Important status" } : {}),
|
||||||
|
memberCount: 1,
|
||||||
|
}));
|
||||||
|
|
||||||
|
const out = renderVom(input);
|
||||||
|
const surfaceRefs = out.refs.filter((ref) => ref.kind === "surface");
|
||||||
|
|
||||||
|
expect(surfaceRefs).toHaveLength(8);
|
||||||
|
expect(surfaceRefs[0]).toMatchObject({ backendNodeId: 111, name: "Important status" });
|
||||||
|
expect(out.text).toContain("… 4 additional visual surfaces omitted");
|
||||||
|
expect(out.truncated).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not let a visual surface exhaust the budget before later DOM content", () => {
|
||||||
|
const input = scene([
|
||||||
|
node({ id: 1, role: "RootWebArea", name: "Doc", frameId: "main" }),
|
||||||
|
node({ id: 2, parentId: 1, role: "button", name: "First", tag: "button" }),
|
||||||
|
node({ id: 3, parentId: 1, role: "button", name: "Later", tag: "button" }),
|
||||||
|
]);
|
||||||
|
input.visualSurfaces = [
|
||||||
|
{
|
||||||
|
parentId: 1,
|
||||||
|
backendNodeId: 99,
|
||||||
|
frameId: "main",
|
||||||
|
renderingKind: "canvas",
|
||||||
|
visibleRect: { x: 10, y: 20, w: 800, h: 500 },
|
||||||
|
memberCount: 1,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const out = renderVom(input, { maxTokens: 30 });
|
||||||
|
|
||||||
|
expect(out.text).toContain('@e2 button "Later"');
|
||||||
|
expect(out.refs.filter((ref) => ref.kind === "surface")).toHaveLength(0);
|
||||||
|
expect(out.truncated).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
it("does not derive handle context across frame scopes", () => {
|
it("does not derive handle context across frame scopes", () => {
|
||||||
const out = renderVom(
|
const out = renderVom(
|
||||||
scene([
|
scene([
|
||||||
|
|||||||
+91
-43
@@ -88,6 +88,7 @@ const MAX_CUSTOM_RECOVERY_AREA = 180_000;
|
|||||||
const MAX_RECOVERABLE_SINGLE_INTENT_AREA = 360_000;
|
const MAX_RECOVERABLE_SINGLE_INTENT_AREA = 360_000;
|
||||||
const MAX_HANDLE_CONTEXT_ITEMS = 3;
|
const MAX_HANDLE_CONTEXT_ITEMS = 3;
|
||||||
const MAX_SURFACE_ITEMS = 12;
|
const MAX_SURFACE_ITEMS = 12;
|
||||||
|
const MAX_VISUAL_SURFACES = 8;
|
||||||
const MAX_SCOPE_LINES = 40;
|
const MAX_SCOPE_LINES = 40;
|
||||||
const RECOVERY_HANDLER_ATTRS = ["onclick", "onmousedown", "onkeydown", "onkeyup", "onkeypress"];
|
const RECOVERY_HANDLER_ATTRS = ["onclick", "onmousedown", "onkeydown", "onkeyup", "onkeypress"];
|
||||||
const NATIVE_TAGS = new Set(["button", "input", "select", "textarea"]);
|
const NATIVE_TAGS = new Set(["button", "input", "select", "textarea"]);
|
||||||
@@ -794,40 +795,81 @@ interface RenderState {
|
|||||||
duplicateRefNames: Set<string>;
|
duplicateRefNames: Set<string>;
|
||||||
surfaceMap: Map<number, CondSurface>;
|
surfaceMap: Map<number, CondSurface>;
|
||||||
scopeMap: Map<number, ActiveScopeBlock>;
|
scopeMap: Map<number, ActiveScopeBlock>;
|
||||||
visualSurfaces: Map<number | null, VomVisualSurface[]>;
|
visualSurfaces: VomVisualSurface[];
|
||||||
}
|
}
|
||||||
|
|
||||||
function groupedByParent<T extends { parentId: number | null }>(
|
function visualSurfaceArea(surface: VomVisualSurface): number {
|
||||||
items: T[],
|
return Math.max(0, surface.visibleRect.w) * Math.max(0, surface.visibleRect.h);
|
||||||
): Map<number | null, T[]> {
|
|
||||||
const grouped = new Map<number | null, T[]>();
|
|
||||||
for (const item of items) {
|
|
||||||
const siblings = grouped.get(item.parentId) ?? [];
|
|
||||||
siblings.push(item);
|
|
||||||
grouped.set(item.parentId, siblings);
|
|
||||||
}
|
|
||||||
return grouped;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function emitVisualEntries(parentId: number | null, depth: number, state: RenderState): void {
|
function compareVisualSurfaces(a: VomVisualSurface, b: VomVisualSurface): number {
|
||||||
if (state.stopped) return;
|
const aLabeled = cleaned(a.label) !== undefined;
|
||||||
for (const surface of state.visualSurfaces.get(parentId) ?? []) {
|
const bLabeled = cleaned(b.label) !== undefined;
|
||||||
if (depth > state.maxDepth) {
|
return (
|
||||||
state.truncated = true;
|
Number(bLabeled) - Number(aLabeled) ||
|
||||||
|
visualSurfaceArea(b) - visualSurfaceArea(a) ||
|
||||||
|
a.frameId.localeCompare(b.frameId) ||
|
||||||
|
a.visibleRect.y - b.visibleRect.y ||
|
||||||
|
a.visibleRect.x - b.visibleRect.x ||
|
||||||
|
a.backendNodeId - b.backendNodeId
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function visualSurfaceLine(surface: VomVisualSurface, ref: string): string {
|
||||||
|
const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`;
|
||||||
|
const layers = surface.memberCount > 1 ? `; layers=${surface.memberCount}` : "";
|
||||||
|
const { x, y, w, h } = surface.visibleRect;
|
||||||
|
const bounds = [x, y, w, h].map(Math.round).join(",");
|
||||||
|
return ` @${ref} surface ${JSON.stringify(label)} [bounds=${bounds}; rendering=${surface.renderingKind}${layers}; visual-only; requires=image-understanding; use: bsk screenshot --ref @${ref}]`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function selectVisualSurfaces(
|
||||||
|
surfaces: readonly VomVisualSurface[],
|
||||||
|
nodesById: ReadonlyMap<number, VomNode>,
|
||||||
|
): { selected: VomVisualSurface[]; eligibleCount: number } {
|
||||||
|
const selected: VomVisualSurface[] = [];
|
||||||
|
let eligibleCount = 0;
|
||||||
|
|
||||||
|
for (const surface of surfaces) {
|
||||||
|
if (surface.parentId !== null && !nodesById.has(surface.parentId)) continue;
|
||||||
|
eligibleCount += 1;
|
||||||
|
|
||||||
|
const insertionIndex = selected.findIndex(
|
||||||
|
(candidate) => compareVisualSurfaces(surface, candidate) < 0,
|
||||||
|
);
|
||||||
|
if (insertionIndex < 0) {
|
||||||
|
if (selected.length < MAX_VISUAL_SURFACES) selected.push(surface);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
selected.splice(insertionIndex, 0, surface);
|
||||||
|
if (selected.length > MAX_VISUAL_SURFACES) selected.pop();
|
||||||
|
}
|
||||||
|
|
||||||
|
return { selected, eligibleCount };
|
||||||
|
}
|
||||||
|
|
||||||
|
function emitVisualAppendix(state: RenderState): void {
|
||||||
|
if (state.stopped) return;
|
||||||
|
const { selected, eligibleCount } = selectVisualSurfaces(state.visualSurfaces, state.nodesById);
|
||||||
|
if (eligibleCount === 0) return;
|
||||||
|
|
||||||
|
const header = "[visual surfaces]";
|
||||||
|
let headerEmitted = false;
|
||||||
|
let emitted = 0;
|
||||||
|
for (const surface of selected) {
|
||||||
const ref = `e${state.nextRef}`;
|
const ref = `e${state.nextRef}`;
|
||||||
const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`;
|
const line = visualSurfaceLine(surface, ref);
|
||||||
const layers = surface.memberCount > 1 ? `; layers=${surface.memberCount}` : "";
|
const nextTokens =
|
||||||
const line = `${" ".repeat(depth)}@${ref} surface ${JSON.stringify(label)} [rendering=${surface.renderingKind}${layers}; visual-only; requires=image-understanding; use: bsk screenshot --ref @${ref}]`;
|
state.tokens + estimateTokens(line) + (headerEmitted ? 0 : estimateTokens(header));
|
||||||
const nextTokens = state.tokens + estimateTokens(line);
|
if (nextTokens > state.maxTokens) break;
|
||||||
if (nextTokens > state.maxTokens) {
|
if (!headerEmitted) {
|
||||||
state.truncated = true;
|
state.lines.push(header);
|
||||||
state.stopped = true;
|
state.tokens += estimateTokens(header);
|
||||||
return;
|
headerEmitted = true;
|
||||||
}
|
}
|
||||||
state.lines.push(line);
|
state.lines.push(line);
|
||||||
state.tokens = nextTokens;
|
state.tokens += estimateTokens(line);
|
||||||
|
const label = cleaned(surface.label) ?? `${surface.renderingKind} visual surface`;
|
||||||
state.refs.push({
|
state.refs.push({
|
||||||
ref,
|
ref,
|
||||||
backendNodeId: surface.backendNodeId,
|
backendNodeId: surface.backendNodeId,
|
||||||
@@ -840,6 +882,20 @@ function emitVisualEntries(parentId: number | null, depth: number, state: Render
|
|||||||
line: state.lines.length - 1,
|
line: state.lines.length - 1,
|
||||||
});
|
});
|
||||||
state.nextRef += 1;
|
state.nextRef += 1;
|
||||||
|
emitted += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const omitted = eligibleCount - emitted;
|
||||||
|
if (omitted > 0) {
|
||||||
|
state.truncated = true;
|
||||||
|
if (headerEmitted) {
|
||||||
|
const summary = ` … ${omitted} additional visual surfaces omitted`;
|
||||||
|
const nextTokens = state.tokens + estimateTokens(summary);
|
||||||
|
if (nextTokens <= state.maxTokens) {
|
||||||
|
state.lines.push(summary);
|
||||||
|
state.tokens = nextTokens;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -872,34 +928,24 @@ function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: Ren
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type RenderTask =
|
|
||||||
| { kind: "node"; node: VomNode; depth: number }
|
|
||||||
| { kind: "visual"; parentId: number | null; depth: number };
|
|
||||||
|
|
||||||
function pushRenderChildren(
|
function pushRenderChildren(
|
||||||
stack: RenderTask[],
|
stack: Array<{ node: VomNode; depth: number }>,
|
||||||
children: readonly VomNode[],
|
children: readonly VomNode[],
|
||||||
depth: number,
|
depth: number,
|
||||||
): void {
|
): void {
|
||||||
for (let index = children.length - 1; index >= 0; index -= 1) {
|
for (let index = children.length - 1; index >= 0; index -= 1) {
|
||||||
stack.push({ kind: "node", node: children[index], depth });
|
stack.push({ node: children[index], depth });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function renderTree(children: Map<number | null, VomNode[]>, state: RenderState): void {
|
function renderTree(children: Map<number | null, VomNode[]>, state: RenderState): void {
|
||||||
const stack: RenderTask[] = [{ kind: "visual", parentId: null, depth: 1 }];
|
const stack: Array<{ node: VomNode; depth: number }> = [];
|
||||||
pushRenderChildren(stack, children.get(null) ?? [], 1);
|
pushRenderChildren(stack, children.get(null) ?? [], 1);
|
||||||
|
|
||||||
while (stack.length > 0 && !state.stopped) {
|
while (stack.length > 0 && !state.stopped) {
|
||||||
const task = stack.pop() as RenderTask;
|
const { node, depth } = stack.pop() as { node: VomNode; depth: number };
|
||||||
if (task.kind === "visual") {
|
|
||||||
emitVisualEntries(task.parentId, task.depth, state);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const { node, depth } = task;
|
|
||||||
|
|
||||||
if (!shouldRender(node)) {
|
if (!shouldRender(node)) {
|
||||||
stack.push({ kind: "visual", parentId: node.id, depth });
|
|
||||||
pushRenderChildren(stack, children.get(node.id) ?? [], depth);
|
pushRenderChildren(stack, children.get(node.id) ?? [], depth);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -942,10 +988,8 @@ function renderTree(children: Map<number | null, VomNode[]>, state: RenderState)
|
|||||||
if (scope) emitActiveScopeBlock(scope, depth + 1, state);
|
if (scope) emitActiveScopeBlock(scope, depth + 1, state);
|
||||||
|
|
||||||
if ((ref && shouldSkipRedundantRefChildren(node)) || shouldSkipRedundantChildren(node, state)) {
|
if ((ref && shouldSkipRedundantRefChildren(node)) || shouldSkipRedundantChildren(node, state)) {
|
||||||
emitVisualEntries(node.id, depth + 1, state);
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
stack.push({ kind: "visual", parentId: node.id, depth: depth + 1 });
|
|
||||||
pushRenderChildren(stack, children.get(node.id) ?? [], depth + 1);
|
pushRenderChildren(stack, children.get(node.id) ?? [], depth + 1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -976,7 +1020,7 @@ function renderNodes(
|
|||||||
duplicateRefNames: duplicateReferenceNames(nodes),
|
duplicateRefNames: duplicateReferenceNames(nodes),
|
||||||
surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])),
|
surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])),
|
||||||
scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])),
|
scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])),
|
||||||
visualSurfaces: groupedByParent(visualSurfaces),
|
visualSurfaces,
|
||||||
};
|
};
|
||||||
|
|
||||||
renderTree(children, state);
|
renderTree(children, state);
|
||||||
@@ -1198,7 +1242,10 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp
|
|||||||
scene.activeScopeBlocks,
|
scene.activeScopeBlocks,
|
||||||
scene.visualSurfaces?.filter((surface) => visibleSurface(surface.parentId)),
|
scene.visualSurfaces?.filter((surface) => visibleSurface(surface.parentId)),
|
||||||
);
|
);
|
||||||
state.lines.push(renderPageOcclusionLine(hiddenCount));
|
const occlusionLine = renderPageOcclusionLine(hiddenCount);
|
||||||
|
state.lines.push(occlusionLine);
|
||||||
|
state.tokens += estimateTokens(occlusionLine);
|
||||||
|
emitVisualAppendix(state);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
text: state.lines.join("\n"),
|
text: state.lines.join("\n"),
|
||||||
@@ -1227,6 +1274,7 @@ export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult
|
|||||||
scene.activeScopeBlocks,
|
scene.activeScopeBlocks,
|
||||||
scene.visualSurfaces,
|
scene.visualSurfaces,
|
||||||
);
|
);
|
||||||
|
emitVisualAppendix(state);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
text: state.lines.join("\n"),
|
text: state.lines.join("\n"),
|
||||||
|
|||||||
Reference in New Issue
Block a user