feat: support on-demand screenshots for canvas visual refs

This commit is contained in:
Ljy-0827
2026-09-10 12:21:20 +08:00
committed by GitHub
parent a6de73aea3
commit 31ddcb139c
11 changed files with 922 additions and 61 deletions
@@ -4,7 +4,7 @@ import { RefStore, type VisualRefInput } from "@/session-manager/ref-store";
import { handleDownload } from "../download";
import { handleRequestHelp, resetHelpLifecycleForTests } from "../human-loop";
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "../interaction";
import { handleGetHtml, handleScreenshot } from "../observation";
import { handleGetHtml } from "../observation";
import type { CdpRunner } from "../shared";
import { lookupRefTarget, lookupSnapshotRef, resolveSnapshotRef } from "../snapshot-ref";
import { handleUpload } from "../upload";
@@ -117,12 +117,10 @@ describe("typed visual refs", () => {
"press",
"select",
"get_html",
"screenshot",
])("rejects visual refs in %s before target effects", async (tool) => {
const { manager, send, cdp, tabsApi } = await setup();
const params = { session_id: "test", tab_id: 4, ref: "@e1" };
const deps = { cdp, tabsApi };
const captureVisibleTab = vi.fn();
let result: unknown;
switch (tool) {
case "click":
@@ -143,16 +141,9 @@ describe("typed visual refs", () => {
case "get_html":
result = await handleGetHtml(manager, params, deps);
break;
case "screenshot":
result = await handleScreenshot(manager, params, {
...deps,
captureApi: { ...tabsApi, captureVisibleTab },
});
break;
}
expect(result).toMatchObject({ code: "unsupported", data: { reason: "ref_kind_unsupported" } });
expect(send).not.toHaveBeenCalled();
expect(captureVisibleTab).not.toHaveBeenCalled();
});
it.each([
@@ -0,0 +1,373 @@
import { describe, expect, it, vi } from "vitest";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { SessionManager } from "@/session-manager/manager";
import { handleScreenshot } from "../observation";
import type { CdpRunner } from "../shared";
import { captureVisualScreenshot, visualScreenshotScale } from "../visual-screenshot";
import type { VisualCandidate, VisualFramePath } from "../vom/visual-discovery";
const styles = {
position: "static",
visibility: "visible",
opacity: "1",
display: "block",
"overflow-x": "visible",
"overflow-y": "visible",
transform: "none",
zoom: "1",
"clip-path": "none",
"mask-image": "none",
rotate: "none",
scale: "none",
perspective: "none",
clip: "auto",
contain: "none",
"overflow-clip-margin": "0px",
};
const rect = (x = 10, y = 20, width = 100, height = 40) => ({ x, y, width, height });
const root = {
attachmentId: "a",
target: { tabId: 4 },
frameId: "top",
documentElementBackendNodeId: 1,
};
function row(id: number, tag: string, box = rect()) {
return {
node: { backend: id },
tag,
box,
client: box,
contentSize: { width: box.width, height: box.height },
styles: { ...styles },
};
}
function encode(value: unknown): unknown {
if (value === null) return { type: "null" };
if (Array.isArray(value)) return { type: "array", value: value.map(encode) };
if (typeof value === "object") {
const record = value as Record<string, unknown>;
if ("backend" in record) return { type: "node", value: { backendNodeId: record.backend } };
return { type: "object", value: Object.entries(record).map(([k, v]) => [k, encode(v)]) };
}
return { type: typeof value, value };
}
function png(width: number, height: number) {
const bytes = new Uint8Array(33);
bytes.set([137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82]);
const view = new DataView(bytes.buffer);
view.setUint32(16, width);
view.setUint32(20, height);
return btoa(String.fromCharCode(...bytes));
}
function fixture(child = false, oopif = false) {
const parent: VisualFramePath = { document: root };
const frame: VisualFramePath = child
? {
document: {
...root,
frameId: "child",
documentElementBackendNodeId: 11,
target: oopif ? { tabId: 4, sessionId: "oopif" } : { tabId: 4 },
},
parent: { frame: parent, ownerBackendNodeId: 2 },
}
: parent;
const crop = child ? rect(120, 240, 200, 80) : rect();
const candidate: VisualCandidate = {
document: frame.document,
backendNodeId: child ? 12 : 3,
parentBackendNodeId: frame.document.documentElementBackendNodeId,
framePath: frame,
region: { status: "available", borderBox: crop, crop },
};
const topRows = child
? [row(2, "iframe", rect(100, 200, 400, 200)), row(1, "html", rect(0, 0, 1200, 800))]
: [row(3, "canvas"), row(1, "html", rect(0, 0, 1200, 800))];
if (child) topRows[0].contentSize = { width: 200, height: 100 };
const childRows = [row(12, "canvas"), row(11, "html", rect(0, 0, 200, 100))];
let calls = 0;
const control = {
wrongOwner: false,
rootChanged: false,
failRead: false,
abortRead: false,
dpr: 1,
shots: [png(crop.width, crop.height)],
};
const controller = new AbortController();
const send = vi.fn(
async (target: CdpTarget, method: string, params: Record<string, unknown> = {}) => {
if (method === "DOM.getFrameOwner") return { backendNodeId: control.wrongOwner ? 999 : 2 };
if (method === "Page.createIsolatedWorld")
return { executionContextId: params.frameId === "child" ? 11 : 1 };
if (method === "DOM.resolveNode")
return { object: { objectId: String(params.backendNodeId) } };
if (method === "Runtime.callFunctionOn") {
const isChild = params.objectId === "12";
if (!(params.functionDeclaration as string).includes("styleNames"))
return {
result: {
deepSerializedValue: {
type: "node",
value: { backendNodeId: control.rootChanged ? 999 : isChild ? 11 : 1 },
},
},
};
if (control.failRead) throw new Error("read failed");
if (control.abortRead) controller.abort();
return {
result: {
deepSerializedValue: encode({
top: !isChild,
dpr: control.dpr,
rows: isChild ? childRows : topRows,
}),
},
};
}
if (method === "Runtime.releaseObjectGroup") return {};
if (method === "Page.getLayoutMetrics")
return {
cssLayoutViewport: { clientWidth: 1200, clientHeight: 800, pageX: 0, pageY: 0 },
cssVisualViewport: { scale: 1, zoom: 1 },
};
if (method === "DOM.getBoxModel")
return { model: { content: [100, 200, 500, 200, 500, 400, 100, 400] } };
if (method === "Page.captureScreenshot")
return { data: control.shots[Math.min(calls++, control.shots.length - 1)] };
throw new Error(`unexpected ${method} ${target.sessionId}`);
},
);
const cdp = {
send: ((tabId, method, params) =>
send({ tabId }, method, params as Record<string, unknown>)) as CdpRunner["send"],
sendToTarget: send as unknown as CdpRunner["sendToTarget"],
getAttachmentId: () => "a",
getFrameGraph: vi.fn(async () => {
throw new Error("whole page discovery forbidden");
}),
};
return { candidate, cdp, send, control, controller, topRows, childRows };
}
describe("visual screenshot", () => {
it.each([
[false, false],
[true, false],
[true, true],
])("screenshots the local target child=%s oopif=%s without whole-page discovery", async (child, oopif) => {
const f = fixture(child, oopif);
const result = await captureVisualScreenshot(f.cdp, f.candidate);
expect(result).toMatchObject({ width: child ? 200 : 100, height: child ? 80 : 40 });
expect(f.cdp.getFrameGraph).not.toHaveBeenCalled();
const names = f.send.mock.calls.map((c) => c[1]);
expect(names.filter((n) => n === "DOM.getFrameOwner")).toHaveLength(child ? 1 : 0);
expect(names.filter((n) => n === "DOM.resolveNode")).toHaveLength(child ? 2 : 1);
expect(names.some((n) => /scroll|DOMSnapshot|getFrameTree|Accessibility/.test(n))).toBe(false);
expect(f.send.mock.calls.find((c) => c[1] === "Page.captureScreenshot")?.[2]).toMatchObject({
clip: { ...f.candidate.region.crop, scale: 1 },
});
});
it.each([
false,
true,
])("projects a nested child through a parent target oopif=%s", async (oopif) => {
const f = fixture(true, oopif);
const parent = f.candidate.framePath!;
const document = {
...parent.document,
frameId: "grandchild",
documentElementBackendNodeId: 21,
};
const crop = rect(130, 250, 40, 20);
const candidate: VisualCandidate = {
...f.candidate,
document,
backendNodeId: 22,
framePath: { document, parent: { frame: parent, ownerBackendNodeId: 12 } },
region: { status: "available", borderBox: crop, crop },
};
f.childRows[0].tag = "iframe";
f.control.shots = [png(40, 20)];
const original = f.send.getMockImplementation()!;
f.send.mockImplementation(async (target, method, params = {}) => {
if (method === "DOM.getFrameOwner" && params.frameId === "grandchild")
return { backendNodeId: 12 };
if (method === "Runtime.callFunctionOn" && params.objectId === "22") {
if (!(params.functionDeclaration as string).includes("styleNames"))
return {
result: { deepSerializedValue: { type: "node", value: { backendNodeId: 21 } } },
};
return {
result: {
deepSerializedValue: encode({
top: false,
dpr: 1,
rows: [row(22, "canvas", rect(5, 5, 20, 10)), row(21, "html", rect(0, 0, 100, 40))],
}),
},
};
}
if (method === "DOM.getBoxModel" && params.backendNodeId === 12)
return {
model: {
content: oopif
? [10, 20, 110, 20, 110, 60, 10, 60]
: [120, 240, 320, 240, 320, 320, 120, 320],
},
};
if (method === "Page.getLayoutMetrics" && target.sessionId)
return {
cssLayoutViewport: { clientWidth: 200, clientHeight: 100, pageX: 0, pageY: 0 },
cssVisualViewport: { scale: 1, zoom: 1 },
};
return original(target, method, params);
});
expect(await captureVisualScreenshot(f.cdp, candidate)).toMatchObject({
width: 40,
height: 20,
});
expect(f.send.mock.calls.filter((c) => c[1] === "DOM.getFrameOwner")).toHaveLength(2);
expect(f.send.mock.calls.find((c) => c[1] === "Page.captureScreenshot")?.[2]).toMatchObject({
clip: crop,
});
expect(f.cdp.getFrameGraph).not.toHaveBeenCalled();
});
it("runs the live reader on only connected local ancestry", async () => {
const f = fixture();
await captureVisualScreenshot(f.cdp, f.candidate);
const params = f.send.mock.calls.find(
(c) =>
c[1] === "Runtime.callFunctionOn" &&
String(c[2]?.functionDeclaration).includes("styleNames"),
)![2]!;
const read = new Function(`return (${params.functionDeclaration});`)() as (
this: Element,
names: string[],
) => { rows: { node: Element }[] } | null;
const canvas = document.createElement("canvas");
document.body.append(canvas);
try {
const result = read.call(canvas, ["display"]);
expect(result?.rows.map((row) => row.node)).toEqual([
canvas,
document.body,
document.documentElement,
]);
canvas.remove();
expect(read.call(canvas, ["display"])).toBeNull();
const other = document.implementation.createHTMLDocument();
const foreign = other.createElement("canvas");
other.body.append(foreign);
expect(foreign.ownerDocument).toBe(other);
expect(read.call(foreign, ["display"])).toBeNull();
} finally {
canvas.remove();
}
});
it("keeps requests constant as ordinary ancestry grows", async () => {
const small = fixture(),
large = fixture();
large.topRows.splice(
1,
0,
...Array.from({ length: 100 }, (_, i) => row(100 + i, "div", rect(0, 0, 1200, 800))),
);
await captureVisualScreenshot(small.cdp, small.candidate);
await captureVisualScreenshot(large.cdp, large.candidate);
expect(large.send.mock.calls.map((c) => c[1])).toEqual(small.send.mock.calls.map((c) => c[1]));
});
it("preserves missing-path candidates but cannot execute them", async () => {
const f = fixture();
const { framePath: _, ...candidate } = f.candidate;
expect(await captureVisualScreenshot(f.cdp, candidate)).toMatchObject({
data: { reason: "visual_target_changed" },
});
expect(f.send).not.toHaveBeenCalled();
});
it.each([0.249, 0.25, 0.251])("compares rectangle edges with tolerance %s", async (delta) => {
const f = fixture();
f.topRows[0].box.x += delta;
const result = await captureVisualScreenshot(f.cdp, f.candidate);
expect("code" in result).toBe(delta > 0.25);
});
it("rejects a new clipping ancestor even if the resulting crop is identical", async () => {
const f = fixture();
const clip = row(8, "div", rect(0, 0, 1200, 800));
clip.styles["overflow-x"] = "hidden";
f.topRows.splice(1, 0, clip);
expect(await captureVisualScreenshot(f.cdp, f.candidate)).toMatchObject({
data: { reason: "visual_target_changed" },
});
expect(f.send.mock.calls.some((c) => c[1] === "Page.captureScreenshot")).toBe(false);
});
it("rejects changed owner and changed DOM before capture", async () => {
for (const kind of ["wrongOwner", "rootChanged"] as const) {
const f = fixture(true);
f.control[kind] = true;
expect(await captureVisualScreenshot(f.cdp, f.candidate)).toMatchObject({
data: { reason: "visual_target_changed" },
});
expect(f.send.mock.calls.some((c) => c[1] === "Page.captureScreenshot")).toBe(false);
}
});
it("releases retained objects on read failure and cancellation", async () => {
for (const kind of ["failRead", "abortRead"] as const) {
const f = fixture();
f.control[kind] = true;
const result = await captureVisualScreenshot(f.cdp, f.candidate, f.controller.signal);
expect(result).toMatchObject({ code: kind === "abortRead" ? "cancelled" : "cdp_failed" });
expect(f.send.mock.calls.at(-1)?.[1]).toBe("Runtime.releaseObjectGroup");
}
});
it("plans pixels without enlarging images and retries at most once", async () => {
expect(visualScreenshotScale(4096, 2048, 2)).toBe(0.25);
expect(visualScreenshotScale(4000, 4000, 1)).toBe(0.5);
const f = fixture();
f.control.shots = [png(3000, 3000), png(1800, 1800)];
expect(await captureVisualScreenshot(f.cdp, f.candidate)).toMatchObject({
width: 1800,
height: 1800,
});
const shots = f.send.mock.calls.filter((c) => c[1] === "Page.captureScreenshot");
expect(shots).toHaveLength(2);
expect((shots[1][2]!.clip as { scale: number }).scale).toBeLessThan(1);
const fail = fixture();
fail.control.shots = [png(3000, 3000)];
expect(await captureVisualScreenshot(fail.cdp, fail.candidate)).toMatchObject({
data: { reason: "visual_pixel_budget_exceeded" },
});
expect(fail.send.mock.calls.filter((c) => c[1] === "Page.captureScreenshot")).toHaveLength(2);
});
it("rejects invalid PNG without guessing dimensions", async () => {
const f = fixture();
f.control.shots = ["invalid"];
expect(await captureVisualScreenshot(f.cdp, f.candidate)).toMatchObject({
data: { reason: "screenshot_capture_failed" },
});
});
it("dispatches a visual ref through the screenshot tool and restores overlays", async () => {
const f = fixture();
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("test");
ctx.refStore.replace([["e1", { kind: "visual-region", candidate: f.candidate }]]);
const tab = { id: 4, windowId: 100, active: true } as chrome.tabs.Tab;
const tabsApi = { get: async () => tab, query: async () => [tab] };
const sendToTab = vi.fn(async () => ({}));
const result = await handleScreenshot(
manager,
{ session_id: "test", ref: "e1" },
{ cdp: f.cdp, tabsApi, captureApi: { ...tabsApi, captureVisibleTab: vi.fn() }, sendToTab },
);
expect(result).toMatchObject({ width: 100, height: 40, format: "png" });
expect(sendToTab).toHaveBeenCalledTimes(2);
});
});
@@ -212,11 +212,16 @@ export class GeometryContext {
ownerBackendNodeId: number,
ancestorClips: Polygon[],
viewport: Size,
contentSize?: Size,
): Promise<SnapshotProjectionResult> {
const key = `${cdpTargetKey(source.target)}:${ownerBackendNodeId}`;
let promise = this.snapshotOwners.get(key);
if (!promise) {
promise = this.snapshotOwner(source.target, ownerBackendNodeId);
promise = contentSize
? this.ownerContent(source.target, ownerBackendNodeId).then((quad) =>
quad ? { quad, size: contentSize } : null,
)
: this.snapshotOwner(source.target, ownerBackendNodeId);
this.snapshotOwners.set(key, promise);
}
let owner: { quad: Quad; size: Size } | null;
+15 -37
View File
@@ -1,3 +1,8 @@
import { parsePngDimensions } from "./png";
import { captureVisualScreenshot } from "./visual-screenshot";
export { parsePngDimensions } from "./png";
import type { CapturedSceneInput } from "./vom/facts";
// Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`,
// and semantic `tool.observe` (design §7). Each handler resolves the target
@@ -101,38 +106,6 @@ export function stripDataUrlPrefix(dataUrl: string): string {
return m ? dataUrl.slice(m[0].length) : dataUrl;
}
/**
* Parse a PNG's IHDR chunk and return `(width, height)`. Returns
* `null` on any malformed input so callers fall back to `0/0` instead
* of throwing.
*
* PNG layout: 8-byte signature, then a 4-byte length, 4-byte type
* ("IHDR"), then the chunk data — width is bytes 16-19 BE, height is
* 20-23 BE.
*/
export function parsePngDimensions(base64: string): { width: number; height: number } | null {
try {
// atob is available in MV3 service workers.
const head = base64.length > 64 ? base64.slice(0, 64) : base64;
const bin = atob(head);
if (bin.length < 24) return null;
if (bin.charCodeAt(0) !== 0x89 || bin.charCodeAt(1) !== 0x50 || bin.charCodeAt(2) !== 0x4e) {
return null;
}
const u32 = (off: number) =>
(bin.charCodeAt(off) << 24) |
(bin.charCodeAt(off + 1) << 16) |
(bin.charCodeAt(off + 2) << 8) |
bin.charCodeAt(off + 3);
const width = u32(16) >>> 0;
const height = u32(20) >>> 0;
if (width === 0 || height === 0) return null;
return { width, height };
} catch {
return null;
}
}
export interface ScreenshotDeps {
cdp?: SharedCdpRunner;
tabsApi: ChromeTabsApi;
@@ -308,12 +281,17 @@ export async function handleScreenshot(
if (!deps.cdp) {
return { code: "cdp_failed", message: "screenshot ref capture requires CDP" };
}
if (lookupRefTarget(ctx, ref, target.tabId)?.kind === "visual-region")
return rpcError(
"unsupported",
"ref_kind_unsupported",
"visual-region screenshot execution is not available in this build",
const entry = lookupRefTarget(ctx, ref, target.tabId);
if (entry?.kind === "visual-region") {
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
const captured = await withExtensionOverlayHidden(
target.tabId,
() => captureVisualScreenshot(deps.cdp!, entry.candidate, signal),
deps.sendToTab,
);
if (isRpcError(captured)) return captured;
return withShotDialogs({ ...captured, format: "png", tab_id: target.tabId });
}
const node = resolveSnapshotRef(ctx, ref, target.tabId);
if (isRpcError(node)) return node;
if (signal?.aborted) return cancelled("screenshot");
+31
View File
@@ -0,0 +1,31 @@
/**
* Parse a PNG's IHDR chunk and return `(width, height)`. Returns
* `null` on any malformed input so callers fall back to `0/0` instead
* of throwing.
*
* PNG layout: 8-byte signature, then a 4-byte length, 4-byte type
* ("IHDR"), then the chunk data — width is bytes 16-19 BE, height is
* 20-23 BE.
*/
export function parsePngDimensions(base64: string): { width: number; height: number } | null {
try {
// atob is available in MV3 service workers.
const head = base64.length > 64 ? base64.slice(0, 64) : base64;
const bin = atob(head);
if (bin.length < 33) return null;
if (bin.slice(0, 8) !== "\x89PNG\r\n\x1a\n" || bin.slice(8, 16) !== "\x00\x00\x00\x0dIHDR") {
return null;
}
const u32 = (off: number) =>
(bin.charCodeAt(off) << 24) |
(bin.charCodeAt(off + 1) << 16) |
(bin.charCodeAt(off + 2) << 8) |
bin.charCodeAt(off + 3);
const width = u32(16) >>> 0;
const height = u32(20) >>> 0;
if (width === 0 || height === 0) return null;
return { width, height };
} catch {
return null;
}
}
@@ -0,0 +1,382 @@
import type { CdpFrame } from "@/browser-driver/frame-graph";
import type { RpcError } from "@/transport/types";
import { rpcError } from "./errors";
import {
type GeometryProjection,
projectRectToViewport,
type Size,
type ViewportRect,
} from "./geometry";
import { type CssViewport, screenshotPageRect } from "./geometry/coordinate-types";
import { cssViewport, GeometryContext } from "./geometry/frame-context";
import { parsePngDimensions } from "./png";
import { type CdpRunner, sendToCdpTarget } from "./shared";
import { isAbortError, throwIfAborted } from "./vom/capture-abort";
import { resolveVerifiedNode } from "./vom/document-identity";
import type { DocumentIdentity } from "./vom/facts";
import { VISUAL_STYLES } from "./vom/snapshot";
import type { VisualCandidate, VisualFramePath } from "./vom/visual-discovery";
import {
EMPTY_VISUAL_CONTEXT,
extendVisualContext,
projectVisualBox,
resolveVisualRegion,
type VisualClipSource,
type VisualContext,
visualProjectionIssue,
} from "./vom/visual-region";
interface LiveRow {
node: number;
tag: string;
box: ViewportRect;
client: ViewportRect;
contentSize: Size;
styles: Record<string, string>;
}
interface LiveFrame {
top: boolean;
dpr: number;
rows: LiveRow[]; // Anchor first, root last.
}
// Read only one current ancestry, including shadow hosts, in the verified isolated world.
const READ_ANCESTRY = `function(styleNames) {
if (!this.isConnected || this.ownerDocument !== document) return null;
const rows = [];
for (let node = this; node; node = node.parentElement || node.getRootNode().host) {
if (!(node instanceof Element)) return null;
const s = getComputedStyle(node), r = node.getBoundingClientRect();
rows.push({ node, tag: node.localName,
box: { x:r.x, y:r.y, width:r.width, height:r.height },
client: { x:r.x+node.clientLeft, y:r.y+node.clientTop, width:node.clientWidth, height:node.clientHeight },
contentSize: { width:node.clientWidth-parseFloat(s.paddingLeft)-parseFloat(s.paddingRight), height:node.clientHeight-parseFloat(s.paddingTop)-parseFloat(s.paddingBottom) },
styles: Object.fromEntries(styleNames.map(key => [key,s.getPropertyValue(key)])) });
}
return { top: window === window.top, dpr: devicePixelRatio, rows };
}`;
interface DeepValue {
type: string;
value?: unknown;
}
/** This read returns JSON primitives plus DOM nodes (backend IDs), never remote object handles. */
function decode(value: DeepValue): unknown {
if (value.type === "node") return (value.value as { backendNodeId?: number })?.backendNodeId;
if (value.type === "array") return (value.value as DeepValue[]).map(decode);
if (value.type === "object")
return Object.fromEntries(
(value.value as [string, DeepValue][]).map(([key, item]) => [key, decode(item)]),
);
if (value.type === "null") return null;
if (["string", "number", "boolean"].includes(value.type)) return value.value;
throw new Error("unexpected live ancestry serialization");
}
function stale(message: string): RpcError {
return rpcError(
"not_found",
"visual_target_changed",
`${message}; observe again before requesting a screenshot`,
);
}
function sameIdentity(a: DocumentIdentity, b: DocumentIdentity): boolean {
return (
a.attachmentId === b.attachmentId &&
a.frameId === b.frameId &&
a.target.tabId === b.target.tabId &&
a.target.sessionId === b.target.sessionId &&
a.documentElementBackendNodeId === b.documentElementBackendNodeId
);
}
function closeRect(a: ViewportRect, b: ViewportRect): boolean {
return [
a.x - b.x,
a.y - b.y,
a.x + a.width - b.x - b.width,
a.y + a.height - b.y - b.height,
].every((delta) => Number.isFinite(delta) && Math.abs(delta) <= 0.25);
}
function sameClips(a?: VisualClipSource, b?: VisualClipSource): boolean {
while (a && b) {
if (
!sameIdentity(a.document, b.document) ||
a.backendNodeId !== b.backendNodeId ||
a.x !== b.x ||
a.y !== b.y ||
a.overflowX !== b.overflowX ||
a.overflowY !== b.overflowY ||
!closeRect(a.box, b.box)
)
return false;
a = a.parent;
b = b.parent;
}
return !a && !b;
}
/** Scale affects raster size only, never the selected CSS region. */
export function visualScreenshotScale(width: number, height: number, pixelsPerCss: number): number {
if (![width, height, pixelsPerCss].every((n) => Number.isFinite(n) && n > 0))
throw new Error("invalid screenshot pixel dimensions");
return Math.min(
1,
2048 / (Math.max(width, height) * pixelsPerCss),
Math.sqrt(4_000_000 / (width * height * pixelsPerCss * pixelsPerCss)),
);
}
async function readFrame(
cdp: CdpRunner,
document: DocumentIdentity,
anchor: number,
signal?: AbortSignal,
): Promise<LiveFrame | RpcError> {
const verified = await resolveVerifiedNode(cdp, document, anchor, signal);
if (verified.status !== "current") return stale(`visual DOM identity ${verified.status}`);
try {
throwIfAborted(signal);
const reply = await sendToCdpTarget<{
result?: { deepSerializedValue?: DeepValue };
exceptionDetails?: unknown;
}>(cdp, document.target, "Runtime.callFunctionOn", {
objectId: verified.objectId,
objectGroup: verified.objectGroup,
functionDeclaration: READ_ANCESTRY,
arguments: [{ value: VISUAL_STYLES }],
serializationOptions: {
serialization: "deep",
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
},
});
throwIfAborted(signal);
if (reply.exceptionDetails || !reply.result?.deepSerializedValue)
return stale("visual ancestry unavailable");
const result = decode(reply.result.deepSerializedValue) as LiveFrame | null;
if (
!result?.rows?.length ||
result.rows[0].node !== anchor ||
result.rows.at(-1)?.node !== document.documentElementBackendNodeId ||
!result.rows.every((row) => Number.isSafeInteger(row.node) && row.node > 0)
)
return stale("visual ancestry incomplete");
if (cdp.getAttachmentId?.(document.target.tabId) !== document.attachmentId)
return stale("visual attachment changed");
return result;
} finally {
await sendToCdpTarget(cdp, document.target, "Runtime.releaseObjectGroup", {
objectGroup: verified.objectGroup,
}).catch(() => {});
throwIfAborted(signal);
}
}
/** All live reads for one screenshot use this one measurement context. */
async function resolveVisualRegionNow(
cdp: CdpRunner,
candidate: VisualCandidate,
signal?: AbortSignal,
): Promise<{ crop: ViewportRect; viewport: CssViewport; dpr: number } | RpcError> {
throwIfAborted(signal);
if (!candidate.framePath || !sameIdentity(candidate.document, candidate.framePath.document))
return stale("visual frame path unavailable");
const path: { frame: VisualFramePath; anchor: number }[] = [];
const seen = new Set<string>();
let frame: VisualFramePath | undefined = candidate.framePath;
let anchor = candidate.backendNodeId;
while (frame) {
if (
seen.has(frame.document.frameId) ||
frame.document.target.tabId !== candidate.document.target.tabId ||
cdp.getAttachmentId?.(frame.document.target.tabId) !== frame.document.attachmentId
)
return stale("invalid visual frame path");
seen.add(frame.document.frameId);
path.push({ frame, anchor });
anchor = frame.parent?.ownerBackendNodeId ?? 0;
frame = frame.parent?.frame;
}
path.reverse();
const frames: CdpFrame[] = path.map(({ frame }) => ({
frameId: frame.document.frameId,
target: frame.document.target,
...(frame.parent
? {
parentFrameId: frame.parent.frame.document.frameId,
ownerBackendNodeId: frame.parent.ownerBackendNodeId,
}
: {}),
}));
// Validate each recorded edge instead of asking the driver to fill all page owners.
for (const { frame } of path) {
if (!frame.parent) continue;
throwIfAborted(signal);
const owner = await sendToCdpTarget<{ backendNodeId?: number }>(
cdp,
frame.parent.frame.document.target,
"DOM.getFrameOwner",
{ frameId: frame.document.frameId },
);
if (owner.backendNodeId !== frame.parent.ownerBackendNodeId)
return stale("visual frame owner changed");
}
const geometry = new GeometryContext(
cdp,
candidate.document.target.tabId,
{ rootFrameId: frames[0].frameId, frames },
signal,
);
let context: VisualContext = EMPTY_VISUAL_CONTEXT;
let parentRead: LiveFrame | undefined;
let finalRegion: VisualCandidate["region"] | undefined;
let topDpr = 0;
for (let i = 0; i < path.length; i++) {
const { frame, anchor } = path[i];
const live = await readFrame(cdp, frame.document, anchor, signal);
if ("code" in live) return live;
if (live.top !== (i === 0)) return stale("visual root frame changed");
if (i === 0) topDpr = live.dpr;
const metrics = await geometry.layoutMetrics(frame.document.target);
const viewport = cssViewport(metrics);
const projections: GeometryProjection[] = [];
if (frame.parent) {
const parent = frame.parent.frame.document;
const size = parentRead!.rows[0].contentSize;
if (![size.width, size.height].every((n) => Number.isFinite(n) && n > 0))
return stale("iframe content size unavailable");
const parentViewport = await geometry.viewport(parent.target);
if (!parentViewport) return stale("parent viewport unavailable");
const local = await geometry.snapshotProjection(
{ target: parent.target, frameId: frame.document.frameId },
frame.parent.ownerBackendNodeId,
[],
parentViewport,
size,
);
const outer = await geometry.targetProjection(parent.frameId);
if (local.status !== "available" || !outer)
return stale("visual frame projection unavailable");
projections.push(local.projection.geometry, outer);
} else projections.push({ sourceClips: [], edges: [], topViewport: viewport });
const issue = visualProjectionIssue({
projections,
coordinates: { layoutUnitsPerCssPixel: 1, scrollCss: { x: 0, y: 0 } },
pageScale: metrics.cssVisualViewport?.scale ?? metrics.visualViewport?.scale,
});
if (issue) return stale(issue);
for (let j = live.rows.length - 1; j >= 0; j--) {
const row = live.rows[j];
const isAnchor = j === 0;
const node = {
document: frame.document,
backendNodeId: row.node,
styles: row.styles,
clientBox: projectVisualBox(row.client, projections),
};
context = extendVisualContext(
context,
node,
!isAnchor && row.tag !== "iframe" && row.tag !== "frame",
);
}
if (i < path.length - 1) {
const owner = live.rows[0];
context = {
...context,
hidden:
context.hidden ||
owner.styles.visibility === "hidden" ||
owner.styles.visibility === "collapse",
transformed: false,
localClip: false,
unpositionedClip: false,
};
} else {
const row = live.rows[0];
if (row.tag !== "canvas") return stale("visual anchor is no longer Canvas");
let visible: ViewportRect | null = row.box;
for (const projection of projections)
visible = visible
? projectRectToViewport(
{ x: visible.x, y: visible.y, w: visible.width, h: visible.height },
projection,
)
: null;
const region = resolveVisualRegion({
borderBox: projectVisualBox(row.box, projections),
frameVisibleBox: visible,
context,
visibility: row.styles.visibility,
});
if (region.status !== "available")
return stale(
`visual region ${region.status}${region.status === "unavailable" ? `: ${region.reason}` : ""}`,
);
finalRegion = region;
}
parentRead = live;
}
if (
!finalRegion ||
!closeRect(candidate.region.borderBox, finalRegion.borderBox) ||
!closeRect(candidate.region.crop, finalRegion.crop) ||
!sameClips(candidate.region.clips, finalRegion.clips)
)
return stale("visual region changed");
const viewport = cssViewport(await geometry.layoutMetrics(frames[0].target));
return { crop: finalRegion.crop, viewport, dpr: topDpr };
}
/** One target only. No frame discovery, scrolling, AX, or snapshot recapture. */
export async function captureVisualScreenshot(
cdp: CdpRunner,
candidate: VisualCandidate,
signal?: AbortSignal,
): Promise<{ image_base64: string; width: number; height: number } | RpcError> {
try {
const region = await resolveVisualRegionNow(cdp, candidate, signal);
if ("code" in region) return region;
const clip = screenshotPageRect(region.crop, region.viewport);
if (!clip) return stale("invalid screenshot coordinates");
let scale = visualScreenshotScale(region.crop.width, region.crop.height, region.dpr);
for (let attempt = 0; attempt < 2; attempt++) {
throwIfAborted(signal);
if (
cdp.getAttachmentId?.(candidate.document.target.tabId) !== candidate.document.attachmentId
)
return stale("visual attachment changed");
const shot = await cdp.send<{ data?: string }>(
candidate.document.target.tabId,
"Page.captureScreenshot",
{ format: "png", captureBeyondViewport: false, clip: { ...clip.rect, scale } },
);
throwIfAborted(signal);
const dims = shot.data ? parsePngDimensions(shot.data) : null;
if (!dims)
return rpcError(
"cdp_failed",
"screenshot_capture_failed",
"visual screenshot returned invalid PNG dimensions",
);
const correction = visualScreenshotScale(dims.width, dims.height, 1);
if (correction === 1) return { image_base64: shot.data!, ...dims };
scale *= correction * 0.99;
}
return rpcError(
"cdp_failed",
"visual_pixel_budget_exceeded",
"visual screenshot exceeds the pixel budget",
);
} catch (error) {
if (isAbortError(error) || signal?.aborted)
return { code: "cancelled", message: "visual screenshot aborted" };
return rpcError(
"cdp_failed",
"screenshot_capture_failed",
error instanceof Error ? error.message : String(error),
);
}
}
@@ -141,6 +141,32 @@ function facts(
}
describe("Canvas discovery", () => {
it("shares frame address paths from existing Facts without changing candidate evidence", async () => {
const parent = await document([{ id: 2, parent: 1, tag: "iframe" }]);
const child = await document(
[
{ id: 3, parent: 1 },
{ id: 4, parent: 1 },
],
{
frame: {
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 2,
target: { tabId: 1 },
},
},
);
const result = await discoverVisualCandidates(facts([child, parent]));
expect(result.candidates).toHaveLength(2);
const path = result.candidates[0].framePath;
expect(path).toBe(result.candidates[1].framePath);
expect(path?.document).toBe(child.identity);
expect(path?.parent?.ownerBackendNodeId).toBe(2);
expect(path?.parent?.frame.document).toBe(parent.identity);
expect(path?.parent?.frame.parent).toBeUndefined();
});
it("distinguishes visual facts not collected from complete empty discovery and honors cancellation", async () => {
const empty = facts([]);
expect(await discoverVisualCandidates(empty)).toMatchObject({
@@ -11,7 +11,7 @@ export async function verifyDocumentIdentity(
identity: DocumentIdentity,
signal?: AbortSignal,
): Promise<"current" | "changed" | "unavailable"> {
return verifyIdentity(cdp, identity, signal);
return (await verifyIdentity(cdp, identity, signal)).status;
}
/** Verify the visual anchor in its original frame; geometry is checked by screenshot execution. */
@@ -20,7 +20,24 @@ export async function verifyVisualTargetIdentity(
candidate: VisualCandidate,
signal?: AbortSignal,
): Promise<"current" | "changed" | "unavailable"> {
return verifyIdentity(cdp, candidate.document, signal, candidate.backendNodeId);
return (await verifyIdentity(cdp, candidate.document, signal, candidate.backendNodeId)).status;
}
export type VerifiedNodeResult =
| { status: "changed" | "unavailable" }
| { status: "current"; objectId: string; objectGroup: string };
/** Caller must release objectGroup in finally after its local read. */
export async function resolveVerifiedNode(
cdp: CdpRunner,
identity: DocumentIdentity,
backendNodeId: number,
signal?: AbortSignal,
): Promise<VerifiedNodeResult> {
const result = await verifyIdentity(cdp, identity, signal, backendNodeId, true);
return result.status === "current" && result.objectId && result.objectGroup
? { status: "current", objectId: result.objectId, objectGroup: result.objectGroup }
: { status: result.status === "changed" ? "changed" : "unavailable" };
}
async function verifyIdentity(
@@ -28,15 +45,22 @@ async function verifyIdentity(
identity: DocumentIdentity,
signal?: AbortSignal,
anchorBackendNodeId?: number,
): Promise<"current" | "changed" | "unavailable"> {
retain = false,
): Promise<{
status: "current" | "changed" | "unavailable";
objectId?: string;
objectGroup?: string;
}> {
const attached = () => cdp.getAttachmentId?.(identity.target.tabId) === identity.attachmentId;
throwIfAborted(signal);
if (!attached()) return "changed";
if (!attached()) return { status: "changed" };
const objectGroup = `bsk-document-identity-${crypto.randomUUID()}`;
const send = <T>(method: string, params: object) => {
throwIfAborted(signal);
return sendToCdpTarget<T>(cdp, identity.target, method, params);
};
let objectId: string | undefined;
let retained = false;
try {
const world = await send<{ executionContextId: number }>("Page.createIsolatedWorld", {
frameId: identity.frameId,
@@ -63,7 +87,8 @@ async function verifyIdentity(
executionContextId: world.executionContextId,
objectGroup,
});
if (!anchor.object?.objectId) return "unavailable";
if (!anchor.object?.objectId) return { status: "unavailable" };
objectId = anchor.object.objectId;
reply = await send<RootReply>("Runtime.callFunctionOn", {
objectId: anchor.object.objectId,
functionDeclaration:
@@ -72,20 +97,24 @@ async function verifyIdentity(
serializationOptions,
});
}
if (!attached()) return "changed";
if (!attached()) return { status: "changed" };
const root = reply.result?.deepSerializedValue;
if (root?.type === "null") return "changed";
if (root?.type === "null") return { status: "changed" };
const id = root?.type === "node" ? root.value?.backendNodeId : undefined;
if (id === undefined) return "unavailable";
return id === identity.documentElementBackendNodeId ? "current" : "changed";
if (id === undefined) return { status: "unavailable" };
if (id !== identity.documentElementBackendNodeId) return { status: "changed" };
throwIfAborted(signal);
retained = retain && !!objectId;
return { status: "current", ...(retained ? { objectId, objectGroup } : {}) };
} catch (error) {
throwIfAborted(signal);
if (isAbortError(error)) throw error;
return attached() ? "unavailable" : "changed";
return { status: attached() ? "unavailable" : "changed" };
} finally {
await sendToCdpTarget(cdp, identity.target, "Runtime.releaseObjectGroup", {
objectGroup,
}).catch(() => {});
if (!retained)
await sendToCdpTarget(cdp, identity.target, "Runtime.releaseObjectGroup", {
objectGroup,
}).catch(() => {});
throwIfAborted(signal);
}
}
@@ -20,11 +20,19 @@ import {
visualProjectionIssue,
} from "./visual-region";
/** Shared observation-time addresses, not cached live geometry. */
export interface VisualFramePath {
readonly document: DocumentIdentity;
readonly parent?: { readonly ownerBackendNodeId: number; readonly frame: VisualFramePath };
}
export interface VisualCandidate {
readonly document: DocumentIdentity;
readonly backendNodeId: number;
readonly parentBackendNodeId: number | null;
readonly label?: string;
/** Absent when execution ancestry could not be established; discovery is still retained. */
readonly framePath?: VisualFramePath;
readonly region: Extract<VisualRegionResult, { status: "available" }>;
}
@@ -113,6 +121,7 @@ export async function discoverVisualCandidates(
const candidates: VisualCandidate[] = [];
const issues: VisualDiscoveryIssue[] = [];
const documents = new Map(facts.documents.map((document) => [document.frame.frameId, document]));
const framePaths = new Map<string, VisualFramePath>();
const contexts = new Map<string, Map<number, NodeContext>>();
const roots = new Map<string, VisualContext>();
@@ -248,6 +257,16 @@ export async function discoverVisualCandidates(
root.issue ??
(!doc.identity ? "identity-unverified" : visualProjectionIssue(doc.geometry)),
};
if (doc.identity) {
const parentPath = frame.parentFrameId ? framePaths.get(frame.parentFrameId) : undefined;
if (!frame.parentFrameId && frame.frameId === facts.rootFrameId)
framePaths.set(frame.frameId, { document: doc.identity });
else if (parentPath && frame.ownerBackendNodeId !== undefined)
framePaths.set(frame.frameId, {
document: doc.identity,
parent: { ownerBackendNodeId: frame.ownerBackendNodeId, frame: parentPath },
});
}
roots.set(frame.frameId, root);
contexts.set(frame.frameId, new Map());
}
@@ -291,6 +310,7 @@ export async function discoverVisualCandidates(
const label = node.attrs["aria-label"]?.trim() || node.attrs.title?.trim();
candidates.push({
document: document.identity,
framePath: framePaths.get(document.frame.frameId),
backendNodeId: node.backendNodeId,
parentBackendNodeId: node.parentBackendNodeId,
...(label ? { label } : {}),
+2
View File
@@ -25,6 +25,8 @@ export type RpcErrorReason =
| "element_not_visible"
| "ref_not_found"
| "ref_kind_unsupported"
| "visual_target_changed"
| "visual_pixel_budget_exceeded"
| "selector_not_found"
| "target_not_fillable"
| "fill_value_invalid"
+24
View File
@@ -40,6 +40,8 @@ use bsk_protocol::ErrorCode;
pub mod reason {
pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope";
pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible";
pub const VISUAL_TARGET_CHANGED: &str = "visual_target_changed";
pub const VISUAL_PIXEL_BUDGET_EXCEEDED: &str = "visual_pixel_budget_exceeded";
pub const REF_KIND_UNSUPPORTED: &str = "ref_kind_unsupported";
pub const REF_NOT_FOUND: &str = "ref_not_found";
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
@@ -248,6 +250,18 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
),
exit_code: base.exit_code,
},
(ErrorCode::NotFound, reason::VISUAL_TARGET_CHANGED) => RenderInfo {
summary: "visual target needs a fresh observation",
hint: Some(
"observe the page again and use the current visual ref; the previous identity or region cannot be used",
),
exit_code: base.exit_code,
},
(ErrorCode::CdpFailed, reason::VISUAL_PIXEL_BUDGET_EXCEEDED) => RenderInfo {
summary: "visual screenshot exceeds the image size limit",
hint: Some("the screenshot could not be reduced within the pixel budget"),
exit_code: base.exit_code,
},
(ErrorCode::Unsupported, reason::REF_KIND_UNSUPPORTED) => RenderInfo {
summary: "this tool does not support the ref target type",
hint: Some(
@@ -672,6 +686,16 @@ mod tests {
assert!(info.summary.contains("timed out after browser dispatch"));
}
#[test]
fn visual_screenshot_failures_have_specific_guidance() {
let changed = serde_json::json!({ "reason": reason::VISUAL_TARGET_CHANGED });
let info = info_for_error(ErrorCode::NotFound, Some(&changed));
assert!(info.hint.unwrap().contains("observe"));
let budget = serde_json::json!({ "reason": reason::VISUAL_PIXEL_BUDGET_EXCEEDED });
let info = info_for_error(ErrorCode::CdpFailed, Some(&budget));
assert!(info.summary.contains("image size limit"));
}
#[test]
fn visual_ref_type_error_has_specific_guidance() {
let data = serde_json::json!({ "reason": reason::REF_KIND_UNSUPPORTED });