mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat: support on-demand screenshots for canvas visual refs
This commit is contained in:
@@ -4,7 +4,7 @@ import { RefStore, type VisualRefInput } from "@/session-manager/ref-store";
|
||||
import { handleDownload } from "../download";
|
||||
import { handleRequestHelp, resetHelpLifecycleForTests } from "../human-loop";
|
||||
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "../interaction";
|
||||
import { handleGetHtml, handleScreenshot } from "../observation";
|
||||
import { handleGetHtml } from "../observation";
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { lookupRefTarget, lookupSnapshotRef, resolveSnapshotRef } from "../snapshot-ref";
|
||||
import { handleUpload } from "../upload";
|
||||
@@ -117,12 +117,10 @@ describe("typed visual refs", () => {
|
||||
"press",
|
||||
"select",
|
||||
"get_html",
|
||||
"screenshot",
|
||||
])("rejects visual refs in %s before target effects", async (tool) => {
|
||||
const { manager, send, cdp, tabsApi } = await setup();
|
||||
const params = { session_id: "test", tab_id: 4, ref: "@e1" };
|
||||
const deps = { cdp, tabsApi };
|
||||
const captureVisibleTab = vi.fn();
|
||||
let result: unknown;
|
||||
switch (tool) {
|
||||
case "click":
|
||||
@@ -143,16 +141,9 @@ describe("typed visual refs", () => {
|
||||
case "get_html":
|
||||
result = await handleGetHtml(manager, params, deps);
|
||||
break;
|
||||
case "screenshot":
|
||||
result = await handleScreenshot(manager, params, {
|
||||
...deps,
|
||||
captureApi: { ...tabsApi, captureVisibleTab },
|
||||
});
|
||||
break;
|
||||
}
|
||||
expect(result).toMatchObject({ code: "unsupported", data: { reason: "ref_kind_unsupported" } });
|
||||
expect(send).not.toHaveBeenCalled();
|
||||
expect(captureVisibleTab).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
|
||||
@@ -0,0 +1,373 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { handleScreenshot } from "../observation";
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { captureVisualScreenshot, visualScreenshotScale } from "../visual-screenshot";
|
||||
import type { VisualCandidate, VisualFramePath } from "../vom/visual-discovery";
|
||||
|
||||
const styles = {
|
||||
position: "static",
|
||||
visibility: "visible",
|
||||
opacity: "1",
|
||||
display: "block",
|
||||
"overflow-x": "visible",
|
||||
"overflow-y": "visible",
|
||||
transform: "none",
|
||||
zoom: "1",
|
||||
"clip-path": "none",
|
||||
"mask-image": "none",
|
||||
rotate: "none",
|
||||
scale: "none",
|
||||
perspective: "none",
|
||||
clip: "auto",
|
||||
contain: "none",
|
||||
"overflow-clip-margin": "0px",
|
||||
};
|
||||
const rect = (x = 10, y = 20, width = 100, height = 40) => ({ x, y, width, height });
|
||||
const root = {
|
||||
attachmentId: "a",
|
||||
target: { tabId: 4 },
|
||||
frameId: "top",
|
||||
documentElementBackendNodeId: 1,
|
||||
};
|
||||
function row(id: number, tag: string, box = rect()) {
|
||||
return {
|
||||
node: { backend: id },
|
||||
tag,
|
||||
box,
|
||||
client: box,
|
||||
contentSize: { width: box.width, height: box.height },
|
||||
styles: { ...styles },
|
||||
};
|
||||
}
|
||||
function encode(value: unknown): unknown {
|
||||
if (value === null) return { type: "null" };
|
||||
if (Array.isArray(value)) return { type: "array", value: value.map(encode) };
|
||||
if (typeof value === "object") {
|
||||
const record = value as Record<string, unknown>;
|
||||
if ("backend" in record) return { type: "node", value: { backendNodeId: record.backend } };
|
||||
return { type: "object", value: Object.entries(record).map(([k, v]) => [k, encode(v)]) };
|
||||
}
|
||||
return { type: typeof value, value };
|
||||
}
|
||||
function png(width: number, height: number) {
|
||||
const bytes = new Uint8Array(33);
|
||||
bytes.set([137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82]);
|
||||
const view = new DataView(bytes.buffer);
|
||||
view.setUint32(16, width);
|
||||
view.setUint32(20, height);
|
||||
return btoa(String.fromCharCode(...bytes));
|
||||
}
|
||||
function fixture(child = false, oopif = false) {
|
||||
const parent: VisualFramePath = { document: root };
|
||||
const frame: VisualFramePath = child
|
||||
? {
|
||||
document: {
|
||||
...root,
|
||||
frameId: "child",
|
||||
documentElementBackendNodeId: 11,
|
||||
target: oopif ? { tabId: 4, sessionId: "oopif" } : { tabId: 4 },
|
||||
},
|
||||
parent: { frame: parent, ownerBackendNodeId: 2 },
|
||||
}
|
||||
: parent;
|
||||
const crop = child ? rect(120, 240, 200, 80) : rect();
|
||||
const candidate: VisualCandidate = {
|
||||
document: frame.document,
|
||||
backendNodeId: child ? 12 : 3,
|
||||
parentBackendNodeId: frame.document.documentElementBackendNodeId,
|
||||
framePath: frame,
|
||||
region: { status: "available", borderBox: crop, crop },
|
||||
};
|
||||
const topRows = child
|
||||
? [row(2, "iframe", rect(100, 200, 400, 200)), row(1, "html", rect(0, 0, 1200, 800))]
|
||||
: [row(3, "canvas"), row(1, "html", rect(0, 0, 1200, 800))];
|
||||
if (child) topRows[0].contentSize = { width: 200, height: 100 };
|
||||
const childRows = [row(12, "canvas"), row(11, "html", rect(0, 0, 200, 100))];
|
||||
let calls = 0;
|
||||
const control = {
|
||||
wrongOwner: false,
|
||||
rootChanged: false,
|
||||
failRead: false,
|
||||
abortRead: false,
|
||||
dpr: 1,
|
||||
shots: [png(crop.width, crop.height)],
|
||||
};
|
||||
const controller = new AbortController();
|
||||
const send = vi.fn(
|
||||
async (target: CdpTarget, method: string, params: Record<string, unknown> = {}) => {
|
||||
if (method === "DOM.getFrameOwner") return { backendNodeId: control.wrongOwner ? 999 : 2 };
|
||||
if (method === "Page.createIsolatedWorld")
|
||||
return { executionContextId: params.frameId === "child" ? 11 : 1 };
|
||||
if (method === "DOM.resolveNode")
|
||||
return { object: { objectId: String(params.backendNodeId) } };
|
||||
if (method === "Runtime.callFunctionOn") {
|
||||
const isChild = params.objectId === "12";
|
||||
if (!(params.functionDeclaration as string).includes("styleNames"))
|
||||
return {
|
||||
result: {
|
||||
deepSerializedValue: {
|
||||
type: "node",
|
||||
value: { backendNodeId: control.rootChanged ? 999 : isChild ? 11 : 1 },
|
||||
},
|
||||
},
|
||||
};
|
||||
if (control.failRead) throw new Error("read failed");
|
||||
if (control.abortRead) controller.abort();
|
||||
return {
|
||||
result: {
|
||||
deepSerializedValue: encode({
|
||||
top: !isChild,
|
||||
dpr: control.dpr,
|
||||
rows: isChild ? childRows : topRows,
|
||||
}),
|
||||
},
|
||||
};
|
||||
}
|
||||
if (method === "Runtime.releaseObjectGroup") return {};
|
||||
if (method === "Page.getLayoutMetrics")
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 1200, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
cssVisualViewport: { scale: 1, zoom: 1 },
|
||||
};
|
||||
if (method === "DOM.getBoxModel")
|
||||
return { model: { content: [100, 200, 500, 200, 500, 400, 100, 400] } };
|
||||
if (method === "Page.captureScreenshot")
|
||||
return { data: control.shots[Math.min(calls++, control.shots.length - 1)] };
|
||||
throw new Error(`unexpected ${method} ${target.sessionId}`);
|
||||
},
|
||||
);
|
||||
const cdp = {
|
||||
send: ((tabId, method, params) =>
|
||||
send({ tabId }, method, params as Record<string, unknown>)) as CdpRunner["send"],
|
||||
sendToTarget: send as unknown as CdpRunner["sendToTarget"],
|
||||
getAttachmentId: () => "a",
|
||||
getFrameGraph: vi.fn(async () => {
|
||||
throw new Error("whole page discovery forbidden");
|
||||
}),
|
||||
};
|
||||
return { candidate, cdp, send, control, controller, topRows, childRows };
|
||||
}
|
||||
|
||||
describe("visual screenshot", () => {
|
||||
it.each([
|
||||
[false, false],
|
||||
[true, false],
|
||||
[true, true],
|
||||
])("screenshots the local target child=%s oopif=%s without whole-page discovery", async (child, oopif) => {
|
||||
const f = fixture(child, oopif);
|
||||
const result = await captureVisualScreenshot(f.cdp, f.candidate);
|
||||
expect(result).toMatchObject({ width: child ? 200 : 100, height: child ? 80 : 40 });
|
||||
expect(f.cdp.getFrameGraph).not.toHaveBeenCalled();
|
||||
const names = f.send.mock.calls.map((c) => c[1]);
|
||||
expect(names.filter((n) => n === "DOM.getFrameOwner")).toHaveLength(child ? 1 : 0);
|
||||
expect(names.filter((n) => n === "DOM.resolveNode")).toHaveLength(child ? 2 : 1);
|
||||
expect(names.some((n) => /scroll|DOMSnapshot|getFrameTree|Accessibility/.test(n))).toBe(false);
|
||||
expect(f.send.mock.calls.find((c) => c[1] === "Page.captureScreenshot")?.[2]).toMatchObject({
|
||||
clip: { ...f.candidate.region.crop, scale: 1 },
|
||||
});
|
||||
});
|
||||
it.each([
|
||||
false,
|
||||
true,
|
||||
])("projects a nested child through a parent target oopif=%s", async (oopif) => {
|
||||
const f = fixture(true, oopif);
|
||||
const parent = f.candidate.framePath!;
|
||||
const document = {
|
||||
...parent.document,
|
||||
frameId: "grandchild",
|
||||
documentElementBackendNodeId: 21,
|
||||
};
|
||||
const crop = rect(130, 250, 40, 20);
|
||||
const candidate: VisualCandidate = {
|
||||
...f.candidate,
|
||||
document,
|
||||
backendNodeId: 22,
|
||||
framePath: { document, parent: { frame: parent, ownerBackendNodeId: 12 } },
|
||||
region: { status: "available", borderBox: crop, crop },
|
||||
};
|
||||
f.childRows[0].tag = "iframe";
|
||||
f.control.shots = [png(40, 20)];
|
||||
const original = f.send.getMockImplementation()!;
|
||||
f.send.mockImplementation(async (target, method, params = {}) => {
|
||||
if (method === "DOM.getFrameOwner" && params.frameId === "grandchild")
|
||||
return { backendNodeId: 12 };
|
||||
if (method === "Runtime.callFunctionOn" && params.objectId === "22") {
|
||||
if (!(params.functionDeclaration as string).includes("styleNames"))
|
||||
return {
|
||||
result: { deepSerializedValue: { type: "node", value: { backendNodeId: 21 } } },
|
||||
};
|
||||
return {
|
||||
result: {
|
||||
deepSerializedValue: encode({
|
||||
top: false,
|
||||
dpr: 1,
|
||||
rows: [row(22, "canvas", rect(5, 5, 20, 10)), row(21, "html", rect(0, 0, 100, 40))],
|
||||
}),
|
||||
},
|
||||
};
|
||||
}
|
||||
if (method === "DOM.getBoxModel" && params.backendNodeId === 12)
|
||||
return {
|
||||
model: {
|
||||
content: oopif
|
||||
? [10, 20, 110, 20, 110, 60, 10, 60]
|
||||
: [120, 240, 320, 240, 320, 320, 120, 320],
|
||||
},
|
||||
};
|
||||
if (method === "Page.getLayoutMetrics" && target.sessionId)
|
||||
return {
|
||||
cssLayoutViewport: { clientWidth: 200, clientHeight: 100, pageX: 0, pageY: 0 },
|
||||
cssVisualViewport: { scale: 1, zoom: 1 },
|
||||
};
|
||||
return original(target, method, params);
|
||||
});
|
||||
expect(await captureVisualScreenshot(f.cdp, candidate)).toMatchObject({
|
||||
width: 40,
|
||||
height: 20,
|
||||
});
|
||||
expect(f.send.mock.calls.filter((c) => c[1] === "DOM.getFrameOwner")).toHaveLength(2);
|
||||
expect(f.send.mock.calls.find((c) => c[1] === "Page.captureScreenshot")?.[2]).toMatchObject({
|
||||
clip: crop,
|
||||
});
|
||||
expect(f.cdp.getFrameGraph).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("runs the live reader on only connected local ancestry", async () => {
|
||||
const f = fixture();
|
||||
await captureVisualScreenshot(f.cdp, f.candidate);
|
||||
const params = f.send.mock.calls.find(
|
||||
(c) =>
|
||||
c[1] === "Runtime.callFunctionOn" &&
|
||||
String(c[2]?.functionDeclaration).includes("styleNames"),
|
||||
)![2]!;
|
||||
const read = new Function(`return (${params.functionDeclaration});`)() as (
|
||||
this: Element,
|
||||
names: string[],
|
||||
) => { rows: { node: Element }[] } | null;
|
||||
const canvas = document.createElement("canvas");
|
||||
document.body.append(canvas);
|
||||
try {
|
||||
const result = read.call(canvas, ["display"]);
|
||||
expect(result?.rows.map((row) => row.node)).toEqual([
|
||||
canvas,
|
||||
document.body,
|
||||
document.documentElement,
|
||||
]);
|
||||
canvas.remove();
|
||||
expect(read.call(canvas, ["display"])).toBeNull();
|
||||
const other = document.implementation.createHTMLDocument();
|
||||
const foreign = other.createElement("canvas");
|
||||
other.body.append(foreign);
|
||||
expect(foreign.ownerDocument).toBe(other);
|
||||
expect(read.call(foreign, ["display"])).toBeNull();
|
||||
} finally {
|
||||
canvas.remove();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps requests constant as ordinary ancestry grows", async () => {
|
||||
const small = fixture(),
|
||||
large = fixture();
|
||||
large.topRows.splice(
|
||||
1,
|
||||
0,
|
||||
...Array.from({ length: 100 }, (_, i) => row(100 + i, "div", rect(0, 0, 1200, 800))),
|
||||
);
|
||||
await captureVisualScreenshot(small.cdp, small.candidate);
|
||||
await captureVisualScreenshot(large.cdp, large.candidate);
|
||||
expect(large.send.mock.calls.map((c) => c[1])).toEqual(small.send.mock.calls.map((c) => c[1]));
|
||||
});
|
||||
it("preserves missing-path candidates but cannot execute them", async () => {
|
||||
const f = fixture();
|
||||
const { framePath: _, ...candidate } = f.candidate;
|
||||
expect(await captureVisualScreenshot(f.cdp, candidate)).toMatchObject({
|
||||
data: { reason: "visual_target_changed" },
|
||||
});
|
||||
expect(f.send).not.toHaveBeenCalled();
|
||||
});
|
||||
it.each([0.249, 0.25, 0.251])("compares rectangle edges with tolerance %s", async (delta) => {
|
||||
const f = fixture();
|
||||
f.topRows[0].box.x += delta;
|
||||
const result = await captureVisualScreenshot(f.cdp, f.candidate);
|
||||
expect("code" in result).toBe(delta > 0.25);
|
||||
});
|
||||
it("rejects a new clipping ancestor even if the resulting crop is identical", async () => {
|
||||
const f = fixture();
|
||||
const clip = row(8, "div", rect(0, 0, 1200, 800));
|
||||
clip.styles["overflow-x"] = "hidden";
|
||||
f.topRows.splice(1, 0, clip);
|
||||
expect(await captureVisualScreenshot(f.cdp, f.candidate)).toMatchObject({
|
||||
data: { reason: "visual_target_changed" },
|
||||
});
|
||||
expect(f.send.mock.calls.some((c) => c[1] === "Page.captureScreenshot")).toBe(false);
|
||||
});
|
||||
it("rejects changed owner and changed DOM before capture", async () => {
|
||||
for (const kind of ["wrongOwner", "rootChanged"] as const) {
|
||||
const f = fixture(true);
|
||||
f.control[kind] = true;
|
||||
expect(await captureVisualScreenshot(f.cdp, f.candidate)).toMatchObject({
|
||||
data: { reason: "visual_target_changed" },
|
||||
});
|
||||
expect(f.send.mock.calls.some((c) => c[1] === "Page.captureScreenshot")).toBe(false);
|
||||
}
|
||||
});
|
||||
it("releases retained objects on read failure and cancellation", async () => {
|
||||
for (const kind of ["failRead", "abortRead"] as const) {
|
||||
const f = fixture();
|
||||
f.control[kind] = true;
|
||||
const result = await captureVisualScreenshot(f.cdp, f.candidate, f.controller.signal);
|
||||
expect(result).toMatchObject({ code: kind === "abortRead" ? "cancelled" : "cdp_failed" });
|
||||
expect(f.send.mock.calls.at(-1)?.[1]).toBe("Runtime.releaseObjectGroup");
|
||||
}
|
||||
});
|
||||
it("plans pixels without enlarging images and retries at most once", async () => {
|
||||
expect(visualScreenshotScale(4096, 2048, 2)).toBe(0.25);
|
||||
expect(visualScreenshotScale(4000, 4000, 1)).toBe(0.5);
|
||||
const f = fixture();
|
||||
f.control.shots = [png(3000, 3000), png(1800, 1800)];
|
||||
expect(await captureVisualScreenshot(f.cdp, f.candidate)).toMatchObject({
|
||||
width: 1800,
|
||||
height: 1800,
|
||||
});
|
||||
const shots = f.send.mock.calls.filter((c) => c[1] === "Page.captureScreenshot");
|
||||
expect(shots).toHaveLength(2);
|
||||
expect((shots[1][2]!.clip as { scale: number }).scale).toBeLessThan(1);
|
||||
const fail = fixture();
|
||||
fail.control.shots = [png(3000, 3000)];
|
||||
expect(await captureVisualScreenshot(fail.cdp, fail.candidate)).toMatchObject({
|
||||
data: { reason: "visual_pixel_budget_exceeded" },
|
||||
});
|
||||
expect(fail.send.mock.calls.filter((c) => c[1] === "Page.captureScreenshot")).toHaveLength(2);
|
||||
});
|
||||
it("rejects invalid PNG without guessing dimensions", async () => {
|
||||
const f = fixture();
|
||||
f.control.shots = ["invalid"];
|
||||
expect(await captureVisualScreenshot(f.cdp, f.candidate)).toMatchObject({
|
||||
data: { reason: "screenshot_capture_failed" },
|
||||
});
|
||||
});
|
||||
it("dispatches a visual ref through the screenshot tool and restores overlays", async () => {
|
||||
const f = fixture();
|
||||
const manager = new SessionManager({
|
||||
agentWindow: {
|
||||
create: async () => 100,
|
||||
remove: async () => {},
|
||||
ensureActiveTab: async () => 4,
|
||||
},
|
||||
});
|
||||
const ctx = await manager.start("test");
|
||||
ctx.refStore.replace([["e1", { kind: "visual-region", candidate: f.candidate }]]);
|
||||
const tab = { id: 4, windowId: 100, active: true } as chrome.tabs.Tab;
|
||||
const tabsApi = { get: async () => tab, query: async () => [tab] };
|
||||
const sendToTab = vi.fn(async () => ({}));
|
||||
const result = await handleScreenshot(
|
||||
manager,
|
||||
{ session_id: "test", ref: "e1" },
|
||||
{ cdp: f.cdp, tabsApi, captureApi: { ...tabsApi, captureVisibleTab: vi.fn() }, sendToTab },
|
||||
);
|
||||
expect(result).toMatchObject({ width: 100, height: 40, format: "png" });
|
||||
expect(sendToTab).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
@@ -212,11 +212,16 @@ export class GeometryContext {
|
||||
ownerBackendNodeId: number,
|
||||
ancestorClips: Polygon[],
|
||||
viewport: Size,
|
||||
contentSize?: Size,
|
||||
): Promise<SnapshotProjectionResult> {
|
||||
const key = `${cdpTargetKey(source.target)}:${ownerBackendNodeId}`;
|
||||
let promise = this.snapshotOwners.get(key);
|
||||
if (!promise) {
|
||||
promise = this.snapshotOwner(source.target, ownerBackendNodeId);
|
||||
promise = contentSize
|
||||
? this.ownerContent(source.target, ownerBackendNodeId).then((quad) =>
|
||||
quad ? { quad, size: contentSize } : null,
|
||||
)
|
||||
: this.snapshotOwner(source.target, ownerBackendNodeId);
|
||||
this.snapshotOwners.set(key, promise);
|
||||
}
|
||||
let owner: { quad: Quad; size: Size } | null;
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
import { parsePngDimensions } from "./png";
|
||||
import { captureVisualScreenshot } from "./visual-screenshot";
|
||||
|
||||
export { parsePngDimensions } from "./png";
|
||||
|
||||
import type { CapturedSceneInput } from "./vom/facts";
|
||||
// Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`,
|
||||
// and semantic `tool.observe` (design §7). Each handler resolves the target
|
||||
@@ -101,38 +106,6 @@ export function stripDataUrlPrefix(dataUrl: string): string {
|
||||
return m ? dataUrl.slice(m[0].length) : dataUrl;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a PNG's IHDR chunk and return `(width, height)`. Returns
|
||||
* `null` on any malformed input so callers fall back to `0/0` instead
|
||||
* of throwing.
|
||||
*
|
||||
* PNG layout: 8-byte signature, then a 4-byte length, 4-byte type
|
||||
* ("IHDR"), then the chunk data — width is bytes 16-19 BE, height is
|
||||
* 20-23 BE.
|
||||
*/
|
||||
export function parsePngDimensions(base64: string): { width: number; height: number } | null {
|
||||
try {
|
||||
// atob is available in MV3 service workers.
|
||||
const head = base64.length > 64 ? base64.slice(0, 64) : base64;
|
||||
const bin = atob(head);
|
||||
if (bin.length < 24) return null;
|
||||
if (bin.charCodeAt(0) !== 0x89 || bin.charCodeAt(1) !== 0x50 || bin.charCodeAt(2) !== 0x4e) {
|
||||
return null;
|
||||
}
|
||||
const u32 = (off: number) =>
|
||||
(bin.charCodeAt(off) << 24) |
|
||||
(bin.charCodeAt(off + 1) << 16) |
|
||||
(bin.charCodeAt(off + 2) << 8) |
|
||||
bin.charCodeAt(off + 3);
|
||||
const width = u32(16) >>> 0;
|
||||
const height = u32(20) >>> 0;
|
||||
if (width === 0 || height === 0) return null;
|
||||
return { width, height };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export interface ScreenshotDeps {
|
||||
cdp?: SharedCdpRunner;
|
||||
tabsApi: ChromeTabsApi;
|
||||
@@ -308,12 +281,17 @@ export async function handleScreenshot(
|
||||
if (!deps.cdp) {
|
||||
return { code: "cdp_failed", message: "screenshot ref capture requires CDP" };
|
||||
}
|
||||
if (lookupRefTarget(ctx, ref, target.tabId)?.kind === "visual-region")
|
||||
return rpcError(
|
||||
"unsupported",
|
||||
"ref_kind_unsupported",
|
||||
"visual-region screenshot execution is not available in this build",
|
||||
const entry = lookupRefTarget(ctx, ref, target.tabId);
|
||||
if (entry?.kind === "visual-region") {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const captured = await withExtensionOverlayHidden(
|
||||
target.tabId,
|
||||
() => captureVisualScreenshot(deps.cdp!, entry.candidate, signal),
|
||||
deps.sendToTab,
|
||||
);
|
||||
if (isRpcError(captured)) return captured;
|
||||
return withShotDialogs({ ...captured, format: "png", tab_id: target.tabId });
|
||||
}
|
||||
const node = resolveSnapshotRef(ctx, ref, target.tabId);
|
||||
if (isRpcError(node)) return node;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
/**
|
||||
* Parse a PNG's IHDR chunk and return `(width, height)`. Returns
|
||||
* `null` on any malformed input so callers fall back to `0/0` instead
|
||||
* of throwing.
|
||||
*
|
||||
* PNG layout: 8-byte signature, then a 4-byte length, 4-byte type
|
||||
* ("IHDR"), then the chunk data — width is bytes 16-19 BE, height is
|
||||
* 20-23 BE.
|
||||
*/
|
||||
export function parsePngDimensions(base64: string): { width: number; height: number } | null {
|
||||
try {
|
||||
// atob is available in MV3 service workers.
|
||||
const head = base64.length > 64 ? base64.slice(0, 64) : base64;
|
||||
const bin = atob(head);
|
||||
if (bin.length < 33) return null;
|
||||
if (bin.slice(0, 8) !== "\x89PNG\r\n\x1a\n" || bin.slice(8, 16) !== "\x00\x00\x00\x0dIHDR") {
|
||||
return null;
|
||||
}
|
||||
const u32 = (off: number) =>
|
||||
(bin.charCodeAt(off) << 24) |
|
||||
(bin.charCodeAt(off + 1) << 16) |
|
||||
(bin.charCodeAt(off + 2) << 8) |
|
||||
bin.charCodeAt(off + 3);
|
||||
const width = u32(16) >>> 0;
|
||||
const height = u32(20) >>> 0;
|
||||
if (width === 0 || height === 0) return null;
|
||||
return { width, height };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,382 @@
|
||||
import type { CdpFrame } from "@/browser-driver/frame-graph";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
import {
|
||||
type GeometryProjection,
|
||||
projectRectToViewport,
|
||||
type Size,
|
||||
type ViewportRect,
|
||||
} from "./geometry";
|
||||
import { type CssViewport, screenshotPageRect } from "./geometry/coordinate-types";
|
||||
import { cssViewport, GeometryContext } from "./geometry/frame-context";
|
||||
import { parsePngDimensions } from "./png";
|
||||
import { type CdpRunner, sendToCdpTarget } from "./shared";
|
||||
import { isAbortError, throwIfAborted } from "./vom/capture-abort";
|
||||
import { resolveVerifiedNode } from "./vom/document-identity";
|
||||
import type { DocumentIdentity } from "./vom/facts";
|
||||
import { VISUAL_STYLES } from "./vom/snapshot";
|
||||
import type { VisualCandidate, VisualFramePath } from "./vom/visual-discovery";
|
||||
import {
|
||||
EMPTY_VISUAL_CONTEXT,
|
||||
extendVisualContext,
|
||||
projectVisualBox,
|
||||
resolveVisualRegion,
|
||||
type VisualClipSource,
|
||||
type VisualContext,
|
||||
visualProjectionIssue,
|
||||
} from "./vom/visual-region";
|
||||
|
||||
interface LiveRow {
|
||||
node: number;
|
||||
tag: string;
|
||||
box: ViewportRect;
|
||||
client: ViewportRect;
|
||||
contentSize: Size;
|
||||
styles: Record<string, string>;
|
||||
}
|
||||
interface LiveFrame {
|
||||
top: boolean;
|
||||
dpr: number;
|
||||
rows: LiveRow[]; // Anchor first, root last.
|
||||
}
|
||||
|
||||
// Read only one current ancestry, including shadow hosts, in the verified isolated world.
|
||||
const READ_ANCESTRY = `function(styleNames) {
|
||||
if (!this.isConnected || this.ownerDocument !== document) return null;
|
||||
const rows = [];
|
||||
for (let node = this; node; node = node.parentElement || node.getRootNode().host) {
|
||||
if (!(node instanceof Element)) return null;
|
||||
const s = getComputedStyle(node), r = node.getBoundingClientRect();
|
||||
rows.push({ node, tag: node.localName,
|
||||
box: { x:r.x, y:r.y, width:r.width, height:r.height },
|
||||
client: { x:r.x+node.clientLeft, y:r.y+node.clientTop, width:node.clientWidth, height:node.clientHeight },
|
||||
contentSize: { width:node.clientWidth-parseFloat(s.paddingLeft)-parseFloat(s.paddingRight), height:node.clientHeight-parseFloat(s.paddingTop)-parseFloat(s.paddingBottom) },
|
||||
styles: Object.fromEntries(styleNames.map(key => [key,s.getPropertyValue(key)])) });
|
||||
}
|
||||
return { top: window === window.top, dpr: devicePixelRatio, rows };
|
||||
}`;
|
||||
|
||||
interface DeepValue {
|
||||
type: string;
|
||||
value?: unknown;
|
||||
}
|
||||
/** This read returns JSON primitives plus DOM nodes (backend IDs), never remote object handles. */
|
||||
function decode(value: DeepValue): unknown {
|
||||
if (value.type === "node") return (value.value as { backendNodeId?: number })?.backendNodeId;
|
||||
if (value.type === "array") return (value.value as DeepValue[]).map(decode);
|
||||
if (value.type === "object")
|
||||
return Object.fromEntries(
|
||||
(value.value as [string, DeepValue][]).map(([key, item]) => [key, decode(item)]),
|
||||
);
|
||||
if (value.type === "null") return null;
|
||||
if (["string", "number", "boolean"].includes(value.type)) return value.value;
|
||||
throw new Error("unexpected live ancestry serialization");
|
||||
}
|
||||
|
||||
function stale(message: string): RpcError {
|
||||
return rpcError(
|
||||
"not_found",
|
||||
"visual_target_changed",
|
||||
`${message}; observe again before requesting a screenshot`,
|
||||
);
|
||||
}
|
||||
|
||||
function sameIdentity(a: DocumentIdentity, b: DocumentIdentity): boolean {
|
||||
return (
|
||||
a.attachmentId === b.attachmentId &&
|
||||
a.frameId === b.frameId &&
|
||||
a.target.tabId === b.target.tabId &&
|
||||
a.target.sessionId === b.target.sessionId &&
|
||||
a.documentElementBackendNodeId === b.documentElementBackendNodeId
|
||||
);
|
||||
}
|
||||
|
||||
function closeRect(a: ViewportRect, b: ViewportRect): boolean {
|
||||
return [
|
||||
a.x - b.x,
|
||||
a.y - b.y,
|
||||
a.x + a.width - b.x - b.width,
|
||||
a.y + a.height - b.y - b.height,
|
||||
].every((delta) => Number.isFinite(delta) && Math.abs(delta) <= 0.25);
|
||||
}
|
||||
|
||||
function sameClips(a?: VisualClipSource, b?: VisualClipSource): boolean {
|
||||
while (a && b) {
|
||||
if (
|
||||
!sameIdentity(a.document, b.document) ||
|
||||
a.backendNodeId !== b.backendNodeId ||
|
||||
a.x !== b.x ||
|
||||
a.y !== b.y ||
|
||||
a.overflowX !== b.overflowX ||
|
||||
a.overflowY !== b.overflowY ||
|
||||
!closeRect(a.box, b.box)
|
||||
)
|
||||
return false;
|
||||
a = a.parent;
|
||||
b = b.parent;
|
||||
}
|
||||
return !a && !b;
|
||||
}
|
||||
|
||||
/** Scale affects raster size only, never the selected CSS region. */
|
||||
export function visualScreenshotScale(width: number, height: number, pixelsPerCss: number): number {
|
||||
if (![width, height, pixelsPerCss].every((n) => Number.isFinite(n) && n > 0))
|
||||
throw new Error("invalid screenshot pixel dimensions");
|
||||
return Math.min(
|
||||
1,
|
||||
2048 / (Math.max(width, height) * pixelsPerCss),
|
||||
Math.sqrt(4_000_000 / (width * height * pixelsPerCss * pixelsPerCss)),
|
||||
);
|
||||
}
|
||||
|
||||
async function readFrame(
|
||||
cdp: CdpRunner,
|
||||
document: DocumentIdentity,
|
||||
anchor: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<LiveFrame | RpcError> {
|
||||
const verified = await resolveVerifiedNode(cdp, document, anchor, signal);
|
||||
if (verified.status !== "current") return stale(`visual DOM identity ${verified.status}`);
|
||||
try {
|
||||
throwIfAborted(signal);
|
||||
const reply = await sendToCdpTarget<{
|
||||
result?: { deepSerializedValue?: DeepValue };
|
||||
exceptionDetails?: unknown;
|
||||
}>(cdp, document.target, "Runtime.callFunctionOn", {
|
||||
objectId: verified.objectId,
|
||||
objectGroup: verified.objectGroup,
|
||||
functionDeclaration: READ_ANCESTRY,
|
||||
arguments: [{ value: VISUAL_STYLES }],
|
||||
serializationOptions: {
|
||||
serialization: "deep",
|
||||
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
|
||||
},
|
||||
});
|
||||
throwIfAborted(signal);
|
||||
if (reply.exceptionDetails || !reply.result?.deepSerializedValue)
|
||||
return stale("visual ancestry unavailable");
|
||||
const result = decode(reply.result.deepSerializedValue) as LiveFrame | null;
|
||||
if (
|
||||
!result?.rows?.length ||
|
||||
result.rows[0].node !== anchor ||
|
||||
result.rows.at(-1)?.node !== document.documentElementBackendNodeId ||
|
||||
!result.rows.every((row) => Number.isSafeInteger(row.node) && row.node > 0)
|
||||
)
|
||||
return stale("visual ancestry incomplete");
|
||||
if (cdp.getAttachmentId?.(document.target.tabId) !== document.attachmentId)
|
||||
return stale("visual attachment changed");
|
||||
return result;
|
||||
} finally {
|
||||
await sendToCdpTarget(cdp, document.target, "Runtime.releaseObjectGroup", {
|
||||
objectGroup: verified.objectGroup,
|
||||
}).catch(() => {});
|
||||
throwIfAborted(signal);
|
||||
}
|
||||
}
|
||||
|
||||
/** All live reads for one screenshot use this one measurement context. */
|
||||
async function resolveVisualRegionNow(
|
||||
cdp: CdpRunner,
|
||||
candidate: VisualCandidate,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ crop: ViewportRect; viewport: CssViewport; dpr: number } | RpcError> {
|
||||
throwIfAborted(signal);
|
||||
if (!candidate.framePath || !sameIdentity(candidate.document, candidate.framePath.document))
|
||||
return stale("visual frame path unavailable");
|
||||
const path: { frame: VisualFramePath; anchor: number }[] = [];
|
||||
const seen = new Set<string>();
|
||||
let frame: VisualFramePath | undefined = candidate.framePath;
|
||||
let anchor = candidate.backendNodeId;
|
||||
while (frame) {
|
||||
if (
|
||||
seen.has(frame.document.frameId) ||
|
||||
frame.document.target.tabId !== candidate.document.target.tabId ||
|
||||
cdp.getAttachmentId?.(frame.document.target.tabId) !== frame.document.attachmentId
|
||||
)
|
||||
return stale("invalid visual frame path");
|
||||
seen.add(frame.document.frameId);
|
||||
path.push({ frame, anchor });
|
||||
anchor = frame.parent?.ownerBackendNodeId ?? 0;
|
||||
frame = frame.parent?.frame;
|
||||
}
|
||||
path.reverse();
|
||||
const frames: CdpFrame[] = path.map(({ frame }) => ({
|
||||
frameId: frame.document.frameId,
|
||||
target: frame.document.target,
|
||||
...(frame.parent
|
||||
? {
|
||||
parentFrameId: frame.parent.frame.document.frameId,
|
||||
ownerBackendNodeId: frame.parent.ownerBackendNodeId,
|
||||
}
|
||||
: {}),
|
||||
}));
|
||||
// Validate each recorded edge instead of asking the driver to fill all page owners.
|
||||
for (const { frame } of path) {
|
||||
if (!frame.parent) continue;
|
||||
throwIfAborted(signal);
|
||||
const owner = await sendToCdpTarget<{ backendNodeId?: number }>(
|
||||
cdp,
|
||||
frame.parent.frame.document.target,
|
||||
"DOM.getFrameOwner",
|
||||
{ frameId: frame.document.frameId },
|
||||
);
|
||||
if (owner.backendNodeId !== frame.parent.ownerBackendNodeId)
|
||||
return stale("visual frame owner changed");
|
||||
}
|
||||
const geometry = new GeometryContext(
|
||||
cdp,
|
||||
candidate.document.target.tabId,
|
||||
{ rootFrameId: frames[0].frameId, frames },
|
||||
signal,
|
||||
);
|
||||
let context: VisualContext = EMPTY_VISUAL_CONTEXT;
|
||||
let parentRead: LiveFrame | undefined;
|
||||
let finalRegion: VisualCandidate["region"] | undefined;
|
||||
let topDpr = 0;
|
||||
for (let i = 0; i < path.length; i++) {
|
||||
const { frame, anchor } = path[i];
|
||||
const live = await readFrame(cdp, frame.document, anchor, signal);
|
||||
if ("code" in live) return live;
|
||||
if (live.top !== (i === 0)) return stale("visual root frame changed");
|
||||
if (i === 0) topDpr = live.dpr;
|
||||
const metrics = await geometry.layoutMetrics(frame.document.target);
|
||||
const viewport = cssViewport(metrics);
|
||||
const projections: GeometryProjection[] = [];
|
||||
if (frame.parent) {
|
||||
const parent = frame.parent.frame.document;
|
||||
const size = parentRead!.rows[0].contentSize;
|
||||
if (![size.width, size.height].every((n) => Number.isFinite(n) && n > 0))
|
||||
return stale("iframe content size unavailable");
|
||||
const parentViewport = await geometry.viewport(parent.target);
|
||||
if (!parentViewport) return stale("parent viewport unavailable");
|
||||
const local = await geometry.snapshotProjection(
|
||||
{ target: parent.target, frameId: frame.document.frameId },
|
||||
frame.parent.ownerBackendNodeId,
|
||||
[],
|
||||
parentViewport,
|
||||
size,
|
||||
);
|
||||
const outer = await geometry.targetProjection(parent.frameId);
|
||||
if (local.status !== "available" || !outer)
|
||||
return stale("visual frame projection unavailable");
|
||||
projections.push(local.projection.geometry, outer);
|
||||
} else projections.push({ sourceClips: [], edges: [], topViewport: viewport });
|
||||
const issue = visualProjectionIssue({
|
||||
projections,
|
||||
coordinates: { layoutUnitsPerCssPixel: 1, scrollCss: { x: 0, y: 0 } },
|
||||
pageScale: metrics.cssVisualViewport?.scale ?? metrics.visualViewport?.scale,
|
||||
});
|
||||
if (issue) return stale(issue);
|
||||
for (let j = live.rows.length - 1; j >= 0; j--) {
|
||||
const row = live.rows[j];
|
||||
const isAnchor = j === 0;
|
||||
const node = {
|
||||
document: frame.document,
|
||||
backendNodeId: row.node,
|
||||
styles: row.styles,
|
||||
clientBox: projectVisualBox(row.client, projections),
|
||||
};
|
||||
context = extendVisualContext(
|
||||
context,
|
||||
node,
|
||||
!isAnchor && row.tag !== "iframe" && row.tag !== "frame",
|
||||
);
|
||||
}
|
||||
if (i < path.length - 1) {
|
||||
const owner = live.rows[0];
|
||||
context = {
|
||||
...context,
|
||||
hidden:
|
||||
context.hidden ||
|
||||
owner.styles.visibility === "hidden" ||
|
||||
owner.styles.visibility === "collapse",
|
||||
transformed: false,
|
||||
localClip: false,
|
||||
unpositionedClip: false,
|
||||
};
|
||||
} else {
|
||||
const row = live.rows[0];
|
||||
if (row.tag !== "canvas") return stale("visual anchor is no longer Canvas");
|
||||
let visible: ViewportRect | null = row.box;
|
||||
for (const projection of projections)
|
||||
visible = visible
|
||||
? projectRectToViewport(
|
||||
{ x: visible.x, y: visible.y, w: visible.width, h: visible.height },
|
||||
projection,
|
||||
)
|
||||
: null;
|
||||
const region = resolveVisualRegion({
|
||||
borderBox: projectVisualBox(row.box, projections),
|
||||
frameVisibleBox: visible,
|
||||
context,
|
||||
visibility: row.styles.visibility,
|
||||
});
|
||||
if (region.status !== "available")
|
||||
return stale(
|
||||
`visual region ${region.status}${region.status === "unavailable" ? `: ${region.reason}` : ""}`,
|
||||
);
|
||||
finalRegion = region;
|
||||
}
|
||||
parentRead = live;
|
||||
}
|
||||
if (
|
||||
!finalRegion ||
|
||||
!closeRect(candidate.region.borderBox, finalRegion.borderBox) ||
|
||||
!closeRect(candidate.region.crop, finalRegion.crop) ||
|
||||
!sameClips(candidate.region.clips, finalRegion.clips)
|
||||
)
|
||||
return stale("visual region changed");
|
||||
const viewport = cssViewport(await geometry.layoutMetrics(frames[0].target));
|
||||
return { crop: finalRegion.crop, viewport, dpr: topDpr };
|
||||
}
|
||||
|
||||
/** One target only. No frame discovery, scrolling, AX, or snapshot recapture. */
|
||||
export async function captureVisualScreenshot(
|
||||
cdp: CdpRunner,
|
||||
candidate: VisualCandidate,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ image_base64: string; width: number; height: number } | RpcError> {
|
||||
try {
|
||||
const region = await resolveVisualRegionNow(cdp, candidate, signal);
|
||||
if ("code" in region) return region;
|
||||
const clip = screenshotPageRect(region.crop, region.viewport);
|
||||
if (!clip) return stale("invalid screenshot coordinates");
|
||||
let scale = visualScreenshotScale(region.crop.width, region.crop.height, region.dpr);
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
throwIfAborted(signal);
|
||||
if (
|
||||
cdp.getAttachmentId?.(candidate.document.target.tabId) !== candidate.document.attachmentId
|
||||
)
|
||||
return stale("visual attachment changed");
|
||||
const shot = await cdp.send<{ data?: string }>(
|
||||
candidate.document.target.tabId,
|
||||
"Page.captureScreenshot",
|
||||
{ format: "png", captureBeyondViewport: false, clip: { ...clip.rect, scale } },
|
||||
);
|
||||
throwIfAborted(signal);
|
||||
const dims = shot.data ? parsePngDimensions(shot.data) : null;
|
||||
if (!dims)
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"screenshot_capture_failed",
|
||||
"visual screenshot returned invalid PNG dimensions",
|
||||
);
|
||||
const correction = visualScreenshotScale(dims.width, dims.height, 1);
|
||||
if (correction === 1) return { image_base64: shot.data!, ...dims };
|
||||
scale *= correction * 0.99;
|
||||
}
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"visual_pixel_budget_exceeded",
|
||||
"visual screenshot exceeds the pixel budget",
|
||||
);
|
||||
} catch (error) {
|
||||
if (isAbortError(error) || signal?.aborted)
|
||||
return { code: "cancelled", message: "visual screenshot aborted" };
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"screenshot_capture_failed",
|
||||
error instanceof Error ? error.message : String(error),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -141,6 +141,32 @@ function facts(
|
||||
}
|
||||
|
||||
describe("Canvas discovery", () => {
|
||||
it("shares frame address paths from existing Facts without changing candidate evidence", async () => {
|
||||
const parent = await document([{ id: 2, parent: 1, tag: "iframe" }]);
|
||||
const child = await document(
|
||||
[
|
||||
{ id: 3, parent: 1 },
|
||||
{ id: 4, parent: 1 },
|
||||
],
|
||||
{
|
||||
frame: {
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 2,
|
||||
target: { tabId: 1 },
|
||||
},
|
||||
},
|
||||
);
|
||||
const result = await discoverVisualCandidates(facts([child, parent]));
|
||||
expect(result.candidates).toHaveLength(2);
|
||||
const path = result.candidates[0].framePath;
|
||||
expect(path).toBe(result.candidates[1].framePath);
|
||||
expect(path?.document).toBe(child.identity);
|
||||
expect(path?.parent?.ownerBackendNodeId).toBe(2);
|
||||
expect(path?.parent?.frame.document).toBe(parent.identity);
|
||||
expect(path?.parent?.frame.parent).toBeUndefined();
|
||||
});
|
||||
|
||||
it("distinguishes visual facts not collected from complete empty discovery and honors cancellation", async () => {
|
||||
const empty = facts([]);
|
||||
expect(await discoverVisualCandidates(empty)).toMatchObject({
|
||||
|
||||
@@ -11,7 +11,7 @@ export async function verifyDocumentIdentity(
|
||||
identity: DocumentIdentity,
|
||||
signal?: AbortSignal,
|
||||
): Promise<"current" | "changed" | "unavailable"> {
|
||||
return verifyIdentity(cdp, identity, signal);
|
||||
return (await verifyIdentity(cdp, identity, signal)).status;
|
||||
}
|
||||
|
||||
/** Verify the visual anchor in its original frame; geometry is checked by screenshot execution. */
|
||||
@@ -20,7 +20,24 @@ export async function verifyVisualTargetIdentity(
|
||||
candidate: VisualCandidate,
|
||||
signal?: AbortSignal,
|
||||
): Promise<"current" | "changed" | "unavailable"> {
|
||||
return verifyIdentity(cdp, candidate.document, signal, candidate.backendNodeId);
|
||||
return (await verifyIdentity(cdp, candidate.document, signal, candidate.backendNodeId)).status;
|
||||
}
|
||||
|
||||
export type VerifiedNodeResult =
|
||||
| { status: "changed" | "unavailable" }
|
||||
| { status: "current"; objectId: string; objectGroup: string };
|
||||
|
||||
/** Caller must release objectGroup in finally after its local read. */
|
||||
export async function resolveVerifiedNode(
|
||||
cdp: CdpRunner,
|
||||
identity: DocumentIdentity,
|
||||
backendNodeId: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<VerifiedNodeResult> {
|
||||
const result = await verifyIdentity(cdp, identity, signal, backendNodeId, true);
|
||||
return result.status === "current" && result.objectId && result.objectGroup
|
||||
? { status: "current", objectId: result.objectId, objectGroup: result.objectGroup }
|
||||
: { status: result.status === "changed" ? "changed" : "unavailable" };
|
||||
}
|
||||
|
||||
async function verifyIdentity(
|
||||
@@ -28,15 +45,22 @@ async function verifyIdentity(
|
||||
identity: DocumentIdentity,
|
||||
signal?: AbortSignal,
|
||||
anchorBackendNodeId?: number,
|
||||
): Promise<"current" | "changed" | "unavailable"> {
|
||||
retain = false,
|
||||
): Promise<{
|
||||
status: "current" | "changed" | "unavailable";
|
||||
objectId?: string;
|
||||
objectGroup?: string;
|
||||
}> {
|
||||
const attached = () => cdp.getAttachmentId?.(identity.target.tabId) === identity.attachmentId;
|
||||
throwIfAborted(signal);
|
||||
if (!attached()) return "changed";
|
||||
if (!attached()) return { status: "changed" };
|
||||
const objectGroup = `bsk-document-identity-${crypto.randomUUID()}`;
|
||||
const send = <T>(method: string, params: object) => {
|
||||
throwIfAborted(signal);
|
||||
return sendToCdpTarget<T>(cdp, identity.target, method, params);
|
||||
};
|
||||
let objectId: string | undefined;
|
||||
let retained = false;
|
||||
try {
|
||||
const world = await send<{ executionContextId: number }>("Page.createIsolatedWorld", {
|
||||
frameId: identity.frameId,
|
||||
@@ -63,7 +87,8 @@ async function verifyIdentity(
|
||||
executionContextId: world.executionContextId,
|
||||
objectGroup,
|
||||
});
|
||||
if (!anchor.object?.objectId) return "unavailable";
|
||||
if (!anchor.object?.objectId) return { status: "unavailable" };
|
||||
objectId = anchor.object.objectId;
|
||||
reply = await send<RootReply>("Runtime.callFunctionOn", {
|
||||
objectId: anchor.object.objectId,
|
||||
functionDeclaration:
|
||||
@@ -72,20 +97,24 @@ async function verifyIdentity(
|
||||
serializationOptions,
|
||||
});
|
||||
}
|
||||
if (!attached()) return "changed";
|
||||
if (!attached()) return { status: "changed" };
|
||||
const root = reply.result?.deepSerializedValue;
|
||||
if (root?.type === "null") return "changed";
|
||||
if (root?.type === "null") return { status: "changed" };
|
||||
const id = root?.type === "node" ? root.value?.backendNodeId : undefined;
|
||||
if (id === undefined) return "unavailable";
|
||||
return id === identity.documentElementBackendNodeId ? "current" : "changed";
|
||||
if (id === undefined) return { status: "unavailable" };
|
||||
if (id !== identity.documentElementBackendNodeId) return { status: "changed" };
|
||||
throwIfAborted(signal);
|
||||
retained = retain && !!objectId;
|
||||
return { status: "current", ...(retained ? { objectId, objectGroup } : {}) };
|
||||
} catch (error) {
|
||||
throwIfAborted(signal);
|
||||
if (isAbortError(error)) throw error;
|
||||
return attached() ? "unavailable" : "changed";
|
||||
return { status: attached() ? "unavailable" : "changed" };
|
||||
} finally {
|
||||
await sendToCdpTarget(cdp, identity.target, "Runtime.releaseObjectGroup", {
|
||||
objectGroup,
|
||||
}).catch(() => {});
|
||||
if (!retained)
|
||||
await sendToCdpTarget(cdp, identity.target, "Runtime.releaseObjectGroup", {
|
||||
objectGroup,
|
||||
}).catch(() => {});
|
||||
throwIfAborted(signal);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,11 +20,19 @@ import {
|
||||
visualProjectionIssue,
|
||||
} from "./visual-region";
|
||||
|
||||
/** Shared observation-time addresses, not cached live geometry. */
|
||||
export interface VisualFramePath {
|
||||
readonly document: DocumentIdentity;
|
||||
readonly parent?: { readonly ownerBackendNodeId: number; readonly frame: VisualFramePath };
|
||||
}
|
||||
|
||||
export interface VisualCandidate {
|
||||
readonly document: DocumentIdentity;
|
||||
readonly backendNodeId: number;
|
||||
readonly parentBackendNodeId: number | null;
|
||||
readonly label?: string;
|
||||
/** Absent when execution ancestry could not be established; discovery is still retained. */
|
||||
readonly framePath?: VisualFramePath;
|
||||
readonly region: Extract<VisualRegionResult, { status: "available" }>;
|
||||
}
|
||||
|
||||
@@ -113,6 +121,7 @@ export async function discoverVisualCandidates(
|
||||
const candidates: VisualCandidate[] = [];
|
||||
const issues: VisualDiscoveryIssue[] = [];
|
||||
const documents = new Map(facts.documents.map((document) => [document.frame.frameId, document]));
|
||||
const framePaths = new Map<string, VisualFramePath>();
|
||||
const contexts = new Map<string, Map<number, NodeContext>>();
|
||||
const roots = new Map<string, VisualContext>();
|
||||
|
||||
@@ -248,6 +257,16 @@ export async function discoverVisualCandidates(
|
||||
root.issue ??
|
||||
(!doc.identity ? "identity-unverified" : visualProjectionIssue(doc.geometry)),
|
||||
};
|
||||
if (doc.identity) {
|
||||
const parentPath = frame.parentFrameId ? framePaths.get(frame.parentFrameId) : undefined;
|
||||
if (!frame.parentFrameId && frame.frameId === facts.rootFrameId)
|
||||
framePaths.set(frame.frameId, { document: doc.identity });
|
||||
else if (parentPath && frame.ownerBackendNodeId !== undefined)
|
||||
framePaths.set(frame.frameId, {
|
||||
document: doc.identity,
|
||||
parent: { ownerBackendNodeId: frame.ownerBackendNodeId, frame: parentPath },
|
||||
});
|
||||
}
|
||||
roots.set(frame.frameId, root);
|
||||
contexts.set(frame.frameId, new Map());
|
||||
}
|
||||
@@ -291,6 +310,7 @@ export async function discoverVisualCandidates(
|
||||
const label = node.attrs["aria-label"]?.trim() || node.attrs.title?.trim();
|
||||
candidates.push({
|
||||
document: document.identity,
|
||||
framePath: framePaths.get(document.frame.frameId),
|
||||
backendNodeId: node.backendNodeId,
|
||||
parentBackendNodeId: node.parentBackendNodeId,
|
||||
...(label ? { label } : {}),
|
||||
|
||||
@@ -25,6 +25,8 @@ export type RpcErrorReason =
|
||||
| "element_not_visible"
|
||||
| "ref_not_found"
|
||||
| "ref_kind_unsupported"
|
||||
| "visual_target_changed"
|
||||
| "visual_pixel_budget_exceeded"
|
||||
| "selector_not_found"
|
||||
| "target_not_fillable"
|
||||
| "fill_value_invalid"
|
||||
|
||||
@@ -40,6 +40,8 @@ use bsk_protocol::ErrorCode;
|
||||
pub mod reason {
|
||||
pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope";
|
||||
pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible";
|
||||
pub const VISUAL_TARGET_CHANGED: &str = "visual_target_changed";
|
||||
pub const VISUAL_PIXEL_BUDGET_EXCEEDED: &str = "visual_pixel_budget_exceeded";
|
||||
pub const REF_KIND_UNSUPPORTED: &str = "ref_kind_unsupported";
|
||||
pub const REF_NOT_FOUND: &str = "ref_not_found";
|
||||
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
|
||||
@@ -248,6 +250,18 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::NotFound, reason::VISUAL_TARGET_CHANGED) => RenderInfo {
|
||||
summary: "visual target needs a fresh observation",
|
||||
hint: Some(
|
||||
"observe the page again and use the current visual ref; the previous identity or region cannot be used",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::CdpFailed, reason::VISUAL_PIXEL_BUDGET_EXCEEDED) => RenderInfo {
|
||||
summary: "visual screenshot exceeds the image size limit",
|
||||
hint: Some("the screenshot could not be reduced within the pixel budget"),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::Unsupported, reason::REF_KIND_UNSUPPORTED) => RenderInfo {
|
||||
summary: "this tool does not support the ref target type",
|
||||
hint: Some(
|
||||
@@ -672,6 +686,16 @@ mod tests {
|
||||
assert!(info.summary.contains("timed out after browser dispatch"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn visual_screenshot_failures_have_specific_guidance() {
|
||||
let changed = serde_json::json!({ "reason": reason::VISUAL_TARGET_CHANGED });
|
||||
let info = info_for_error(ErrorCode::NotFound, Some(&changed));
|
||||
assert!(info.hint.unwrap().contains("observe"));
|
||||
let budget = serde_json::json!({ "reason": reason::VISUAL_PIXEL_BUDGET_EXCEEDED });
|
||||
let info = info_for_error(ErrorCode::CdpFailed, Some(&budget));
|
||||
assert!(info.summary.contains("image size limit"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn visual_ref_type_error_has_specific_guidance() {
|
||||
let data = serde_json::json!({ "reason": reason::REF_KIND_UNSUPPORTED });
|
||||
|
||||
Reference in New Issue
Block a user