mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat: support on-demand screenshots for canvas visual refs
This commit is contained in:
1 parent
a6de73aea3
commit
31ddcb139c
11 files changed
+922
-61
No files matched your search
@@ -4,7 +4,7 @@ import { RefStore, type VisualRefInput } from "@/session-manager/ref-store";
|
|||||||
import { handleDownload } from "../download";
|
import { handleDownload } from "../download";
|
||||||
import { handleRequestHelp, resetHelpLifecycleForTests } from "../human-loop";
|
import { handleRequestHelp, resetHelpLifecycleForTests } from "../human-loop";
|
||||||
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "../interaction";
|
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "../interaction";
|
||||||
import { handleGetHtml, handleScreenshot } from "../observation";
|
import { handleGetHtml } from "../observation";
|
||||||
import type { CdpRunner } from "../shared";
|
import type { CdpRunner } from "../shared";
|
||||||
import { lookupRefTarget, lookupSnapshotRef, resolveSnapshotRef } from "../snapshot-ref";
|
import { lookupRefTarget, lookupSnapshotRef, resolveSnapshotRef } from "../snapshot-ref";
|
||||||
import { handleUpload } from "../upload";
|
import { handleUpload } from "../upload";
|
||||||
@@ -117,12 +117,10 @@ describe("typed visual refs", () => {
|
|||||||
"press",
|
"press",
|
||||||
"select",
|
"select",
|
||||||
"get_html",
|
"get_html",
|
||||||
"screenshot",
|
|
||||||
])("rejects visual refs in %s before target effects", async (tool) => {
|
])("rejects visual refs in %s before target effects", async (tool) => {
|
||||||
const { manager, send, cdp, tabsApi } = await setup();
|
const { manager, send, cdp, tabsApi } = await setup();
|
||||||
const params = { session_id: "test", tab_id: 4, ref: "@e1" };
|
const params = { session_id: "test", tab_id: 4, ref: "@e1" };
|
||||||
const deps = { cdp, tabsApi };
|
const deps = { cdp, tabsApi };
|
||||||
const captureVisibleTab = vi.fn();
|
|
||||||
let result: unknown;
|
let result: unknown;
|
||||||
switch (tool) {
|
switch (tool) {
|
||||||
case "click":
|
case "click":
|
||||||
@@ -143,16 +141,9 @@ describe("typed visual refs", () => {
|
|||||||
case "get_html":
|
case "get_html":
|
||||||
result = await handleGetHtml(manager, params, deps);
|
result = await handleGetHtml(manager, params, deps);
|
||||||
break;
|
break;
|
||||||
case "screenshot":
|
|
||||||
result = await handleScreenshot(manager, params, {
|
|
||||||
...deps,
|
|
||||||
captureApi: { ...tabsApi, captureVisibleTab },
|
|
||||||
});
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
expect(result).toMatchObject({ code: "unsupported", data: { reason: "ref_kind_unsupported" } });
|
expect(result).toMatchObject({ code: "unsupported", data: { reason: "ref_kind_unsupported" } });
|
||||||
expect(send).not.toHaveBeenCalled();
|
expect(send).not.toHaveBeenCalled();
|
||||||
expect(captureVisibleTab).not.toHaveBeenCalled();
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it.each([
|
it.each([
|
||||||
|
|||||||
@@ -0,0 +1,373 @@
|
|||||||
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||||
|
import { SessionManager } from "@/session-manager/manager";
|
||||||
|
import { handleScreenshot } from "../observation";
|
||||||
|
import type { CdpRunner } from "../shared";
|
||||||
|
import { captureVisualScreenshot, visualScreenshotScale } from "../visual-screenshot";
|
||||||
|
import type { VisualCandidate, VisualFramePath } from "../vom/visual-discovery";
|
||||||
|
|
||||||
|
const styles = {
|
||||||
|
position: "static",
|
||||||
|
visibility: "visible",
|
||||||
|
opacity: "1",
|
||||||
|
display: "block",
|
||||||
|
"overflow-x": "visible",
|
||||||
|
"overflow-y": "visible",
|
||||||
|
transform: "none",
|
||||||
|
zoom: "1",
|
||||||
|
"clip-path": "none",
|
||||||
|
"mask-image": "none",
|
||||||
|
rotate: "none",
|
||||||
|
scale: "none",
|
||||||
|
perspective: "none",
|
||||||
|
clip: "auto",
|
||||||
|
contain: "none",
|
||||||
|
"overflow-clip-margin": "0px",
|
||||||
|
};
|
||||||
|
const rect = (x = 10, y = 20, width = 100, height = 40) => ({ x, y, width, height });
|
||||||
|
const root = {
|
||||||
|
attachmentId: "a",
|
||||||
|
target: { tabId: 4 },
|
||||||
|
frameId: "top",
|
||||||
|
documentElementBackendNodeId: 1,
|
||||||
|
};
|
||||||
|
function row(id: number, tag: string, box = rect()) {
|
||||||
|
return {
|
||||||
|
node: { backend: id },
|
||||||
|
tag,
|
||||||
|
box,
|
||||||
|
client: box,
|
||||||
|
contentSize: { width: box.width, height: box.height },
|
||||||
|
styles: { ...styles },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
function encode(value: unknown): unknown {
|
||||||
|
if (value === null) return { type: "null" };
|
||||||
|
if (Array.isArray(value)) return { type: "array", value: value.map(encode) };
|
||||||
|
if (typeof value === "object") {
|
||||||
|
const record = value as Record<string, unknown>;
|
||||||
|
if ("backend" in record) return { type: "node", value: { backendNodeId: record.backend } };
|
||||||
|
return { type: "object", value: Object.entries(record).map(([k, v]) => [k, encode(v)]) };
|
||||||
|
}
|
||||||
|
return { type: typeof value, value };
|
||||||
|
}
|
||||||
|
function png(width: number, height: number) {
|
||||||
|
const bytes = new Uint8Array(33);
|
||||||
|
bytes.set([137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82]);
|
||||||
|
const view = new DataView(bytes.buffer);
|
||||||
|
view.setUint32(16, width);
|
||||||
|
view.setUint32(20, height);
|
||||||
|
return btoa(String.fromCharCode(...bytes));
|
||||||
|
}
|
||||||
|
function fixture(child = false, oopif = false) {
|
||||||
|
const parent: VisualFramePath = { document: root };
|
||||||
|
const frame: VisualFramePath = child
|
||||||
|
? {
|
||||||
|
document: {
|
||||||
|
...root,
|
||||||
|
frameId: "child",
|
||||||
|
documentElementBackendNodeId: 11,
|
||||||
|
target: oopif ? { tabId: 4, sessionId: "oopif" } : { tabId: 4 },
|
||||||
|
},
|
||||||
|
parent: { frame: parent, ownerBackendNodeId: 2 },
|
||||||
|
}
|
||||||
|
: parent;
|
||||||
|
const crop = child ? rect(120, 240, 200, 80) : rect();
|
||||||
|
const candidate: VisualCandidate = {
|
||||||
|
document: frame.document,
|
||||||
|
backendNodeId: child ? 12 : 3,
|
||||||
|
parentBackendNodeId: frame.document.documentElementBackendNodeId,
|
||||||
|
framePath: frame,
|
||||||
|
region: { status: "available", borderBox: crop, crop },
|
||||||
|
};
|
||||||
|
const topRows = child
|
||||||
|
? [row(2, "iframe", rect(100, 200, 400, 200)), row(1, "html", rect(0, 0, 1200, 800))]
|
||||||
|
: [row(3, "canvas"), row(1, "html", rect(0, 0, 1200, 800))];
|
||||||
|
if (child) topRows[0].contentSize = { width: 200, height: 100 };
|
||||||
|
const childRows = [row(12, "canvas"), row(11, "html", rect(0, 0, 200, 100))];
|
||||||
|
let calls = 0;
|
||||||
|
const control = {
|
||||||
|
wrongOwner: false,
|
||||||
|
rootChanged: false,
|
||||||
|
failRead: false,
|
||||||
|
abortRead: false,
|
||||||
|
dpr: 1,
|
||||||
|
shots: [png(crop.width, crop.height)],
|
||||||
|
};
|
||||||
|
const controller = new AbortController();
|
||||||
|
const send = vi.fn(
|
||||||
|
async (target: CdpTarget, method: string, params: Record<string, unknown> = {}) => {
|
||||||
|
if (method === "DOM.getFrameOwner") return { backendNodeId: control.wrongOwner ? 999 : 2 };
|
||||||
|
if (method === "Page.createIsolatedWorld")
|
||||||
|
return { executionContextId: params.frameId === "child" ? 11 : 1 };
|
||||||
|
if (method === "DOM.resolveNode")
|
||||||
|
return { object: { objectId: String(params.backendNodeId) } };
|
||||||
|
if (method === "Runtime.callFunctionOn") {
|
||||||
|
const isChild = params.objectId === "12";
|
||||||
|
if (!(params.functionDeclaration as string).includes("styleNames"))
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
deepSerializedValue: {
|
||||||
|
type: "node",
|
||||||
|
value: { backendNodeId: control.rootChanged ? 999 : isChild ? 11 : 1 },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
if (control.failRead) throw new Error("read failed");
|
||||||
|
if (control.abortRead) controller.abort();
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
deepSerializedValue: encode({
|
||||||
|
top: !isChild,
|
||||||
|
dpr: control.dpr,
|
||||||
|
rows: isChild ? childRows : topRows,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (method === "Runtime.releaseObjectGroup") return {};
|
||||||
|
if (method === "Page.getLayoutMetrics")
|
||||||
|
return {
|
||||||
|
cssLayoutViewport: { clientWidth: 1200, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||||
|
cssVisualViewport: { scale: 1, zoom: 1 },
|
||||||
|
};
|
||||||
|
if (method === "DOM.getBoxModel")
|
||||||
|
return { model: { content: [100, 200, 500, 200, 500, 400, 100, 400] } };
|
||||||
|
if (method === "Page.captureScreenshot")
|
||||||
|
return { data: control.shots[Math.min(calls++, control.shots.length - 1)] };
|
||||||
|
throw new Error(`unexpected ${method} ${target.sessionId}`);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const cdp = {
|
||||||
|
send: ((tabId, method, params) =>
|
||||||
|
send({ tabId }, method, params as Record<string, unknown>)) as CdpRunner["send"],
|
||||||
|
sendToTarget: send as unknown as CdpRunner["sendToTarget"],
|
||||||
|
getAttachmentId: () => "a",
|
||||||
|
getFrameGraph: vi.fn(async () => {
|
||||||
|
throw new Error("whole page discovery forbidden");
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
return { candidate, cdp, send, control, controller, topRows, childRows };
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("visual screenshot", () => {
|
||||||
|
it.each([
|
||||||
|
[false, false],
|
||||||
|
[true, false],
|
||||||
|
[true, true],
|
||||||
|
])("screenshots the local target child=%s oopif=%s without whole-page discovery", async (child, oopif) => {
|
||||||
|
const f = fixture(child, oopif);
|
||||||
|
const result = await captureVisualScreenshot(f.cdp, f.candidate);
|
||||||
|
expect(result).toMatchObject({ width: child ? 200 : 100, height: child ? 80 : 40 });
|
||||||
|
expect(f.cdp.getFrameGraph).not.toHaveBeenCalled();
|
||||||
|
const names = f.send.mock.calls.map((c) => c[1]);
|
||||||
|
expect(names.filter((n) => n === "DOM.getFrameOwner")).toHaveLength(child ? 1 : 0);
|
||||||
|
expect(names.filter((n) => n === "DOM.resolveNode")).toHaveLength(child ? 2 : 1);
|
||||||
|
expect(names.some((n) => /scroll|DOMSnapshot|getFrameTree|Accessibility/.test(n))).toBe(false);
|
||||||
|
expect(f.send.mock.calls.find((c) => c[1] === "Page.captureScreenshot")?.[2]).toMatchObject({
|
||||||
|
clip: { ...f.candidate.region.crop, scale: 1 },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
it.each([
|
||||||
|
false,
|
||||||
|
true,
|
||||||
|
])("projects a nested child through a parent target oopif=%s", async (oopif) => {
|
||||||
|
const f = fixture(true, oopif);
|
||||||
|
const parent = f.candidate.framePath!;
|
||||||
|
const document = {
|
||||||
|
...parent.document,
|
||||||
|
frameId: "grandchild",
|
||||||
|
documentElementBackendNodeId: 21,
|
||||||
|
};
|
||||||
|
const crop = rect(130, 250, 40, 20);
|
||||||
|
const candidate: VisualCandidate = {
|
||||||
|
...f.candidate,
|
||||||
|
document,
|
||||||
|
backendNodeId: 22,
|
||||||
|
framePath: { document, parent: { frame: parent, ownerBackendNodeId: 12 } },
|
||||||
|
region: { status: "available", borderBox: crop, crop },
|
||||||
|
};
|
||||||
|
f.childRows[0].tag = "iframe";
|
||||||
|
f.control.shots = [png(40, 20)];
|
||||||
|
const original = f.send.getMockImplementation()!;
|
||||||
|
f.send.mockImplementation(async (target, method, params = {}) => {
|
||||||
|
if (method === "DOM.getFrameOwner" && params.frameId === "grandchild")
|
||||||
|
return { backendNodeId: 12 };
|
||||||
|
if (method === "Runtime.callFunctionOn" && params.objectId === "22") {
|
||||||
|
if (!(params.functionDeclaration as string).includes("styleNames"))
|
||||||
|
return {
|
||||||
|
result: { deepSerializedValue: { type: "node", value: { backendNodeId: 21 } } },
|
||||||
|
};
|
||||||
|
return {
|
||||||
|
result: {
|
||||||
|
deepSerializedValue: encode({
|
||||||
|
top: false,
|
||||||
|
dpr: 1,
|
||||||
|
rows: [row(22, "canvas", rect(5, 5, 20, 10)), row(21, "html", rect(0, 0, 100, 40))],
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (method === "DOM.getBoxModel" && params.backendNodeId === 12)
|
||||||
|
return {
|
||||||
|
model: {
|
||||||
|
content: oopif
|
||||||
|
? [10, 20, 110, 20, 110, 60, 10, 60]
|
||||||
|
: [120, 240, 320, 240, 320, 320, 120, 320],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
if (method === "Page.getLayoutMetrics" && target.sessionId)
|
||||||
|
return {
|
||||||
|
cssLayoutViewport: { clientWidth: 200, clientHeight: 100, pageX: 0, pageY: 0 },
|
||||||
|
cssVisualViewport: { scale: 1, zoom: 1 },
|
||||||
|
};
|
||||||
|
return original(target, method, params);
|
||||||
|
});
|
||||||
|
expect(await captureVisualScreenshot(f.cdp, candidate)).toMatchObject({
|
||||||
|
width: 40,
|
||||||
|
height: 20,
|
||||||
|
});
|
||||||
|
expect(f.send.mock.calls.filter((c) => c[1] === "DOM.getFrameOwner")).toHaveLength(2);
|
||||||
|
expect(f.send.mock.calls.find((c) => c[1] === "Page.captureScreenshot")?.[2]).toMatchObject({
|
||||||
|
clip: crop,
|
||||||
|
});
|
||||||
|
expect(f.cdp.getFrameGraph).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("runs the live reader on only connected local ancestry", async () => {
|
||||||
|
const f = fixture();
|
||||||
|
await captureVisualScreenshot(f.cdp, f.candidate);
|
||||||
|
const params = f.send.mock.calls.find(
|
||||||
|
(c) =>
|
||||||
|
c[1] === "Runtime.callFunctionOn" &&
|
||||||
|
String(c[2]?.functionDeclaration).includes("styleNames"),
|
||||||
|
)![2]!;
|
||||||
|
const read = new Function(`return (${params.functionDeclaration});`)() as (
|
||||||
|
this: Element,
|
||||||
|
names: string[],
|
||||||
|
) => { rows: { node: Element }[] } | null;
|
||||||
|
const canvas = document.createElement("canvas");
|
||||||
|
document.body.append(canvas);
|
||||||
|
try {
|
||||||
|
const result = read.call(canvas, ["display"]);
|
||||||
|
expect(result?.rows.map((row) => row.node)).toEqual([
|
||||||
|
canvas,
|
||||||
|
document.body,
|
||||||
|
document.documentElement,
|
||||||
|
]);
|
||||||
|
canvas.remove();
|
||||||
|
expect(read.call(canvas, ["display"])).toBeNull();
|
||||||
|
const other = document.implementation.createHTMLDocument();
|
||||||
|
const foreign = other.createElement("canvas");
|
||||||
|
other.body.append(foreign);
|
||||||
|
expect(foreign.ownerDocument).toBe(other);
|
||||||
|
expect(read.call(foreign, ["display"])).toBeNull();
|
||||||
|
} finally {
|
||||||
|
canvas.remove();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps requests constant as ordinary ancestry grows", async () => {
|
||||||
|
const small = fixture(),
|
||||||
|
large = fixture();
|
||||||
|
large.topRows.splice(
|
||||||
|
1,
|
||||||
|
0,
|
||||||
|
...Array.from({ length: 100 }, (_, i) => row(100 + i, "div", rect(0, 0, 1200, 800))),
|
||||||
|
);
|
||||||
|
await captureVisualScreenshot(small.cdp, small.candidate);
|
||||||
|
await captureVisualScreenshot(large.cdp, large.candidate);
|
||||||
|
expect(large.send.mock.calls.map((c) => c[1])).toEqual(small.send.mock.calls.map((c) => c[1]));
|
||||||
|
});
|
||||||
|
it("preserves missing-path candidates but cannot execute them", async () => {
|
||||||
|
const f = fixture();
|
||||||
|
const { framePath: _, ...candidate } = f.candidate;
|
||||||
|
expect(await captureVisualScreenshot(f.cdp, candidate)).toMatchObject({
|
||||||
|
data: { reason: "visual_target_changed" },
|
||||||
|
});
|
||||||
|
expect(f.send).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
it.each([0.249, 0.25, 0.251])("compares rectangle edges with tolerance %s", async (delta) => {
|
||||||
|
const f = fixture();
|
||||||
|
f.topRows[0].box.x += delta;
|
||||||
|
const result = await captureVisualScreenshot(f.cdp, f.candidate);
|
||||||
|
expect("code" in result).toBe(delta > 0.25);
|
||||||
|
});
|
||||||
|
it("rejects a new clipping ancestor even if the resulting crop is identical", async () => {
|
||||||
|
const f = fixture();
|
||||||
|
const clip = row(8, "div", rect(0, 0, 1200, 800));
|
||||||
|
clip.styles["overflow-x"] = "hidden";
|
||||||
|
f.topRows.splice(1, 0, clip);
|
||||||
|
expect(await captureVisualScreenshot(f.cdp, f.candidate)).toMatchObject({
|
||||||
|
data: { reason: "visual_target_changed" },
|
||||||
|
});
|
||||||
|
expect(f.send.mock.calls.some((c) => c[1] === "Page.captureScreenshot")).toBe(false);
|
||||||
|
});
|
||||||
|
it("rejects changed owner and changed DOM before capture", async () => {
|
||||||
|
for (const kind of ["wrongOwner", "rootChanged"] as const) {
|
||||||
|
const f = fixture(true);
|
||||||
|
f.control[kind] = true;
|
||||||
|
expect(await captureVisualScreenshot(f.cdp, f.candidate)).toMatchObject({
|
||||||
|
data: { reason: "visual_target_changed" },
|
||||||
|
});
|
||||||
|
expect(f.send.mock.calls.some((c) => c[1] === "Page.captureScreenshot")).toBe(false);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
it("releases retained objects on read failure and cancellation", async () => {
|
||||||
|
for (const kind of ["failRead", "abortRead"] as const) {
|
||||||
|
const f = fixture();
|
||||||
|
f.control[kind] = true;
|
||||||
|
const result = await captureVisualScreenshot(f.cdp, f.candidate, f.controller.signal);
|
||||||
|
expect(result).toMatchObject({ code: kind === "abortRead" ? "cancelled" : "cdp_failed" });
|
||||||
|
expect(f.send.mock.calls.at(-1)?.[1]).toBe("Runtime.releaseObjectGroup");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
it("plans pixels without enlarging images and retries at most once", async () => {
|
||||||
|
expect(visualScreenshotScale(4096, 2048, 2)).toBe(0.25);
|
||||||
|
expect(visualScreenshotScale(4000, 4000, 1)).toBe(0.5);
|
||||||
|
const f = fixture();
|
||||||
|
f.control.shots = [png(3000, 3000), png(1800, 1800)];
|
||||||
|
expect(await captureVisualScreenshot(f.cdp, f.candidate)).toMatchObject({
|
||||||
|
width: 1800,
|
||||||
|
height: 1800,
|
||||||
|
});
|
||||||
|
const shots = f.send.mock.calls.filter((c) => c[1] === "Page.captureScreenshot");
|
||||||
|
expect(shots).toHaveLength(2);
|
||||||
|
expect((shots[1][2]!.clip as { scale: number }).scale).toBeLessThan(1);
|
||||||
|
const fail = fixture();
|
||||||
|
fail.control.shots = [png(3000, 3000)];
|
||||||
|
expect(await captureVisualScreenshot(fail.cdp, fail.candidate)).toMatchObject({
|
||||||
|
data: { reason: "visual_pixel_budget_exceeded" },
|
||||||
|
});
|
||||||
|
expect(fail.send.mock.calls.filter((c) => c[1] === "Page.captureScreenshot")).toHaveLength(2);
|
||||||
|
});
|
||||||
|
it("rejects invalid PNG without guessing dimensions", async () => {
|
||||||
|
const f = fixture();
|
||||||
|
f.control.shots = ["invalid"];
|
||||||
|
expect(await captureVisualScreenshot(f.cdp, f.candidate)).toMatchObject({
|
||||||
|
data: { reason: "screenshot_capture_failed" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
it("dispatches a visual ref through the screenshot tool and restores overlays", async () => {
|
||||||
|
const f = fixture();
|
||||||
|
const manager = new SessionManager({
|
||||||
|
agentWindow: {
|
||||||
|
create: async () => 100,
|
||||||
|
remove: async () => {},
|
||||||
|
ensureActiveTab: async () => 4,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const ctx = await manager.start("test");
|
||||||
|
ctx.refStore.replace([["e1", { kind: "visual-region", candidate: f.candidate }]]);
|
||||||
|
const tab = { id: 4, windowId: 100, active: true } as chrome.tabs.Tab;
|
||||||
|
const tabsApi = { get: async () => tab, query: async () => [tab] };
|
||||||
|
const sendToTab = vi.fn(async () => ({}));
|
||||||
|
const result = await handleScreenshot(
|
||||||
|
manager,
|
||||||
|
{ session_id: "test", ref: "e1" },
|
||||||
|
{ cdp: f.cdp, tabsApi, captureApi: { ...tabsApi, captureVisibleTab: vi.fn() }, sendToTab },
|
||||||
|
);
|
||||||
|
expect(result).toMatchObject({ width: 100, height: 40, format: "png" });
|
||||||
|
expect(sendToTab).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -212,11 +212,16 @@ export class GeometryContext {
|
|||||||
ownerBackendNodeId: number,
|
ownerBackendNodeId: number,
|
||||||
ancestorClips: Polygon[],
|
ancestorClips: Polygon[],
|
||||||
viewport: Size,
|
viewport: Size,
|
||||||
|
contentSize?: Size,
|
||||||
): Promise<SnapshotProjectionResult> {
|
): Promise<SnapshotProjectionResult> {
|
||||||
const key = `${cdpTargetKey(source.target)}:${ownerBackendNodeId}`;
|
const key = `${cdpTargetKey(source.target)}:${ownerBackendNodeId}`;
|
||||||
let promise = this.snapshotOwners.get(key);
|
let promise = this.snapshotOwners.get(key);
|
||||||
if (!promise) {
|
if (!promise) {
|
||||||
promise = this.snapshotOwner(source.target, ownerBackendNodeId);
|
promise = contentSize
|
||||||
|
? this.ownerContent(source.target, ownerBackendNodeId).then((quad) =>
|
||||||
|
quad ? { quad, size: contentSize } : null,
|
||||||
|
)
|
||||||
|
: this.snapshotOwner(source.target, ownerBackendNodeId);
|
||||||
this.snapshotOwners.set(key, promise);
|
this.snapshotOwners.set(key, promise);
|
||||||
}
|
}
|
||||||
let owner: { quad: Quad; size: Size } | null;
|
let owner: { quad: Quad; size: Size } | null;
|
||||||
|
|||||||
@@ -1,3 +1,8 @@
|
|||||||
|
import { parsePngDimensions } from "./png";
|
||||||
|
import { captureVisualScreenshot } from "./visual-screenshot";
|
||||||
|
|
||||||
|
export { parsePngDimensions } from "./png";
|
||||||
|
|
||||||
import type { CapturedSceneInput } from "./vom/facts";
|
import type { CapturedSceneInput } from "./vom/facts";
|
||||||
// Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`,
|
// Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`,
|
||||||
// and semantic `tool.observe` (design §7). Each handler resolves the target
|
// and semantic `tool.observe` (design §7). Each handler resolves the target
|
||||||
@@ -101,38 +106,6 @@ export function stripDataUrlPrefix(dataUrl: string): string {
|
|||||||
return m ? dataUrl.slice(m[0].length) : dataUrl;
|
return m ? dataUrl.slice(m[0].length) : dataUrl;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Parse a PNG's IHDR chunk and return `(width, height)`. Returns
|
|
||||||
* `null` on any malformed input so callers fall back to `0/0` instead
|
|
||||||
* of throwing.
|
|
||||||
*
|
|
||||||
* PNG layout: 8-byte signature, then a 4-byte length, 4-byte type
|
|
||||||
* ("IHDR"), then the chunk data — width is bytes 16-19 BE, height is
|
|
||||||
* 20-23 BE.
|
|
||||||
*/
|
|
||||||
export function parsePngDimensions(base64: string): { width: number; height: number } | null {
|
|
||||||
try {
|
|
||||||
// atob is available in MV3 service workers.
|
|
||||||
const head = base64.length > 64 ? base64.slice(0, 64) : base64;
|
|
||||||
const bin = atob(head);
|
|
||||||
if (bin.length < 24) return null;
|
|
||||||
if (bin.charCodeAt(0) !== 0x89 || bin.charCodeAt(1) !== 0x50 || bin.charCodeAt(2) !== 0x4e) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
const u32 = (off: number) =>
|
|
||||||
(bin.charCodeAt(off) << 24) |
|
|
||||||
(bin.charCodeAt(off + 1) << 16) |
|
|
||||||
(bin.charCodeAt(off + 2) << 8) |
|
|
||||||
bin.charCodeAt(off + 3);
|
|
||||||
const width = u32(16) >>> 0;
|
|
||||||
const height = u32(20) >>> 0;
|
|
||||||
if (width === 0 || height === 0) return null;
|
|
||||||
return { width, height };
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ScreenshotDeps {
|
export interface ScreenshotDeps {
|
||||||
cdp?: SharedCdpRunner;
|
cdp?: SharedCdpRunner;
|
||||||
tabsApi: ChromeTabsApi;
|
tabsApi: ChromeTabsApi;
|
||||||
@@ -308,12 +281,17 @@ export async function handleScreenshot(
|
|||||||
if (!deps.cdp) {
|
if (!deps.cdp) {
|
||||||
return { code: "cdp_failed", message: "screenshot ref capture requires CDP" };
|
return { code: "cdp_failed", message: "screenshot ref capture requires CDP" };
|
||||||
}
|
}
|
||||||
if (lookupRefTarget(ctx, ref, target.tabId)?.kind === "visual-region")
|
const entry = lookupRefTarget(ctx, ref, target.tabId);
|
||||||
return rpcError(
|
if (entry?.kind === "visual-region") {
|
||||||
"unsupported",
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
"ref_kind_unsupported",
|
const captured = await withExtensionOverlayHidden(
|
||||||
"visual-region screenshot execution is not available in this build",
|
target.tabId,
|
||||||
|
() => captureVisualScreenshot(deps.cdp!, entry.candidate, signal),
|
||||||
|
deps.sendToTab,
|
||||||
);
|
);
|
||||||
|
if (isRpcError(captured)) return captured;
|
||||||
|
return withShotDialogs({ ...captured, format: "png", tab_id: target.tabId });
|
||||||
|
}
|
||||||
const node = resolveSnapshotRef(ctx, ref, target.tabId);
|
const node = resolveSnapshotRef(ctx, ref, target.tabId);
|
||||||
if (isRpcError(node)) return node;
|
if (isRpcError(node)) return node;
|
||||||
if (signal?.aborted) return cancelled("screenshot");
|
if (signal?.aborted) return cancelled("screenshot");
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
/**
|
||||||
|
* Parse a PNG's IHDR chunk and return `(width, height)`. Returns
|
||||||
|
* `null` on any malformed input so callers fall back to `0/0` instead
|
||||||
|
* of throwing.
|
||||||
|
*
|
||||||
|
* PNG layout: 8-byte signature, then a 4-byte length, 4-byte type
|
||||||
|
* ("IHDR"), then the chunk data — width is bytes 16-19 BE, height is
|
||||||
|
* 20-23 BE.
|
||||||
|
*/
|
||||||
|
export function parsePngDimensions(base64: string): { width: number; height: number } | null {
|
||||||
|
try {
|
||||||
|
// atob is available in MV3 service workers.
|
||||||
|
const head = base64.length > 64 ? base64.slice(0, 64) : base64;
|
||||||
|
const bin = atob(head);
|
||||||
|
if (bin.length < 33) return null;
|
||||||
|
if (bin.slice(0, 8) !== "\x89PNG\r\n\x1a\n" || bin.slice(8, 16) !== "\x00\x00\x00\x0dIHDR") {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const u32 = (off: number) =>
|
||||||
|
(bin.charCodeAt(off) << 24) |
|
||||||
|
(bin.charCodeAt(off + 1) << 16) |
|
||||||
|
(bin.charCodeAt(off + 2) << 8) |
|
||||||
|
bin.charCodeAt(off + 3);
|
||||||
|
const width = u32(16) >>> 0;
|
||||||
|
const height = u32(20) >>> 0;
|
||||||
|
if (width === 0 || height === 0) return null;
|
||||||
|
return { width, height };
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,382 @@
|
|||||||
|
import type { CdpFrame } from "@/browser-driver/frame-graph";
|
||||||
|
import type { RpcError } from "@/transport/types";
|
||||||
|
import { rpcError } from "./errors";
|
||||||
|
import {
|
||||||
|
type GeometryProjection,
|
||||||
|
projectRectToViewport,
|
||||||
|
type Size,
|
||||||
|
type ViewportRect,
|
||||||
|
} from "./geometry";
|
||||||
|
import { type CssViewport, screenshotPageRect } from "./geometry/coordinate-types";
|
||||||
|
import { cssViewport, GeometryContext } from "./geometry/frame-context";
|
||||||
|
import { parsePngDimensions } from "./png";
|
||||||
|
import { type CdpRunner, sendToCdpTarget } from "./shared";
|
||||||
|
import { isAbortError, throwIfAborted } from "./vom/capture-abort";
|
||||||
|
import { resolveVerifiedNode } from "./vom/document-identity";
|
||||||
|
import type { DocumentIdentity } from "./vom/facts";
|
||||||
|
import { VISUAL_STYLES } from "./vom/snapshot";
|
||||||
|
import type { VisualCandidate, VisualFramePath } from "./vom/visual-discovery";
|
||||||
|
import {
|
||||||
|
EMPTY_VISUAL_CONTEXT,
|
||||||
|
extendVisualContext,
|
||||||
|
projectVisualBox,
|
||||||
|
resolveVisualRegion,
|
||||||
|
type VisualClipSource,
|
||||||
|
type VisualContext,
|
||||||
|
visualProjectionIssue,
|
||||||
|
} from "./vom/visual-region";
|
||||||
|
|
||||||
|
interface LiveRow {
|
||||||
|
node: number;
|
||||||
|
tag: string;
|
||||||
|
box: ViewportRect;
|
||||||
|
client: ViewportRect;
|
||||||
|
contentSize: Size;
|
||||||
|
styles: Record<string, string>;
|
||||||
|
}
|
||||||
|
interface LiveFrame {
|
||||||
|
top: boolean;
|
||||||
|
dpr: number;
|
||||||
|
rows: LiveRow[]; // Anchor first, root last.
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read only one current ancestry, including shadow hosts, in the verified isolated world.
|
||||||
|
const READ_ANCESTRY = `function(styleNames) {
|
||||||
|
if (!this.isConnected || this.ownerDocument !== document) return null;
|
||||||
|
const rows = [];
|
||||||
|
for (let node = this; node; node = node.parentElement || node.getRootNode().host) {
|
||||||
|
if (!(node instanceof Element)) return null;
|
||||||
|
const s = getComputedStyle(node), r = node.getBoundingClientRect();
|
||||||
|
rows.push({ node, tag: node.localName,
|
||||||
|
box: { x:r.x, y:r.y, width:r.width, height:r.height },
|
||||||
|
client: { x:r.x+node.clientLeft, y:r.y+node.clientTop, width:node.clientWidth, height:node.clientHeight },
|
||||||
|
contentSize: { width:node.clientWidth-parseFloat(s.paddingLeft)-parseFloat(s.paddingRight), height:node.clientHeight-parseFloat(s.paddingTop)-parseFloat(s.paddingBottom) },
|
||||||
|
styles: Object.fromEntries(styleNames.map(key => [key,s.getPropertyValue(key)])) });
|
||||||
|
}
|
||||||
|
return { top: window === window.top, dpr: devicePixelRatio, rows };
|
||||||
|
}`;
|
||||||
|
|
||||||
|
interface DeepValue {
|
||||||
|
type: string;
|
||||||
|
value?: unknown;
|
||||||
|
}
|
||||||
|
/** This read returns JSON primitives plus DOM nodes (backend IDs), never remote object handles. */
|
||||||
|
function decode(value: DeepValue): unknown {
|
||||||
|
if (value.type === "node") return (value.value as { backendNodeId?: number })?.backendNodeId;
|
||||||
|
if (value.type === "array") return (value.value as DeepValue[]).map(decode);
|
||||||
|
if (value.type === "object")
|
||||||
|
return Object.fromEntries(
|
||||||
|
(value.value as [string, DeepValue][]).map(([key, item]) => [key, decode(item)]),
|
||||||
|
);
|
||||||
|
if (value.type === "null") return null;
|
||||||
|
if (["string", "number", "boolean"].includes(value.type)) return value.value;
|
||||||
|
throw new Error("unexpected live ancestry serialization");
|
||||||
|
}
|
||||||
|
|
||||||
|
function stale(message: string): RpcError {
|
||||||
|
return rpcError(
|
||||||
|
"not_found",
|
||||||
|
"visual_target_changed",
|
||||||
|
`${message}; observe again before requesting a screenshot`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function sameIdentity(a: DocumentIdentity, b: DocumentIdentity): boolean {
|
||||||
|
return (
|
||||||
|
a.attachmentId === b.attachmentId &&
|
||||||
|
a.frameId === b.frameId &&
|
||||||
|
a.target.tabId === b.target.tabId &&
|
||||||
|
a.target.sessionId === b.target.sessionId &&
|
||||||
|
a.documentElementBackendNodeId === b.documentElementBackendNodeId
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function closeRect(a: ViewportRect, b: ViewportRect): boolean {
|
||||||
|
return [
|
||||||
|
a.x - b.x,
|
||||||
|
a.y - b.y,
|
||||||
|
a.x + a.width - b.x - b.width,
|
||||||
|
a.y + a.height - b.y - b.height,
|
||||||
|
].every((delta) => Number.isFinite(delta) && Math.abs(delta) <= 0.25);
|
||||||
|
}
|
||||||
|
|
||||||
|
function sameClips(a?: VisualClipSource, b?: VisualClipSource): boolean {
|
||||||
|
while (a && b) {
|
||||||
|
if (
|
||||||
|
!sameIdentity(a.document, b.document) ||
|
||||||
|
a.backendNodeId !== b.backendNodeId ||
|
||||||
|
a.x !== b.x ||
|
||||||
|
a.y !== b.y ||
|
||||||
|
a.overflowX !== b.overflowX ||
|
||||||
|
a.overflowY !== b.overflowY ||
|
||||||
|
!closeRect(a.box, b.box)
|
||||||
|
)
|
||||||
|
return false;
|
||||||
|
a = a.parent;
|
||||||
|
b = b.parent;
|
||||||
|
}
|
||||||
|
return !a && !b;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Scale affects raster size only, never the selected CSS region. */
|
||||||
|
export function visualScreenshotScale(width: number, height: number, pixelsPerCss: number): number {
|
||||||
|
if (![width, height, pixelsPerCss].every((n) => Number.isFinite(n) && n > 0))
|
||||||
|
throw new Error("invalid screenshot pixel dimensions");
|
||||||
|
return Math.min(
|
||||||
|
1,
|
||||||
|
2048 / (Math.max(width, height) * pixelsPerCss),
|
||||||
|
Math.sqrt(4_000_000 / (width * height * pixelsPerCss * pixelsPerCss)),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readFrame(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
document: DocumentIdentity,
|
||||||
|
anchor: number,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<LiveFrame | RpcError> {
|
||||||
|
const verified = await resolveVerifiedNode(cdp, document, anchor, signal);
|
||||||
|
if (verified.status !== "current") return stale(`visual DOM identity ${verified.status}`);
|
||||||
|
try {
|
||||||
|
throwIfAborted(signal);
|
||||||
|
const reply = await sendToCdpTarget<{
|
||||||
|
result?: { deepSerializedValue?: DeepValue };
|
||||||
|
exceptionDetails?: unknown;
|
||||||
|
}>(cdp, document.target, "Runtime.callFunctionOn", {
|
||||||
|
objectId: verified.objectId,
|
||||||
|
objectGroup: verified.objectGroup,
|
||||||
|
functionDeclaration: READ_ANCESTRY,
|
||||||
|
arguments: [{ value: VISUAL_STYLES }],
|
||||||
|
serializationOptions: {
|
||||||
|
serialization: "deep",
|
||||||
|
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
throwIfAborted(signal);
|
||||||
|
if (reply.exceptionDetails || !reply.result?.deepSerializedValue)
|
||||||
|
return stale("visual ancestry unavailable");
|
||||||
|
const result = decode(reply.result.deepSerializedValue) as LiveFrame | null;
|
||||||
|
if (
|
||||||
|
!result?.rows?.length ||
|
||||||
|
result.rows[0].node !== anchor ||
|
||||||
|
result.rows.at(-1)?.node !== document.documentElementBackendNodeId ||
|
||||||
|
!result.rows.every((row) => Number.isSafeInteger(row.node) && row.node > 0)
|
||||||
|
)
|
||||||
|
return stale("visual ancestry incomplete");
|
||||||
|
if (cdp.getAttachmentId?.(document.target.tabId) !== document.attachmentId)
|
||||||
|
return stale("visual attachment changed");
|
||||||
|
return result;
|
||||||
|
} finally {
|
||||||
|
await sendToCdpTarget(cdp, document.target, "Runtime.releaseObjectGroup", {
|
||||||
|
objectGroup: verified.objectGroup,
|
||||||
|
}).catch(() => {});
|
||||||
|
throwIfAborted(signal);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** All live reads for one screenshot use this one measurement context. */
|
||||||
|
async function resolveVisualRegionNow(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
candidate: VisualCandidate,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<{ crop: ViewportRect; viewport: CssViewport; dpr: number } | RpcError> {
|
||||||
|
throwIfAborted(signal);
|
||||||
|
if (!candidate.framePath || !sameIdentity(candidate.document, candidate.framePath.document))
|
||||||
|
return stale("visual frame path unavailable");
|
||||||
|
const path: { frame: VisualFramePath; anchor: number }[] = [];
|
||||||
|
const seen = new Set<string>();
|
||||||
|
let frame: VisualFramePath | undefined = candidate.framePath;
|
||||||
|
let anchor = candidate.backendNodeId;
|
||||||
|
while (frame) {
|
||||||
|
if (
|
||||||
|
seen.has(frame.document.frameId) ||
|
||||||
|
frame.document.target.tabId !== candidate.document.target.tabId ||
|
||||||
|
cdp.getAttachmentId?.(frame.document.target.tabId) !== frame.document.attachmentId
|
||||||
|
)
|
||||||
|
return stale("invalid visual frame path");
|
||||||
|
seen.add(frame.document.frameId);
|
||||||
|
path.push({ frame, anchor });
|
||||||
|
anchor = frame.parent?.ownerBackendNodeId ?? 0;
|
||||||
|
frame = frame.parent?.frame;
|
||||||
|
}
|
||||||
|
path.reverse();
|
||||||
|
const frames: CdpFrame[] = path.map(({ frame }) => ({
|
||||||
|
frameId: frame.document.frameId,
|
||||||
|
target: frame.document.target,
|
||||||
|
...(frame.parent
|
||||||
|
? {
|
||||||
|
parentFrameId: frame.parent.frame.document.frameId,
|
||||||
|
ownerBackendNodeId: frame.parent.ownerBackendNodeId,
|
||||||
|
}
|
||||||
|
: {}),
|
||||||
|
}));
|
||||||
|
// Validate each recorded edge instead of asking the driver to fill all page owners.
|
||||||
|
for (const { frame } of path) {
|
||||||
|
if (!frame.parent) continue;
|
||||||
|
throwIfAborted(signal);
|
||||||
|
const owner = await sendToCdpTarget<{ backendNodeId?: number }>(
|
||||||
|
cdp,
|
||||||
|
frame.parent.frame.document.target,
|
||||||
|
"DOM.getFrameOwner",
|
||||||
|
{ frameId: frame.document.frameId },
|
||||||
|
);
|
||||||
|
if (owner.backendNodeId !== frame.parent.ownerBackendNodeId)
|
||||||
|
return stale("visual frame owner changed");
|
||||||
|
}
|
||||||
|
const geometry = new GeometryContext(
|
||||||
|
cdp,
|
||||||
|
candidate.document.target.tabId,
|
||||||
|
{ rootFrameId: frames[0].frameId, frames },
|
||||||
|
signal,
|
||||||
|
);
|
||||||
|
let context: VisualContext = EMPTY_VISUAL_CONTEXT;
|
||||||
|
let parentRead: LiveFrame | undefined;
|
||||||
|
let finalRegion: VisualCandidate["region"] | undefined;
|
||||||
|
let topDpr = 0;
|
||||||
|
for (let i = 0; i < path.length; i++) {
|
||||||
|
const { frame, anchor } = path[i];
|
||||||
|
const live = await readFrame(cdp, frame.document, anchor, signal);
|
||||||
|
if ("code" in live) return live;
|
||||||
|
if (live.top !== (i === 0)) return stale("visual root frame changed");
|
||||||
|
if (i === 0) topDpr = live.dpr;
|
||||||
|
const metrics = await geometry.layoutMetrics(frame.document.target);
|
||||||
|
const viewport = cssViewport(metrics);
|
||||||
|
const projections: GeometryProjection[] = [];
|
||||||
|
if (frame.parent) {
|
||||||
|
const parent = frame.parent.frame.document;
|
||||||
|
const size = parentRead!.rows[0].contentSize;
|
||||||
|
if (![size.width, size.height].every((n) => Number.isFinite(n) && n > 0))
|
||||||
|
return stale("iframe content size unavailable");
|
||||||
|
const parentViewport = await geometry.viewport(parent.target);
|
||||||
|
if (!parentViewport) return stale("parent viewport unavailable");
|
||||||
|
const local = await geometry.snapshotProjection(
|
||||||
|
{ target: parent.target, frameId: frame.document.frameId },
|
||||||
|
frame.parent.ownerBackendNodeId,
|
||||||
|
[],
|
||||||
|
parentViewport,
|
||||||
|
size,
|
||||||
|
);
|
||||||
|
const outer = await geometry.targetProjection(parent.frameId);
|
||||||
|
if (local.status !== "available" || !outer)
|
||||||
|
return stale("visual frame projection unavailable");
|
||||||
|
projections.push(local.projection.geometry, outer);
|
||||||
|
} else projections.push({ sourceClips: [], edges: [], topViewport: viewport });
|
||||||
|
const issue = visualProjectionIssue({
|
||||||
|
projections,
|
||||||
|
coordinates: { layoutUnitsPerCssPixel: 1, scrollCss: { x: 0, y: 0 } },
|
||||||
|
pageScale: metrics.cssVisualViewport?.scale ?? metrics.visualViewport?.scale,
|
||||||
|
});
|
||||||
|
if (issue) return stale(issue);
|
||||||
|
for (let j = live.rows.length - 1; j >= 0; j--) {
|
||||||
|
const row = live.rows[j];
|
||||||
|
const isAnchor = j === 0;
|
||||||
|
const node = {
|
||||||
|
document: frame.document,
|
||||||
|
backendNodeId: row.node,
|
||||||
|
styles: row.styles,
|
||||||
|
clientBox: projectVisualBox(row.client, projections),
|
||||||
|
};
|
||||||
|
context = extendVisualContext(
|
||||||
|
context,
|
||||||
|
node,
|
||||||
|
!isAnchor && row.tag !== "iframe" && row.tag !== "frame",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (i < path.length - 1) {
|
||||||
|
const owner = live.rows[0];
|
||||||
|
context = {
|
||||||
|
...context,
|
||||||
|
hidden:
|
||||||
|
context.hidden ||
|
||||||
|
owner.styles.visibility === "hidden" ||
|
||||||
|
owner.styles.visibility === "collapse",
|
||||||
|
transformed: false,
|
||||||
|
localClip: false,
|
||||||
|
unpositionedClip: false,
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
const row = live.rows[0];
|
||||||
|
if (row.tag !== "canvas") return stale("visual anchor is no longer Canvas");
|
||||||
|
let visible: ViewportRect | null = row.box;
|
||||||
|
for (const projection of projections)
|
||||||
|
visible = visible
|
||||||
|
? projectRectToViewport(
|
||||||
|
{ x: visible.x, y: visible.y, w: visible.width, h: visible.height },
|
||||||
|
projection,
|
||||||
|
)
|
||||||
|
: null;
|
||||||
|
const region = resolveVisualRegion({
|
||||||
|
borderBox: projectVisualBox(row.box, projections),
|
||||||
|
frameVisibleBox: visible,
|
||||||
|
context,
|
||||||
|
visibility: row.styles.visibility,
|
||||||
|
});
|
||||||
|
if (region.status !== "available")
|
||||||
|
return stale(
|
||||||
|
`visual region ${region.status}${region.status === "unavailable" ? `: ${region.reason}` : ""}`,
|
||||||
|
);
|
||||||
|
finalRegion = region;
|
||||||
|
}
|
||||||
|
parentRead = live;
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
!finalRegion ||
|
||||||
|
!closeRect(candidate.region.borderBox, finalRegion.borderBox) ||
|
||||||
|
!closeRect(candidate.region.crop, finalRegion.crop) ||
|
||||||
|
!sameClips(candidate.region.clips, finalRegion.clips)
|
||||||
|
)
|
||||||
|
return stale("visual region changed");
|
||||||
|
const viewport = cssViewport(await geometry.layoutMetrics(frames[0].target));
|
||||||
|
return { crop: finalRegion.crop, viewport, dpr: topDpr };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** One target only. No frame discovery, scrolling, AX, or snapshot recapture. */
|
||||||
|
export async function captureVisualScreenshot(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
candidate: VisualCandidate,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<{ image_base64: string; width: number; height: number } | RpcError> {
|
||||||
|
try {
|
||||||
|
const region = await resolveVisualRegionNow(cdp, candidate, signal);
|
||||||
|
if ("code" in region) return region;
|
||||||
|
const clip = screenshotPageRect(region.crop, region.viewport);
|
||||||
|
if (!clip) return stale("invalid screenshot coordinates");
|
||||||
|
let scale = visualScreenshotScale(region.crop.width, region.crop.height, region.dpr);
|
||||||
|
for (let attempt = 0; attempt < 2; attempt++) {
|
||||||
|
throwIfAborted(signal);
|
||||||
|
if (
|
||||||
|
cdp.getAttachmentId?.(candidate.document.target.tabId) !== candidate.document.attachmentId
|
||||||
|
)
|
||||||
|
return stale("visual attachment changed");
|
||||||
|
const shot = await cdp.send<{ data?: string }>(
|
||||||
|
candidate.document.target.tabId,
|
||||||
|
"Page.captureScreenshot",
|
||||||
|
{ format: "png", captureBeyondViewport: false, clip: { ...clip.rect, scale } },
|
||||||
|
);
|
||||||
|
throwIfAborted(signal);
|
||||||
|
const dims = shot.data ? parsePngDimensions(shot.data) : null;
|
||||||
|
if (!dims)
|
||||||
|
return rpcError(
|
||||||
|
"cdp_failed",
|
||||||
|
"screenshot_capture_failed",
|
||||||
|
"visual screenshot returned invalid PNG dimensions",
|
||||||
|
);
|
||||||
|
const correction = visualScreenshotScale(dims.width, dims.height, 1);
|
||||||
|
if (correction === 1) return { image_base64: shot.data!, ...dims };
|
||||||
|
scale *= correction * 0.99;
|
||||||
|
}
|
||||||
|
return rpcError(
|
||||||
|
"cdp_failed",
|
||||||
|
"visual_pixel_budget_exceeded",
|
||||||
|
"visual screenshot exceeds the pixel budget",
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
if (isAbortError(error) || signal?.aborted)
|
||||||
|
return { code: "cancelled", message: "visual screenshot aborted" };
|
||||||
|
return rpcError(
|
||||||
|
"cdp_failed",
|
||||||
|
"screenshot_capture_failed",
|
||||||
|
error instanceof Error ? error.message : String(error),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -141,6 +141,32 @@ function facts(
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe("Canvas discovery", () => {
|
describe("Canvas discovery", () => {
|
||||||
|
it("shares frame address paths from existing Facts without changing candidate evidence", async () => {
|
||||||
|
const parent = await document([{ id: 2, parent: 1, tag: "iframe" }]);
|
||||||
|
const child = await document(
|
||||||
|
[
|
||||||
|
{ id: 3, parent: 1 },
|
||||||
|
{ id: 4, parent: 1 },
|
||||||
|
],
|
||||||
|
{
|
||||||
|
frame: {
|
||||||
|
frameId: "child",
|
||||||
|
parentFrameId: "main",
|
||||||
|
ownerBackendNodeId: 2,
|
||||||
|
target: { tabId: 1 },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const result = await discoverVisualCandidates(facts([child, parent]));
|
||||||
|
expect(result.candidates).toHaveLength(2);
|
||||||
|
const path = result.candidates[0].framePath;
|
||||||
|
expect(path).toBe(result.candidates[1].framePath);
|
||||||
|
expect(path?.document).toBe(child.identity);
|
||||||
|
expect(path?.parent?.ownerBackendNodeId).toBe(2);
|
||||||
|
expect(path?.parent?.frame.document).toBe(parent.identity);
|
||||||
|
expect(path?.parent?.frame.parent).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
it("distinguishes visual facts not collected from complete empty discovery and honors cancellation", async () => {
|
it("distinguishes visual facts not collected from complete empty discovery and honors cancellation", async () => {
|
||||||
const empty = facts([]);
|
const empty = facts([]);
|
||||||
expect(await discoverVisualCandidates(empty)).toMatchObject({
|
expect(await discoverVisualCandidates(empty)).toMatchObject({
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ export async function verifyDocumentIdentity(
|
|||||||
identity: DocumentIdentity,
|
identity: DocumentIdentity,
|
||||||
signal?: AbortSignal,
|
signal?: AbortSignal,
|
||||||
): Promise<"current" | "changed" | "unavailable"> {
|
): Promise<"current" | "changed" | "unavailable"> {
|
||||||
return verifyIdentity(cdp, identity, signal);
|
return (await verifyIdentity(cdp, identity, signal)).status;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Verify the visual anchor in its original frame; geometry is checked by screenshot execution. */
|
/** Verify the visual anchor in its original frame; geometry is checked by screenshot execution. */
|
||||||
@@ -20,7 +20,24 @@ export async function verifyVisualTargetIdentity(
|
|||||||
candidate: VisualCandidate,
|
candidate: VisualCandidate,
|
||||||
signal?: AbortSignal,
|
signal?: AbortSignal,
|
||||||
): Promise<"current" | "changed" | "unavailable"> {
|
): Promise<"current" | "changed" | "unavailable"> {
|
||||||
return verifyIdentity(cdp, candidate.document, signal, candidate.backendNodeId);
|
return (await verifyIdentity(cdp, candidate.document, signal, candidate.backendNodeId)).status;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type VerifiedNodeResult =
|
||||||
|
| { status: "changed" | "unavailable" }
|
||||||
|
| { status: "current"; objectId: string; objectGroup: string };
|
||||||
|
|
||||||
|
/** Caller must release objectGroup in finally after its local read. */
|
||||||
|
export async function resolveVerifiedNode(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
identity: DocumentIdentity,
|
||||||
|
backendNodeId: number,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
): Promise<VerifiedNodeResult> {
|
||||||
|
const result = await verifyIdentity(cdp, identity, signal, backendNodeId, true);
|
||||||
|
return result.status === "current" && result.objectId && result.objectGroup
|
||||||
|
? { status: "current", objectId: result.objectId, objectGroup: result.objectGroup }
|
||||||
|
: { status: result.status === "changed" ? "changed" : "unavailable" };
|
||||||
}
|
}
|
||||||
|
|
||||||
async function verifyIdentity(
|
async function verifyIdentity(
|
||||||
@@ -28,15 +45,22 @@ async function verifyIdentity(
|
|||||||
identity: DocumentIdentity,
|
identity: DocumentIdentity,
|
||||||
signal?: AbortSignal,
|
signal?: AbortSignal,
|
||||||
anchorBackendNodeId?: number,
|
anchorBackendNodeId?: number,
|
||||||
): Promise<"current" | "changed" | "unavailable"> {
|
retain = false,
|
||||||
|
): Promise<{
|
||||||
|
status: "current" | "changed" | "unavailable";
|
||||||
|
objectId?: string;
|
||||||
|
objectGroup?: string;
|
||||||
|
}> {
|
||||||
const attached = () => cdp.getAttachmentId?.(identity.target.tabId) === identity.attachmentId;
|
const attached = () => cdp.getAttachmentId?.(identity.target.tabId) === identity.attachmentId;
|
||||||
throwIfAborted(signal);
|
throwIfAborted(signal);
|
||||||
if (!attached()) return "changed";
|
if (!attached()) return { status: "changed" };
|
||||||
const objectGroup = `bsk-document-identity-${crypto.randomUUID()}`;
|
const objectGroup = `bsk-document-identity-${crypto.randomUUID()}`;
|
||||||
const send = <T>(method: string, params: object) => {
|
const send = <T>(method: string, params: object) => {
|
||||||
throwIfAborted(signal);
|
throwIfAborted(signal);
|
||||||
return sendToCdpTarget<T>(cdp, identity.target, method, params);
|
return sendToCdpTarget<T>(cdp, identity.target, method, params);
|
||||||
};
|
};
|
||||||
|
let objectId: string | undefined;
|
||||||
|
let retained = false;
|
||||||
try {
|
try {
|
||||||
const world = await send<{ executionContextId: number }>("Page.createIsolatedWorld", {
|
const world = await send<{ executionContextId: number }>("Page.createIsolatedWorld", {
|
||||||
frameId: identity.frameId,
|
frameId: identity.frameId,
|
||||||
@@ -63,7 +87,8 @@ async function verifyIdentity(
|
|||||||
executionContextId: world.executionContextId,
|
executionContextId: world.executionContextId,
|
||||||
objectGroup,
|
objectGroup,
|
||||||
});
|
});
|
||||||
if (!anchor.object?.objectId) return "unavailable";
|
if (!anchor.object?.objectId) return { status: "unavailable" };
|
||||||
|
objectId = anchor.object.objectId;
|
||||||
reply = await send<RootReply>("Runtime.callFunctionOn", {
|
reply = await send<RootReply>("Runtime.callFunctionOn", {
|
||||||
objectId: anchor.object.objectId,
|
objectId: anchor.object.objectId,
|
||||||
functionDeclaration:
|
functionDeclaration:
|
||||||
@@ -72,20 +97,24 @@ async function verifyIdentity(
|
|||||||
serializationOptions,
|
serializationOptions,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
if (!attached()) return "changed";
|
if (!attached()) return { status: "changed" };
|
||||||
const root = reply.result?.deepSerializedValue;
|
const root = reply.result?.deepSerializedValue;
|
||||||
if (root?.type === "null") return "changed";
|
if (root?.type === "null") return { status: "changed" };
|
||||||
const id = root?.type === "node" ? root.value?.backendNodeId : undefined;
|
const id = root?.type === "node" ? root.value?.backendNodeId : undefined;
|
||||||
if (id === undefined) return "unavailable";
|
if (id === undefined) return { status: "unavailable" };
|
||||||
return id === identity.documentElementBackendNodeId ? "current" : "changed";
|
if (id !== identity.documentElementBackendNodeId) return { status: "changed" };
|
||||||
|
throwIfAborted(signal);
|
||||||
|
retained = retain && !!objectId;
|
||||||
|
return { status: "current", ...(retained ? { objectId, objectGroup } : {}) };
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
throwIfAborted(signal);
|
throwIfAborted(signal);
|
||||||
if (isAbortError(error)) throw error;
|
if (isAbortError(error)) throw error;
|
||||||
return attached() ? "unavailable" : "changed";
|
return { status: attached() ? "unavailable" : "changed" };
|
||||||
} finally {
|
} finally {
|
||||||
await sendToCdpTarget(cdp, identity.target, "Runtime.releaseObjectGroup", {
|
if (!retained)
|
||||||
objectGroup,
|
await sendToCdpTarget(cdp, identity.target, "Runtime.releaseObjectGroup", {
|
||||||
}).catch(() => {});
|
objectGroup,
|
||||||
|
}).catch(() => {});
|
||||||
throwIfAborted(signal);
|
throwIfAborted(signal);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -20,11 +20,19 @@ import {
|
|||||||
visualProjectionIssue,
|
visualProjectionIssue,
|
||||||
} from "./visual-region";
|
} from "./visual-region";
|
||||||
|
|
||||||
|
/** Shared observation-time addresses, not cached live geometry. */
|
||||||
|
export interface VisualFramePath {
|
||||||
|
readonly document: DocumentIdentity;
|
||||||
|
readonly parent?: { readonly ownerBackendNodeId: number; readonly frame: VisualFramePath };
|
||||||
|
}
|
||||||
|
|
||||||
export interface VisualCandidate {
|
export interface VisualCandidate {
|
||||||
readonly document: DocumentIdentity;
|
readonly document: DocumentIdentity;
|
||||||
readonly backendNodeId: number;
|
readonly backendNodeId: number;
|
||||||
readonly parentBackendNodeId: number | null;
|
readonly parentBackendNodeId: number | null;
|
||||||
readonly label?: string;
|
readonly label?: string;
|
||||||
|
/** Absent when execution ancestry could not be established; discovery is still retained. */
|
||||||
|
readonly framePath?: VisualFramePath;
|
||||||
readonly region: Extract<VisualRegionResult, { status: "available" }>;
|
readonly region: Extract<VisualRegionResult, { status: "available" }>;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -113,6 +121,7 @@ export async function discoverVisualCandidates(
|
|||||||
const candidates: VisualCandidate[] = [];
|
const candidates: VisualCandidate[] = [];
|
||||||
const issues: VisualDiscoveryIssue[] = [];
|
const issues: VisualDiscoveryIssue[] = [];
|
||||||
const documents = new Map(facts.documents.map((document) => [document.frame.frameId, document]));
|
const documents = new Map(facts.documents.map((document) => [document.frame.frameId, document]));
|
||||||
|
const framePaths = new Map<string, VisualFramePath>();
|
||||||
const contexts = new Map<string, Map<number, NodeContext>>();
|
const contexts = new Map<string, Map<number, NodeContext>>();
|
||||||
const roots = new Map<string, VisualContext>();
|
const roots = new Map<string, VisualContext>();
|
||||||
|
|
||||||
@@ -248,6 +257,16 @@ export async function discoverVisualCandidates(
|
|||||||
root.issue ??
|
root.issue ??
|
||||||
(!doc.identity ? "identity-unverified" : visualProjectionIssue(doc.geometry)),
|
(!doc.identity ? "identity-unverified" : visualProjectionIssue(doc.geometry)),
|
||||||
};
|
};
|
||||||
|
if (doc.identity) {
|
||||||
|
const parentPath = frame.parentFrameId ? framePaths.get(frame.parentFrameId) : undefined;
|
||||||
|
if (!frame.parentFrameId && frame.frameId === facts.rootFrameId)
|
||||||
|
framePaths.set(frame.frameId, { document: doc.identity });
|
||||||
|
else if (parentPath && frame.ownerBackendNodeId !== undefined)
|
||||||
|
framePaths.set(frame.frameId, {
|
||||||
|
document: doc.identity,
|
||||||
|
parent: { ownerBackendNodeId: frame.ownerBackendNodeId, frame: parentPath },
|
||||||
|
});
|
||||||
|
}
|
||||||
roots.set(frame.frameId, root);
|
roots.set(frame.frameId, root);
|
||||||
contexts.set(frame.frameId, new Map());
|
contexts.set(frame.frameId, new Map());
|
||||||
}
|
}
|
||||||
@@ -291,6 +310,7 @@ export async function discoverVisualCandidates(
|
|||||||
const label = node.attrs["aria-label"]?.trim() || node.attrs.title?.trim();
|
const label = node.attrs["aria-label"]?.trim() || node.attrs.title?.trim();
|
||||||
candidates.push({
|
candidates.push({
|
||||||
document: document.identity,
|
document: document.identity,
|
||||||
|
framePath: framePaths.get(document.frame.frameId),
|
||||||
backendNodeId: node.backendNodeId,
|
backendNodeId: node.backendNodeId,
|
||||||
parentBackendNodeId: node.parentBackendNodeId,
|
parentBackendNodeId: node.parentBackendNodeId,
|
||||||
...(label ? { label } : {}),
|
...(label ? { label } : {}),
|
||||||
|
|||||||
@@ -25,6 +25,8 @@ export type RpcErrorReason =
|
|||||||
| "element_not_visible"
|
| "element_not_visible"
|
||||||
| "ref_not_found"
|
| "ref_not_found"
|
||||||
| "ref_kind_unsupported"
|
| "ref_kind_unsupported"
|
||||||
|
| "visual_target_changed"
|
||||||
|
| "visual_pixel_budget_exceeded"
|
||||||
| "selector_not_found"
|
| "selector_not_found"
|
||||||
| "target_not_fillable"
|
| "target_not_fillable"
|
||||||
| "fill_value_invalid"
|
| "fill_value_invalid"
|
||||||
|
|||||||
@@ -40,6 +40,8 @@ use bsk_protocol::ErrorCode;
|
|||||||
pub mod reason {
|
pub mod reason {
|
||||||
pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope";
|
pub const AGENT_WINDOW_SCOPE: &str = "agent_window_scope";
|
||||||
pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible";
|
pub const ELEMENT_NOT_VISIBLE: &str = "element_not_visible";
|
||||||
|
pub const VISUAL_TARGET_CHANGED: &str = "visual_target_changed";
|
||||||
|
pub const VISUAL_PIXEL_BUDGET_EXCEEDED: &str = "visual_pixel_budget_exceeded";
|
||||||
pub const REF_KIND_UNSUPPORTED: &str = "ref_kind_unsupported";
|
pub const REF_KIND_UNSUPPORTED: &str = "ref_kind_unsupported";
|
||||||
pub const REF_NOT_FOUND: &str = "ref_not_found";
|
pub const REF_NOT_FOUND: &str = "ref_not_found";
|
||||||
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
|
pub const SELECTOR_NOT_FOUND: &str = "selector_not_found";
|
||||||
@@ -248,6 +250,18 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
|||||||
),
|
),
|
||||||
exit_code: base.exit_code,
|
exit_code: base.exit_code,
|
||||||
},
|
},
|
||||||
|
(ErrorCode::NotFound, reason::VISUAL_TARGET_CHANGED) => RenderInfo {
|
||||||
|
summary: "visual target needs a fresh observation",
|
||||||
|
hint: Some(
|
||||||
|
"observe the page again and use the current visual ref; the previous identity or region cannot be used",
|
||||||
|
),
|
||||||
|
exit_code: base.exit_code,
|
||||||
|
},
|
||||||
|
(ErrorCode::CdpFailed, reason::VISUAL_PIXEL_BUDGET_EXCEEDED) => RenderInfo {
|
||||||
|
summary: "visual screenshot exceeds the image size limit",
|
||||||
|
hint: Some("the screenshot could not be reduced within the pixel budget"),
|
||||||
|
exit_code: base.exit_code,
|
||||||
|
},
|
||||||
(ErrorCode::Unsupported, reason::REF_KIND_UNSUPPORTED) => RenderInfo {
|
(ErrorCode::Unsupported, reason::REF_KIND_UNSUPPORTED) => RenderInfo {
|
||||||
summary: "this tool does not support the ref target type",
|
summary: "this tool does not support the ref target type",
|
||||||
hint: Some(
|
hint: Some(
|
||||||
@@ -672,6 +686,16 @@ mod tests {
|
|||||||
assert!(info.summary.contains("timed out after browser dispatch"));
|
assert!(info.summary.contains("timed out after browser dispatch"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn visual_screenshot_failures_have_specific_guidance() {
|
||||||
|
let changed = serde_json::json!({ "reason": reason::VISUAL_TARGET_CHANGED });
|
||||||
|
let info = info_for_error(ErrorCode::NotFound, Some(&changed));
|
||||||
|
assert!(info.hint.unwrap().contains("observe"));
|
||||||
|
let budget = serde_json::json!({ "reason": reason::VISUAL_PIXEL_BUDGET_EXCEEDED });
|
||||||
|
let info = info_for_error(ErrorCode::CdpFailed, Some(&budget));
|
||||||
|
assert!(info.summary.contains("image size limit"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn visual_ref_type_error_has_specific_guidance() {
|
fn visual_ref_type_error_has_specific_guidance() {
|
||||||
let data = serde_json::json!({ "reason": reason::REF_KIND_UNSUPPORTED });
|
let data = serde_json::json!({ "reason": reason::REF_KIND_UNSUPPORTED });
|
||||||
|
|||||||
Reference in new issue
Block a user