Compare commits

..
16 Commits
Author SHA1 Message Date
drakezhang 4834d8b5fa test(geometry): identify the regression browser attachment 2026-09-09 16:11:39 +08:00
Ljy-0827 b4006d5b85 perf(observation): validate document identity against snapshot roots 2026-09-09 16:09:09 +08:00
Ljy-0827 2d1e5c6d9d fix(observation): validate document consistency with bounded identity checks 2026-09-09 16:09:09 +08:00
drakezhang 282e6f5623 fix(observation): carry snapshot units through capture coordination 2026-09-09 16:05:42 +08:00
Ljy-0827 bd68c293b6 perf(observation): collect frame AX trees with bounded concurrency 2026-09-09 15:58:56 +08:00
Ljy-0827 35a20582ad refactor(observation): separate capture coordination from document consistency 2026-09-09 15:58:56 +08:00
Ljy-0827 4e3365450a refactor(observation): integrate capture orchestration and document consistency 2026-09-09 15:58:56 +08:00
Ljy-0827 df5ed1a495 fix(observation): validate document consistency during capture 2026-09-09 15:58:40 +08:00
drakezhang eefbe03258 fix(observation): preserve snapshot unit normalization after document extraction 2026-09-09 15:58:28 +08:00
Ljy-0827 1d275f047c perf(observation): reduce document processing scheduling overhead 2026-09-09 15:54:41 +08:00
Ljy-0827 388b3ae2d5 refactor(observation): isolate document processing from capture orchestration 2026-09-09 15:54:41 +08:00
Ljy-0827 36d0d91419 fix(observation): preserve snapshot document provenance and URL metadata 2026-09-09 15:54:41 +08:00
Ljy-0827 bcb9beb4f8 refactor(observation): defer visual facts until visual discovery 2026-09-09 15:54:41 +08:00
Ljy-0827 34ccb96fd6 fix(observation): await capture task cleanup before propagating cancellation 2026-09-09 15:54:41 +08:00
Ljy-0827 8dfa097e55 refactor(observation): organize capture around document facts 2026-09-09 15:54:41 +08:00
drakezhang 383e3128eb fix(geometry): normalize snapshot layout bounds and scroll together 2026-09-09 15:53:35 +08:00
17 changed files with 873 additions and 70 deletions
@@ -5,9 +5,10 @@ import { rectPolygon } from "../geometry";
import {
projectSnapshotRect,
screenshotPageRect,
snapshotCoordinates,
snapshotViewportRect,
} from "../geometry/coordinate-types";
import { GeometryContext } from "../geometry/frame-context";
import { GeometryContext, snapshotLayoutScale } from "../geometry/frame-context";
import type { CdpRunner } from "../shared";
const target = { tabId: 4 };
@@ -22,7 +23,11 @@ const graph: CdpFrameGraph = {
function driver() {
const send = vi.fn(async (_tab: number, method: string): Promise<object> => {
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 1000 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 1000 },
};
if (method === "DOM.getBoxModel")
return { model: { content: [52.5, 612.5, 427.5, 612.5, 427.5, 862.5, 52.5, 862.5] } };
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
@@ -35,6 +40,63 @@ function driver() {
}
describe("measurement geometry context", () => {
it.each([0.8, 1, 1.25, 2])("normalizes snapshot layout units at scale %s", (scale) => {
const measured = snapshotLayoutScale({
visualViewport: { clientWidth: 997.25, zoom: 1.1 },
cssVisualViewport: { clientWidth: 997.25 / scale, zoom: 1.1 },
// Rounded layout dimensions must not supply the scale.
layoutViewport: { clientWidth: 997 },
cssLayoutViewport: { clientWidth: Math.floor(997.25 / scale) },
});
expect(measured).toBeCloseTo(scale, 12);
const coordinates = snapshotCoordinates(
{ scrollOffsetX: 80 * scale, scrollOffsetY: 200 * scale },
measured,
{ x: 999, y: 999 },
);
expect(
snapshotViewportRect(
[160 * scale, 320 * scale, 120 * scale, 40 * scale],
{ target },
coordinates,
)?.rect,
).toEqual({ x: 80, y: 120, width: 120, height: 40 });
});
it("uses CSS root scroll only for missing snapshot offsets, without scaling it twice", () => {
const coordinates = snapshotCoordinates({ scrollOffsetX: 80 }, 2, { x: 999, y: 100 });
expect(snapshotViewportRect([200, 800, 240, 80], { target }, coordinates)?.rect).toEqual({
x: 60,
y: 300,
width: 120,
height: 40,
});
expect(snapshotCoordinates({ scrollOffsetX: 0 }, 2)).toBeNull();
expect(
snapshotCoordinates({ scrollOffsetX: NaN, scrollOffsetY: 0 }, 2, { x: 0, y: 0 }),
).toBeNull();
});
it("does not infer snapshot units from missing, invalid, or CSS-only metrics", () => {
expect(snapshotLayoutScale({ cssLayoutViewport: { clientWidth: 800 } })).toBeNull();
for (const raw of [0, -1, NaN, Infinity]) {
expect(
snapshotLayoutScale({
visualViewport: { clientWidth: raw },
cssVisualViewport: { clientWidth: 800 },
}),
).toBeNull();
}
expect(
snapshotLayoutScale({
visualViewport: { clientWidth: 0, clientHeight: 600 },
cssVisualViewport: { clientWidth: 0, clientHeight: 300 },
}),
).toBe(2);
expect(snapshotCoordinates({ scrollOffsetX: 0, scrollOffsetY: 0 }, null)).toBeNull();
expect(snapshotViewportRect([10, 20, 120, 40], { target }, null)).toBeNull();
});
it("shares in-flight frame measurements, but never shares them with the next context", async () => {
const cdp = driver();
const context = new GeometryContext(cdp, 4);
@@ -61,14 +123,20 @@ describe("measurement geometry context", () => {
height: 1000,
});
if (projection.status !== "available") throw new Error("expected available projection");
const input = snapshotViewportRect([17, 23, 120, 40], source, { x: 0, y: 0 });
const input = snapshotViewportRect([17, 23, 120, 40], source, {
layoutUnitsPerCssPixel: 1,
scrollCss: { x: 0, y: 0 },
});
expect(projectSnapshotRect(input!, projection.projection)).toEqual({
x: 73.75,
y: 641.25,
width: 150,
height: 50,
});
const scrolled = snapshotViewportRect([17, 23, 120, 40], source, { x: 0, y: 10 });
const scrolled = snapshotViewportRect([17, 23, 120, 40], source, {
layoutUnitsPerCssPixel: 1,
scrollCss: { x: 0, y: 10 },
});
expect(projectSnapshotRect(scrolled!, projection.projection)?.y).toBe(628.75);
await context.snapshotProjection(source, 10, [], { width: 1000, height: 1000 });
expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.resolveNode")).toHaveLength(
@@ -173,7 +241,7 @@ describe("measurement geometry context", () => {
const input = snapshotViewportRect(
[30, 50, 120, 40],
{ target, frameId: "main" },
{ x: 10, y: 20 },
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: 10, y: 20 } },
)!;
const projection = {
source: { target, frameId: "main" },
@@ -194,8 +262,20 @@ describe("measurement geometry context", () => {
source: { target: { tabId: 5 }, frameId: "main" },
}),
).toBeNull();
expect(snapshotViewportRect([0, 0, NaN, 40], { target }, { x: 0, y: 0 })).toBeNull();
expect(snapshotViewportRect([0, 0, 10, 40], { target }, { x: Infinity, y: 0 })).toBeNull();
expect(
snapshotViewportRect(
[0, 0, NaN, 40],
{ target },
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: 0, y: 0 } },
),
).toBeNull();
expect(
snapshotViewportRect(
[0, 0, 10, 40],
{ target },
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: Infinity, y: 0 } },
),
).toBeNull();
});
it("does not turn a border quad or a failed owner read into a content projection", async () => {
@@ -228,7 +308,7 @@ describe("measurement geometry context", () => {
const input = snapshotViewportRect(
[0, 0, 300, 200],
{ target, frameId: "nested" },
{ x: 0, y: 0 },
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: 0, y: 0 } },
);
expect(projectSnapshotRect(input!, projection.projection)).toEqual({
x: 60,
@@ -278,7 +358,11 @@ describe("measurement geometry context", () => {
peak = Math.max(peak, active);
await new Promise<void>((resolve) => release.push(resolve));
active--;
return { cssLayoutViewport: { clientWidth: 100, clientHeight: 100 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 100, clientHeight: 100 },
};
});
const controller = new AbortController();
const context = new GeometryContext(
@@ -93,7 +93,11 @@ function makeFakeCdp(handlers: Record<string, (params?: object) => unknown>) {
sent.push({ method, params });
const handler = handlers[method];
if (!handler && method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 },
};
}
if (!handler) throw new Error(`unexpected CDP call ${method}`);
return handler(params);
@@ -2583,7 +2587,11 @@ describe("handleSnapshot", () => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") throw new Error("snapshot unsupported");
@@ -2670,7 +2678,11 @@ describe("handleSnapshot", () => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
@@ -2678,6 +2690,8 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "root",
nodes: {
parentIndex: [-1, 0],
@@ -2762,7 +2776,11 @@ describe("handleSnapshot", () => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
@@ -2770,6 +2788,8 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "root",
nodes: {
parentIndex: [-1, 0],
@@ -2857,7 +2877,11 @@ describe("handleSnapshot", () => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
@@ -2865,6 +2889,8 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "root",
nodes: {
parentIndex: [-1, 0],
@@ -3023,7 +3049,11 @@ describe("handleSnapshot", () => {
};
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
@@ -3130,6 +3160,8 @@ describe("handleSnapshot", () => {
strings: S,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 2],
@@ -3188,6 +3220,8 @@ describe("handleSnapshot", () => {
strings: S,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 1, 1, 1],
@@ -3238,6 +3272,8 @@ describe("handleSnapshot", () => {
}
const VP_METRICS = {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
@@ -3251,6 +3287,8 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "main",
nodes: {
parentIndex: [-1, 0, 1],
@@ -3271,6 +3309,8 @@ describe("handleSnapshot", () => {
},
},
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "child",
nodes: {
parentIndex: [-1, 0, 1],
@@ -3341,7 +3381,11 @@ describe("handleSnapshot", () => {
});
const sendToTarget = vi.fn(async (_target, method: string) => {
if (method === "Page.getLayoutMetrics" && ownerGeometry !== "unavailable")
return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 400, clientHeight: 300 },
};
if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot")
return { ...snapshot, documents: [snapshot.documents[1]] };
@@ -3384,6 +3428,8 @@ describe("handleSnapshot", () => {
strings: ["body", "a", "href", "//outside.test/docs", "https://page.test/start"],
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "main",
...(hasSnapshotUrl ? { documentURL: 4 } : {}),
nodes: {
@@ -3505,6 +3551,8 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 1, 3],
@@ -3649,14 +3697,14 @@ describe("handleSnapshot", () => {
});
expect(result.text.match(/@warning/g)).toHaveLength(2);
expect(result.text).toContain("document identity unverified");
expect(result.text).toContain("iframe geometry incomplete");
expect(result.text).toContain("geometry incomplete");
expect(result.text.startsWith("@vom 1\n")).toBe(true);
if (maxTokens === undefined) {
expect(result.truncated).toBe(false);
expect(result.text).toContain("Frame action");
expect(result.matchNodes.find((node) => node.backendNodeId === 22)).toMatchObject({
rect: null,
localRect: { x: 20, y: 30, w: 120, h: 40 },
localRect: null,
});
}
if (maxTokens === 100) expect(Math.ceil(result.text.length / 4)).toBeLessThanOrEqual(maxTokens);
@@ -3747,7 +3795,7 @@ describe("handleSnapshot", () => {
expect(result.matchNodes.find((node) => node.backendNodeId === 22)).toMatchObject({
frameId: "child",
tag: "button",
localRect: { x: 20, y: 30, w: 120, h: 40 },
localRect: null,
});
expect(result.refs.find((ref) => ref.backendNodeId === 22)).toMatchObject({
frameId: "child",
@@ -4091,6 +4139,8 @@ describe("handleSnapshot", () => {
strings: S,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 0, 3, 4, 5],
@@ -0,0 +1,234 @@
// @vitest-environment node
import { describe, expect, it } from "vitest";
import type { CdpFrame, CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
import type { CdpRunner } from "../shared";
import { captureObservationFacts } from "../vom/capture-coordinator";
type Send = <T = Record<string, unknown>>(
method: string,
params?: object,
sessionId?: string,
) => Promise<T>;
type Tree = { frame: { id: string; parentId?: string; name?: string }; childFrames?: Tree[] };
type Rect = { x: number; y: number; w: number; h: number };
type Oracle = { probe: Rect; owners: Record<string, { x: number; y: number; scale: number }> };
// The independent oracle uses DOM border boxes and this fixture's axis-aligned
// iframe transforms. No production snapshot conversion/projection builds expectations.
const oracleExpression = `(() => {
const box = document.querySelector('#probe').getBoundingClientRect();
const owners = {};
for (const frame of document.querySelectorAll('iframe')) {
const rect = frame.getBoundingClientRect();
const style = getComputedStyle(frame);
const scale = rect.width / frame.offsetWidth;
owners[frame.name] = {
x: rect.x + (frame.clientLeft + parseFloat(style.paddingLeft)) * scale,
y: rect.y + (frame.clientTop + parseFloat(style.paddingTop)) * scale,
scale,
};
}
return { probe: { x: box.x, y: box.y, w: box.width, h: box.height }, owners };
})()`;
describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate contract", () => {
it.each([
{ deviceScale: 1, zoom: 1 },
{ deviceScale: 0.8, zoom: 1 },
{ deviceScale: 1, zoom: 1.25 },
{ deviceScale: 2, zoom: 1 },
{ deviceScale: 2, zoom: 0.8 },
])("matches DOM geometry at device scale $deviceScale and zoom $zoom", async (configuration) => {
const evalRoot = new URL("../../../../../evals/browser/", import.meta.url);
const { createEvalServer } = await import(new URL("lib/server.mjs", evalRoot).href);
const { withChrome } = await import(
new URL("cases/regression/snapshot-coordinates/chrome.mjs", evalRoot).href
);
const server = createEvalServer();
const { baseUrl } = await server.start();
try {
await withChrome(
{ executable: process.env.BSK_GEOMETRY_CHROME, ...configuration },
async (send: Send) => {
const { targetId } = await send<{ targetId: string }>("Target.createTarget", {
url: "about:blank",
});
const { sessionId: rootSession } = await send<{ sessionId: string }>(
"Target.attachToTarget",
{ targetId, flatten: true },
);
await send(
"Page.navigate",
{ url: `${baseUrl}/snapshot-coordinates?run=coordinates` },
rootSession,
);
await expect
.poll(
() =>
server
.snapshot("coordinates")
.events.some(
(event: { type: string; data: { root?: boolean } }) =>
event.type === "geometry.ready" && event.data.root,
),
{ timeout: 10_000 },
)
.toBe(true);
const targets = await send<{ targetInfos: { targetId: string; type: string }[] }>(
"Target.getTargets",
);
const sessions = [rootSession];
for (const target of targets.targetInfos.filter((target) => target.type === "iframe")) {
const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", {
targetId: target.targetId,
flatten: true,
});
sessions.push(sessionId);
}
expect(sessions.length).toBe(2); // The cross-site fixture must actually be an OOPIF.
const frames: CdpFrame[] = [];
const names = new Map<string, string>();
const sessionFor = (target: CdpTarget) => target.sessionId ?? rootSession;
for (const sessionId of sessions) {
const { frameTree } = await send<{ frameTree: Tree }>(
"Page.getFrameTree",
{},
sessionId,
);
const target: CdpTarget = {
tabId: 1,
...(sessionId === rootSession ? {} : { sessionId }),
};
const visit = (tree: Tree, parentFrameId = tree.frame.parentId) => {
frames.push({ frameId: tree.frame.id, parentFrameId, target });
names.set(tree.frame.id, tree.frame.name ?? "");
for (const child of tree.childFrames ?? []) visit(child, tree.frame.id);
};
visit(frameTree);
}
expect(frames).toHaveLength(5);
for (const frame of frames) {
if (!frame.parentFrameId) continue;
const parent = frames.find((item) => item.frameId === frame.parentFrameId)!;
const owner = await send<{ backendNodeId: number }>(
"DOM.getFrameOwner",
{ frameId: frame.frameId },
sessionFor(parent.target),
);
frame.ownerBackendNodeId = owner.backendNodeId;
}
const graph: CdpFrameGraph = { rootFrameId: frames[0].frameId, frames };
const calls: string[] = [];
const cdp: CdpRunner = {
send: (tabId, method, params) => {
calls.push(`${tabId}:${method}`);
return send(method, params, rootSession);
},
sendToTarget: (target, method, params) => {
calls.push(`${target.sessionId ?? target.tabId}:${method}`);
return send(method, params, sessionFor(target));
},
getFrameGraph: async () => graph,
getAttachmentId: () => rootSession,
};
const oracles = new Map<string, Oracle>();
for (const frame of frames) {
const session = sessionFor(frame.target);
const { executionContextId } = await send<{ executionContextId: number }>(
"Page.createIsolatedWorld",
{ frameId: frame.frameId, worldName: "coordinate-oracle" },
session,
);
await expect
.poll(
async () => {
const ready = await send<{ result: { value: boolean } }>(
"Runtime.evaluate",
{
expression: 'document.documentElement.dataset.geometryReady === "true"',
contextId: executionContextId,
returnByValue: true,
},
session,
);
return ready.result.value;
},
{ timeout: 5_000 },
)
.toBe(true);
// Establish scroll after cross-process target creation/layout settles.
const scroll = frame.parentFrameId
? names.get(frame.frameId) === "nested"
? [10, 20]
: [40, 100]
: [80, 240];
await send(
"Runtime.evaluate",
{
expression: `(async () => { scrollTo(${scroll.join(",")}); await new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))); })()`,
contextId: executionContextId,
awaitPromise: true,
},
session,
);
const reply = await send<{ result: { value: Oracle } }>(
"Runtime.evaluate",
{ expression: oracleExpression, contextId: executionContextId, returnByValue: true },
session,
);
oracles.set(frame.frameId, reply.result.value);
}
const facts = await captureObservationFacts(cdp, 1);
expect(facts.issues).toEqual([]);
for (const frame of frames) {
const node = facts.documents
.find((doc) => doc.frame.frameId === frame.frameId)
?.domNodes.find((node) => node.attrs.id === "probe");
expect(node, `missing probe in ${names.get(frame.frameId) || "root"}`).toBeDefined();
const local = oracles.get(frame.frameId)!.probe;
const top = { ...local };
let current = frame;
while (current.parentFrameId) {
const owner = oracles.get(current.parentFrameId)!.owners[names.get(current.frameId)!];
top.x = owner.x + top.x * owner.scale;
top.y = owner.y + top.y * owner.scale;
top.w *= owner.scale;
top.h *= owner.scale;
current = frames.find((item) => item.frameId === current.parentFrameId)!;
}
expect(
node!.rect,
JSON.stringify({
frame: names.get(frame.frameId),
local,
top,
viewport: facts.viewport,
}),
).not.toBeNull();
for (const key of ["x", "y", "w", "h"] as const) {
expect(
Math.abs(node!.localRect![key] - local[key]),
`${names.get(frame.frameId)} local ${key}`,
).toBeLessThan(2);
expect(
Math.abs(node!.rect![key] - top[key]),
`${names.get(frame.frameId)} top ${key}`,
).toBeLessThan(2);
}
}
expect(calls.filter((call) => call.endsWith(":Page.getLayoutMetrics"))).toHaveLength(
sessions.length,
);
const metrics = await send<{
cssVisualViewport: { zoom: number };
visualViewport: { clientWidth: number };
}>("Page.getLayoutMetrics", {}, rootSession);
expect(metrics.cssVisualViewport.zoom).toBeCloseTo(configuration.zoom, 4);
},
);
} finally {
await server.stop();
}
}, 30_000);
});
@@ -25,6 +25,12 @@ export interface UnavailableFrameProjection {
ownerBackendNodeId: number;
}
export interface UnavailableSnapshotCoordinates {
status: "unavailable";
source: CoordinateOwner;
reason: "snapshot-coordinates-unavailable";
}
export type SnapshotProjectionResult =
| { status: "available"; projection: SnapshotProjection }
| UnavailableFrameProjection;
@@ -32,25 +38,74 @@ export type SnapshotProjectionResult =
/** Descendants inherit the failed boundary without attempting another owner read. */
export type FrameProjectionIssue =
| UnavailableFrameProjection
| { status: "blocked"; source: CoordinateOwner; cause: UnavailableFrameProjection };
| UnavailableSnapshotCoordinates
| {
status: "blocked";
source: CoordinateOwner;
cause: UnavailableFrameProjection | UnavailableSnapshotCoordinates;
};
export type FrameProjectionState = SnapshotProjectionResult | FrameProjectionIssue;
/** DOMSnapshot bounds are document-relative CSS pixels, independent of raster DPR. */
export interface SnapshotCoordinates {
layoutUnitsPerCssPixel: number;
scrollCss: { x: number; y: number };
}
/** Snapshot scroll offsets share the raw layout units of its bounds. Only a
* target's root document may fall back to that target's CSS layout viewport. */
export function snapshotCoordinates(
document: { scrollOffsetX?: number; scrollOffsetY?: number },
layoutUnitsPerCssPixel: number | null,
rootScrollCss?: { x?: number; y?: number },
): SnapshotCoordinates | null {
if (
layoutUnitsPerCssPixel === null ||
!Number.isFinite(layoutUnitsPerCssPixel) ||
layoutUnitsPerCssPixel <= 0
)
return null;
const x =
document.scrollOffsetX === undefined
? rootScrollCss?.x
: document.scrollOffsetX / layoutUnitsPerCssPixel;
const y =
document.scrollOffsetY === undefined
? rootScrollCss?.y
: document.scrollOffsetY / layoutUnitsPerCssPixel;
if (x === undefined || y === undefined || !Number.isFinite(x) || !Number.isFinite(y)) return null;
return { layoutUnitsPerCssPixel, scrollCss: { x, y } };
}
/** Convert raw document-relative Blink layout units exactly once, before any
* CSS viewport clipping or frame projection. This scale is not screenshot zoom. */
export function snapshotViewportRect(
bounds: number[],
owner: CoordinateOwner,
scroll: { x: number; y: number },
coordinates: SnapshotCoordinates | null,
): FrameViewportRect | null {
if (bounds.length < 4 || !bounds.slice(0, 4).every(Number.isFinite)) return null;
if (!coordinates || bounds.length < 4 || !bounds.slice(0, 4).every(Number.isFinite)) return null;
const { layoutUnitsPerCssPixel: scale, scrollCss: scroll } = coordinates;
const [x, y, width, height] = bounds;
if (width <= 0 || height <= 0 || !Number.isFinite(scroll.x) || !Number.isFinite(scroll.y)) {
if (
!Number.isFinite(scale) ||
scale <= 0 ||
width <= 0 ||
height <= 0 ||
!Number.isFinite(scroll.x) ||
!Number.isFinite(scroll.y)
) {
return null;
}
return {
space: "frame-viewport-css",
owner,
rect: { x: x - scroll.x, y: y - scroll.y, width, height },
rect: {
x: x / scale - scroll.x,
y: y / scale - scroll.y,
width: width / scale,
height: height / scale,
},
};
}
@@ -18,8 +18,8 @@ import type { CoordinateOwner, CssViewport, SnapshotProjectionResult } from "./c
import { readSnapshotOwnerSizes } from "./snapshot-owner-sizes";
export interface LayoutMetrics {
cssVisualViewport?: { zoom?: number };
visualViewport?: { zoom?: number };
cssVisualViewport?: { zoom?: number; clientWidth?: number; clientHeight?: number };
visualViewport?: { zoom?: number; clientWidth?: number; clientHeight?: number };
cssLayoutViewport?: {
clientWidth?: number;
clientHeight?: number;
@@ -29,6 +29,29 @@ export interface LayoutMetrics {
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
}
/** Chromium emits these paired floating-point dimensions before/after dividing
* by LayoutZoomFactor. Integer layout viewport dimensions lose precision at
* fractional zoom. Neither visualViewport.scale nor .zoom is this conversion.
* See InspectorPageAgent::getLayoutMetrics in Chromium. */
export function snapshotLayoutScale(metrics: LayoutMetrics): number | null {
for (const dimension of ["clientWidth", "clientHeight"] as const) {
const raw = metrics.visualViewport?.[dimension];
const css = metrics.cssVisualViewport?.[dimension];
if (
raw !== undefined &&
css !== undefined &&
Number.isFinite(raw) &&
Number.isFinite(css) &&
raw > 0 &&
css > 0
) {
const scale = raw / css;
if (Number.isFinite(scale) && scale > 0) return scale;
}
}
return null;
}
/** The legacy field is a compatibility fallback, never a source of raster DPR. */
export function cssViewport(metrics: LayoutMetrics): CssViewport {
const value = metrics.cssLayoutViewport ?? metrics.layoutViewport;
+1 -1
View File
@@ -929,7 +929,7 @@ export async function captureVomObservation(
const notices: string[] = [];
if (facts.issues.some((issue) => issue.stage === "geometry"))
notices.push(
"@warning iframe geometry incomplete: some frame content has no top-level coordinates.",
"@warning geometry incomplete: some page or frame content has no top-level coordinates.",
);
const incompleteStages = ["dom", "ax", "forms", "ownership"].filter((stage) =>
facts.issues.some((issue) => issue.stage === stage),
@@ -6,7 +6,7 @@ import { captureVomObservation } from "../../observation";
import type { CdpRunner } from "../../shared";
import { captureObservationFacts, semanticCapture } from "../capture-coordinator";
import { buildSemanticGraph } from "../semantic-graph/build";
import { REQUESTED_STYLES } from "../snapshot";
import { REQUESTED_STYLES, type SnapshotReply } from "../snapshot";
function fixture(
options: {
@@ -76,7 +76,11 @@ function fixture(
};
}
if (method === "Page.getLayoutMetrics")
result = { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
result = {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
if (method === "DOMSnapshot.captureSnapshot") {
snapshots++;
expect((params as { computedStyles: unknown }).computedStyles).toEqual(REQUESTED_STYLES);
@@ -108,6 +112,8 @@ function fixture(
const children = localFrames.filter((f) => f.parentFrameId === frame.frameId);
return {
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: frame.frameId,
nodes: {
backendNodeId: [
@@ -180,6 +186,35 @@ function fixture(
}
describe("captureObservationFacts", () => {
it("normalizes snapshot-owned root scroll instead of mixing in stale CSS scroll", async () => {
const { cdp } = fixture({ frames: [{ frameId: "main", target: { tabId: 4 } }] });
const original = cdp.sendToTarget!;
cdp.sendToTarget = async (target, method, params) => {
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 2000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 999, pageY: 999 },
} as never;
const reply = await original(target, method, params);
if (method === "DOMSnapshot.captureSnapshot") {
const doc = (reply as SnapshotReply).documents![0];
doc.scrollOffsetX = 40;
doc.scrollOffsetY = 200;
doc.layout!.bounds![1] = [200, 800, 240, 80];
}
return reply as never;
};
cdp.send = (tabId, method, params) => cdp.sendToTarget!({ tabId }, method, params);
const facts = await captureObservationFacts(cdp, 4);
expect(facts.documents[0].domNodes.find((node) => node.backendNodeId === 2)).toMatchObject({
localRect: { x: 80, y: 300, w: 120, h: 40 },
rect: { x: 80, y: 300, w: 120, h: 40 },
rendered: true,
});
expect(facts.issues).toEqual([]);
});
it("collects each target once and scopes equal backend IDs", async () => {
const { cdp, logs } = fixture();
const facts = await captureObservationFacts(cdp, 4);
@@ -591,6 +626,8 @@ function childSnapshot(frameId: string, backendNodeId: number) {
strings,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId,
nodes: {
parentIndex: [-1, 0],
@@ -619,7 +656,11 @@ describe("OOPIF capture", () => {
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
const sendToTarget = vi.fn(async (target, method) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 },
};
}
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
@@ -641,7 +682,11 @@ describe("OOPIF capture", () => {
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
@@ -719,7 +764,11 @@ describe("OOPIF capture", () => {
};
const reply = async (_target: unknown, method: string) => {
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 300, clientHeight: 200 },
};
if (method === "DOMSnapshot.captureSnapshot")
return typeof _target === "number" ? childSnapshot("main", 11) : snapshot;
if (method === "DOM.getBoxModel")
@@ -777,7 +826,11 @@ describe("OOPIF capture", () => {
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable");
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
@@ -820,7 +873,7 @@ describe("OOPIF capture", () => {
expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }),
]);
expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual(
expect.objectContaining({ rect: null, localRect: { x: 10, y: 20, w: 100, h: 40 } }),
expect.objectContaining({ rect: null, localRect: null, rendered: true }),
);
});
});
@@ -830,6 +883,8 @@ function siblingCaptureFixture(
beforeReply: (method: string, params: Record<string, unknown>) => Promise<void> = async () => {},
) {
const document = (id: number, owners: number[], childIndexes: number[]) => ({
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: `frame-${id}`,
nodes: {
parentIndex: [-1, ...owners.map(() => 0)],
@@ -861,7 +916,11 @@ function siblingCaptureFixture(
await beforeReply(method, args);
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
if (method === "DOM.getBoxModel")
return { model: { content: [0, 0, 200, 0, 200, 100, 0, 100] } };
if (method === "DOM.resolveNode") return { object: { objectId: String(args.backendNodeId) } };
@@ -1065,7 +1124,11 @@ describe("snapshot document provenance", () => {
(method === "DOMSnapshot.captureSnapshot"
? { ...f.snapshot, documents: [f.snapshot.documents[1]] }
: method === "Page.getLayoutMetrics"
? { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }
? {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 200, clientHeight: 100 },
}
: {}) as T,
};
const facts = await captureObservationFacts(cdp, 4);
@@ -215,7 +215,11 @@ function makeCdp(snapshot: unknown) {
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "Runtime.evaluate") {
return {
@@ -277,6 +281,8 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return hoverTriggerSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -340,6 +346,8 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return twoHoverTriggerSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -388,6 +396,8 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return nestedHoverTriggerSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -733,6 +743,8 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 200 },
};
}
@@ -745,7 +757,7 @@ describe("snapshot capture and hover", () => {
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
});
it("keeps CSS snapshot bounds independent of legacy metrics ratio", async () => {
it("normalizes raw snapshot bounds before viewport clipping", async () => {
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
const i = (s: string) => S.indexOf(s);
const snapshot = {
@@ -780,6 +792,8 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 2000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
layoutViewport: { clientWidth: 2000, clientHeight: 1600 },
};
@@ -797,8 +811,8 @@ describe("snapshot capture and hover", () => {
expect(nodes.find((n) => n.backendNodeId === 12)?.localRect).toEqual({
x: 0,
y: 0,
w: 2000,
h: 1600,
w: 1000,
h: 800,
});
});
@@ -856,6 +870,8 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -946,6 +962,8 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -1035,6 +1053,8 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -1097,6 +1117,8 @@ describe("snapshot capture and hover", () => {
},
},
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "child",
nodes: {
parentIndex: [-1, 0, 1],
@@ -1119,6 +1141,8 @@ describe("snapshot capture and hover", () => {
},
},
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "nested",
nodes: {
parentIndex: [-1, 0],
@@ -1146,6 +1170,8 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -1225,6 +1251,8 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 2000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 100 },
layoutViewport: { clientWidth: 2000, clientHeight: 1600 },
};
@@ -1233,7 +1261,7 @@ describe("snapshot capture and hover", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 - 100);
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 / 2 - 100);
});
it("collectOverlayExcludedBackendIds walks the pierced overlay host subtree", async () => {
@@ -77,7 +77,7 @@ describe("document facts", () => {
);
expect(nodes[0].attrs).toEqual({ "aria-hidden": "true", inert: "" });
expect(nodes[0].layout).toEqual({
boundsSpace: "snapshot-document-css",
boundsSpace: "snapshot-document-layout",
bounds: [10, 20, 120, 40],
styles: {
position: "static",
+2 -2
View File
@@ -6,9 +6,9 @@ import { createCaptureCheckpoint } from "./capture-abort";
import type { CapturedNode } from "./capture-types";
import type { FrameOwnedAxNode } from "./frame-document";
/** Bounds retain their snapshot document CSS units until normalization. */
/** Bounds retain their raw snapshot document layout units until normalization. */
export interface SnapshotLayout {
readonly boundsSpace: "snapshot-document-css";
readonly boundsSpace: "snapshot-document-layout";
bounds?: number[];
styles: Readonly<Record<string, string>>;
}
+53 -20
View File
@@ -4,9 +4,15 @@ import { type GeometryProjection, projectRectToViewport } from "../geometry";
import {
type FrameProjectionState,
projectSnapshotRect,
type SnapshotCoordinates,
snapshotCoordinates,
snapshotViewportRect,
} from "../geometry/coordinate-types";
import { cssViewport, GeometryContext } from "../geometry/frame-context";
import {
GeometryContext,
type LayoutMetrics,
snapshotLayoutScale,
} from "../geometry/frame-context";
import {
createCaptureCheckpoint,
isAbortError as isCaptureAbort,
@@ -26,8 +32,7 @@ export interface FrameContext {
projection: FrameProjectionState | null;
targetProjection?: GeometryProjection | null;
target: CdpTarget;
scrollX: number;
scrollY: number;
coordinates: SnapshotCoordinates | null;
}
export interface NormalizedDocument {
@@ -52,10 +57,11 @@ export async function normalizeDocument(
if (pending) await pending;
}
const node = decoded.nodes[i];
const bounds = node.layout?.bounds ?? [];
const input = snapshotViewportRect(
node.layout?.bounds ?? [],
bounds,
{ target: context.target, frameId: context.frameId },
{ x: context.scrollX, y: context.scrollY },
context.coordinates,
);
const local = input?.rect;
let rect =
@@ -78,7 +84,10 @@ export async function normalizeDocument(
localRect: local ? { x: local.x, y: local.y, w: local.width, h: local.height } : null,
rect: rect ? { x: rect.x, y: rect.y, w: rect.width, h: rect.height } : null,
rendered:
!!local &&
bounds.length >= 4 &&
bounds.slice(0, 4).every(Number.isFinite) &&
bounds[2] > 0 &&
bounds[3] > 0 &&
visibility !== "hidden" &&
visibility !== "collapse" &&
(Number.parseFloat(opacity) || 0) > 0,
@@ -139,17 +148,20 @@ export async function normalizeSnapshot(
}
sources.set(id, doc);
}
let viewport: Viewport = { width: 0, height: 0 };
let scrollX = 0,
scrollY = 0;
let metrics: LayoutMetrics = {};
try {
const measured = cssViewport(await geometry.layoutMetrics(target));
viewport = { width: measured.width, height: measured.height };
scrollX = measured.scrollX;
scrollY = measured.scrollY;
metrics = await geometry.layoutMetrics(target);
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
const viewport: Viewport = {
width: metrics.cssLayoutViewport?.clientWidth ?? 0,
height: metrics.cssLayoutViewport?.clientHeight ?? 0,
};
const layoutUnitsPerCssPixel = snapshotLayoutScale(metrics);
const viewportAvailable = [viewport.width, viewport.height].every(
(size) => Number.isFinite(size) && size > 0,
);
const children = new Map<string, CdpFrame[]>();
const pending: CdpFrame[] = [];
@@ -195,12 +207,24 @@ export async function normalizeSnapshot(
const doc = sources.get(frame.frameId);
if (!doc) continue;
const source = { target, frameId: frame.frameId };
const state: FrameProjectionState | null =
const coordinates = snapshotCoordinates(
doc,
layoutUnitsPerCssPixel,
frame.frameId === rootFrameId
? {
status: "available",
projection: { source, geometry: { sourceClips: [], edges: [], topViewport: viewport } },
}
? { x: metrics.cssLayoutViewport?.pageX, y: metrics.cssLayoutViewport?.pageY }
: undefined,
);
let state: FrameProjectionState | null =
frame.frameId === rootFrameId
? viewportAvailable && coordinates
? {
status: "available",
projection: {
source,
geometry: { sourceClips: [], edges: [], topViewport: viewport },
},
}
: { status: "unavailable", source, reason: "snapshot-coordinates-unavailable" }
: (projections.get(frame.frameId) ?? null);
if (!state && frame.frameId !== rootFrameId)
issues.push({
@@ -209,6 +233,8 @@ export async function normalizeSnapshot(
stage: "ownership",
reason: "frame-ownership-unresolved",
});
if (!coordinates && (!state || state.status === "available"))
state = { status: "unavailable", source, reason: "snapshot-coordinates-unavailable" };
projections.set(frame.frameId, state);
const projection = state?.status === "available" ? state.projection : null;
if (!projection || (target.sessionId && !targetProjection))
@@ -253,6 +279,14 @@ export async function normalizeSnapshot(
projections.set(child.frameId, null);
continue;
}
if (!snapshotCoordinates(sources.get(child.frameId)!, layoutUnitsPerCssPixel)) {
projections.set(child.frameId, {
status: "unavailable",
source: childSource,
reason: "snapshot-coordinates-unavailable",
});
continue;
}
try {
projections.set(
child.frameId,
@@ -281,8 +315,7 @@ export async function normalizeSnapshot(
projection: state,
target,
...(target.sessionId ? { targetProjection } : {}),
scrollX: doc.scrollOffsetX ?? (frame.frameId === rootFrameId ? scrollX : 0),
scrollY: doc.scrollOffsetY ?? (frame.frameId === rootFrameId ? scrollY : 0),
coordinates,
},
signal,
);
+1 -1
View File
@@ -175,7 +175,7 @@ export async function decodeDocument(
li === undefined
? undefined
: {
boundsSpace: "snapshot-document-css" as const,
boundsSpace: "snapshot-document-layout" as const,
bounds: dl?.bounds?.[li],
styles,
};
@@ -0,0 +1,44 @@
# DOMSnapshot layout-unit regression (#195)
Source: https://github.com/Tencent/BrowserSkill/pull/195
DOMSnapshot bounds and document scroll offsets retain Blink layout units. Treating them as CSS
pixels changes both positions and sizes under device scaling/browser zoom, before iframe
projection or viewport clipping even starts.
The fixture contains a scrolled root, a scrolled same-process iframe, an OOPIF served from the
other loopback hostname, and a nested iframe in each. Owners have borders, padding and CSS
transforms. The geometry test compares the production capture's local and top-level rectangles
against independent DOM border boxes, with at most 2 CSS pixels of layout rounding. It also
checks that each target reads layout metrics only once. No credentials or personal tabs are used.
Run the geometric assertions with Node 22+ and a local Chrome executable:
```sh
BSK_GEOMETRY_CHROME=/path/to/chrome pnpm --filter @browser-skill/extension exec vitest run \
src/tools/__tests__/snapshot-coordinates.browser.test.ts
```
The test launches and cleans up its own headless Chrome profiles. It explicitly verifies the
requested browser zoom and OOPIF topology, so unsupported setups fail rather than silently
testing a different configuration. It is skipped in ordinary unit runs unless the executable
is specified. No browser download or new package dependency is required.
The corpus smoke case checks fixture readiness and observable semantics through the real CLI:
```sh
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --case snapshot-coordinates --bsk ./target/debug/bsk
```
Smoke alone does **not** certify coordinate accuracy; CLI observations do not expose raw boxes.
Use the geometric test above for the regression's numeric assertions.
## Separate existing boundary
The OOPIF root uses non-occupying scrollbars in the unit-conversion regression. Root and
same-process frames retain normal scrollbars. Append `&classic-scrollbars` to the fixture URL
to reproduce the separate existing OOPIF projection issue: `targetProjection()` maps the CSS
layout viewport (which excludes occupying scrollbars) onto the entire owner content quad.
This inflates the projected coordinates when scrollbars occupy space. Main already has that
mapping; fixing it also changes shared live geometry and is outside this snapshot-unit fix.
The test does not relax its numeric tolerance to absorb that error.
@@ -0,0 +1,99 @@
import { spawn } from "node:child_process";
import { once } from "node:events";
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
// This regression owns its browser/profile. It never attaches to a user's Chrome.
export async function withChrome({ executable, deviceScale, zoom }, run) {
const profile = await mkdtemp(join(tmpdir(), "bsk-snapshot-coordinates-"));
let chrome;
let socket;
const pending = new Map();
try {
await mkdir(join(profile, "Default"));
await writeFile(
join(profile, "Default", "Preferences"),
JSON.stringify({
partition: { default_zoom_level: { x: Math.log(zoom) / Math.log(1.2) } },
}),
);
chrome = spawn(
executable,
[
"--headless=new",
"--no-first-run",
"--no-default-browser-check",
"--disable-extensions",
"--site-per-process",
"--remote-debugging-port=0",
"--window-size=1600,1200",
`--force-device-scale-factor=${deviceScale}`,
`--user-data-dir=${profile}`,
"about:blank",
],
{ stdio: ["ignore", "ignore", "pipe"] },
);
const endpoint = await new Promise((resolve, reject) => {
let output = "";
const timeout = setTimeout(
() => reject(new Error(`Chrome startup timed out: ${output}`)),
15_000,
);
const fail = (error) => {
clearTimeout(timeout);
reject(error);
};
chrome.once("error", fail);
chrome.once("exit", (code) => fail(new Error(`Chrome exited (${code}): ${output}`)));
chrome.stderr.on("data", (chunk) => {
output += chunk;
const match = output.match(/DevTools listening on (ws:\/\/\S+)/);
if (match) {
clearTimeout(timeout);
resolve(match[1]);
}
});
});
socket = new WebSocket(endpoint);
await new Promise((resolve, reject) => {
socket.addEventListener("open", resolve, { once: true });
socket.addEventListener("error", reject, { once: true });
});
socket.addEventListener("message", ({ data }) => {
const reply = JSON.parse(data);
const request = pending.get(reply.id);
if (!request) return;
clearTimeout(request.timeout);
pending.delete(reply.id);
if (reply.error) request.reject(new Error(JSON.stringify(reply.error)));
else request.resolve(reply.result);
});
let sequence = 0;
const send = (method, params = {}, sessionId) =>
new Promise((resolve, reject) => {
const id = ++sequence;
const timeout = setTimeout(() => {
pending.delete(id);
reject(new Error(`CDP timed out: ${method}`));
}, 10_000);
pending.set(id, { resolve, reject, timeout });
socket.send(JSON.stringify({ id, method, params, sessionId }));
});
await run(send);
} finally {
for (const request of pending.values()) {
clearTimeout(request.timeout);
request.reject(new Error("regression browser closed"));
}
socket?.close();
if (chrome && chrome.exitCode === null && chrome.signalCode === null) {
const exited = once(chrome, "exit");
const timeout = setTimeout(() => chrome.kill("SIGKILL"), 5_000);
chrome.kill();
await exited;
clearTimeout(timeout);
}
await rm(profile, { recursive: true, force: true });
}
}
@@ -0,0 +1,33 @@
{
"$schema": "../../../schemas/case.schema.json",
"schemaVersion": 1,
"id": "snapshot-coordinates",
"title": "Scrolled and nested frame coordinate fixture",
"suite": "regression",
"tags": ["geometry", "iframe", "scroll"],
"fixture": { "startPath": "/snapshot-coordinates" },
"prompts": {
"en": "Open {url}, observe the page and its frames, report the GEOMETRY marker, then close the session.",
"zh-CN": "打开 {url},观察页面及其 iframe,报告 GEOMETRY 标记,然后关闭会话。"
},
"coverage": ["session.start", "session.stop", "page.navigate", "inspect.observe"],
"assertions": {
"site": [
{
"label": "root fixture scrolled and settled",
"type": "geometry.ready",
"where": { "data.root": true },
"minCount": 1
}
],
"response": [{ "label": "observed the fixture marker", "includes": "GEOMETRY-195" }],
"adapter": [{ "label": "browser session was closed", "key": "sessionStopped" }]
},
"smoke": {
"steps": [
{ "action": "navigate" },
{ "action": "wait-site-event", "type": "geometry.ready", "where": { "root": true } },
{ "action": "observe" }
]
}
}
@@ -0,0 +1,52 @@
import { page, withRun } from "../../../lib/fixtures.mjs";
export default {
id: "snapshot-coordinates",
routes: ["/snapshot-coordinates", "/snapshot-coordinates/frame", "/snapshot-coordinates/nested"],
render({ pathname, runId, query }) {
const root = pathname === "/snapshot-coordinates";
const nested = pathname.endsWith("/nested");
return page({
title: "Snapshot coordinate regression",
body: `
<style>
main { margin: 0; width: auto; }
body { width: 2000px; height: 2400px; }
html { overflow-anchor: none; }
${query.get("scrollbars") === "none" ? "html { scrollbar-width: none; }" : ""}
button { position: absolute; box-sizing: border-box; padding: 0; border: 2px solid black; border-radius: 0; }
#probe { left: ${root ? 180 : 100}px; top: ${root ? 360 : 160}px; width: 120px; height: 40px; }
iframe { position: absolute; box-sizing: content-box; width: 320px; height: 220px;
border: 6px solid black; padding: 8px; transform: scale(1.25); transform-origin: 0 0; }
#same { left: 340px; top: 400px; }
#cross { left: 820px; top: 400px; }
#nested { left: 150px; top: 210px; width: 130px; height: 70px; border-width: 2px; padding: 4px; transform: scale(0.8); }
${nested ? "#probe { left: 30px; top: 40px; width: 60px; height: 20px; }" : ""}
</style>
<button id="probe" data-geometry-probe>GEOMETRY-195</button>
${root ? `<iframe id="same" name="same" title="same-process frame" src="${withRun("/snapshot-coordinates/frame", runId)}"></iframe><iframe id="cross" name="cross" title="cross-site frame"></iframe>` : ""}
${!root && !nested ? `<iframe id="nested" name="nested" title="nested frame" src="${withRun("/snapshot-coordinates/nested", runId)}"></iframe>` : ""}
`,
script: `
history.scrollRestoration = "manual";
const cross = document.querySelector("#cross");
if (cross) {
const url = new URL(${JSON.stringify(withRun("/snapshot-coordinates/frame", runId))}, location.href);
url.hostname = location.hostname === "127.0.0.1" ? "localhost" : "127.0.0.1";
// Isolate unit conversion from the pre-existing OOPIF scrollbar-width
// projection issue. ?classic-scrollbars retains that separate reproduction.
if (!new URLSearchParams(location.search).has("classic-scrollbars")) url.searchParams.set("scrollbars", "none");
cross.src = url.href;
}
window.addEventListener("load", () => {
scrollTo(${root ? "80, 240" : nested ? "10, 20" : "40, 100"});
requestAnimationFrame(() => requestAnimationFrame(() => {
window.geometryReady = true;
document.documentElement.dataset.geometryReady = "true";
browserEval.send("geometry.ready", { root: ${root} });
}));
});
`,
});
},
};
+9 -4
View File
@@ -41,17 +41,22 @@ test("case manifests are discovered, ordered, and grouped into suites", () => {
"diagnostics",
"mobile-emulation",
"generated-form",
"snapshot-coordinates",
],
);
assert.deepEqual(repositorySummary(cases, fixtureRegistry).suites, { core: 6, matrix: 1 });
assert.deepEqual(repositorySummary(cases, fixtureRegistry).suites, {
core: 6,
matrix: 1,
regression: 1,
});
});
test("repository validation links every case to a fixture and valid workflow evidence", () => {
assert.deepEqual(validateRepositoryCases(cases, fixtureRegistry), []);
const summary = repositorySummary(cases, fixtureRegistry);
assert.equal(summary.cases, 7);
assert.equal(summary.fixtureModules, 8);
assert.equal(summary.fixtureRoutes, 12);
assert.equal(summary.cases, 8);
assert.equal(summary.fixtureModules, 9);
assert.equal(summary.fixtureRoutes, 15);
});
test("manifest validation rejects unknown operations and incomplete workflow steps", () => {