Compare commits

..
Author SHA1 Message Date
Ljy-0827 ad4c1a92cf perf(observation): collect frame AX trees with bounded concurrency 2026-09-09 04:09:16 +08:00
Ljy-0827 b75ea7f7ce refactor(observation): separate capture coordination from document consistency 2026-09-09 04:09:16 +08:00
Ljy-0827 17f7587c60 refactor(observation): integrate capture orchestration and document consistency 2026-09-09 04:09:16 +08:00
Ljy-0827 fe7ddbd082 fix(observation): validate document consistency during capture 2026-09-09 04:09:16 +08:00
Ljy-0827 3bb84d1c5b perf(observation): reduce document processing scheduling overhead 2026-09-09 04:09:15 +08:00
Ljy-0827 7f8b51a724 refactor(observation): isolate document processing from capture orchestration 2026-09-09 04:09:15 +08:00
Ljy-0827 2fd1b9d8f1 fix(observation): preserve snapshot document provenance and URL metadata 2026-09-09 04:09:15 +08:00
Ljy-0827 ef27aa9832 refactor(observation): defer visual facts until visual discovery 2026-09-09 04:09:15 +08:00
Ljy-0827 e67e393434 fix(observation): await capture task cleanup before propagating cancellation 2026-09-09 04:09:15 +08:00
Ljy-0827 7bccb84d5f refactor(observation): organize capture around document facts 2026-09-09 04:09:15 +08:00
Ljy-0827 f6d129b12e perf(geometry): batch iframe owner size measurements 2026-09-09 04:09:15 +08:00
Ljy-0827 fc39bc4f8f fix(vom): surface incomplete iframe geometry in observations 2026-09-09 04:09:15 +08:00
Ljy-0827 05bee25b2f perf(observation): measure sibling iframe geometry with bounded concurrency 2026-09-09 04:09:15 +08:00
Ljy-0827 2778ab1eee fix(geometry): unify frame projection and screenshot coordinates 2026-09-09 04:09:15 +08:00
23 changed files with 80 additions and 905 deletions

No files matched your search

+1 -18
View File
@@ -5,24 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/).
Starting from 0.2.0, CLI / Extension / DSH Plugin share the same version number.
## [0.2.1] - 2026-09-09
### Changed
- DSH Plugin: clarified installation, configuration, profile restart, and upgrade instructions
### Fixed
- CLI: reliable Windows self-updates, including executable replacement and daemon restart
- CLI: Windows named-pipe connection continuity and stale daemon process detection
- Windows installer: PowerShell 5.1 compatibility, path quoting, and staged executable replacement
- CLI / DSH Plugin: cancellation propagation and upload/download cleanup when the parent process exits
- Extension: tab ownership and session cleanup when agent tabs fail to open or borrowed tabs are returned or closed
- Extension: navigation recovery when other extensions restrict Chrome DevTools Protocol access
- Extension: background input filling, automatic append behavior, and verification of field values before reporting success
- Extension: match observed form state to the correct element after page changes
- Extension: UUID generation fallback for recording and overlays on insecure origins
- Browser locale matching for regional language variants
- DSH Plugin: observation state synchronization, thumbnail capture lifecycle, and stale preview cleanup
- DSH Plugin: removed the obsolete client runtime host dependency
## [Unreleased]
## [0.2.0] - 2026-09-02
Generated
+2 -2
View File
@@ -134,7 +134,7 @@ dependencies = [
[[package]]
name = "bsk"
version = "0.2.1"
version = "0.2.0"
dependencies = [
"anyhow",
"base64",
@@ -169,7 +169,7 @@ dependencies = [
[[package]]
name = "bsk-protocol"
version = "0.2.1"
version = "0.2.0"
dependencies = [
"schemars",
"semver",
+2 -2
View File
@@ -7,7 +7,7 @@ members = [
default-members = ["crates/bsk-cli"]
[workspace.package]
version = "0.2.1"
version = "0.2.0"
edition = "2024"
rust-version = "1.85"
license = "MIT"
@@ -43,7 +43,7 @@ rand = "0.8"
uuid = { version = "1", features = ["v4", "serde"] }
# Workspace crates (version required for `cargo publish` of dependents)
bsk-protocol = { version = "0.2.1", path = "crates/bsk-protocol" }
bsk-protocol = { version = "0.2.0", path = "crates/bsk-protocol" }
[profile.release]
strip = "symbols"
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@browser-skill/extension",
"version": "0.2.1",
"version": "0.2.0",
"private": true,
"type": "module",
"scripts": {
@@ -5,10 +5,9 @@ import { rectPolygon } from "../geometry";
import {
projectSnapshotRect,
screenshotPageRect,
snapshotCoordinates,
snapshotViewportRect,
} from "../geometry/coordinate-types";
import { GeometryContext, snapshotLayoutScale } from "../geometry/frame-context";
import { GeometryContext } from "../geometry/frame-context";
import type { CdpRunner } from "../shared";
const target = { tabId: 4 };
@@ -23,11 +22,7 @@ const graph: CdpFrameGraph = {
function driver() {
const send = vi.fn(async (_tab: number, method: string): Promise<object> => {
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 1000 },
};
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 1000 } };
if (method === "DOM.getBoxModel")
return { model: { content: [52.5, 612.5, 427.5, 612.5, 427.5, 862.5, 52.5, 862.5] } };
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
@@ -40,63 +35,6 @@ function driver() {
}
describe("measurement geometry context", () => {
it.each([0.8, 1, 1.25, 2])("normalizes snapshot layout units at scale %s", (scale) => {
const measured = snapshotLayoutScale({
visualViewport: { clientWidth: 997.25, zoom: 1.1 },
cssVisualViewport: { clientWidth: 997.25 / scale, zoom: 1.1 },
// Rounded layout dimensions must not supply the scale.
layoutViewport: { clientWidth: 997 },
cssLayoutViewport: { clientWidth: Math.floor(997.25 / scale) },
});
expect(measured).toBeCloseTo(scale, 12);
const coordinates = snapshotCoordinates(
{ scrollOffsetX: 80 * scale, scrollOffsetY: 200 * scale },
measured,
{ x: 999, y: 999 },
);
expect(
snapshotViewportRect(
[160 * scale, 320 * scale, 120 * scale, 40 * scale],
{ target },
coordinates,
)?.rect,
).toEqual({ x: 80, y: 120, width: 120, height: 40 });
});
it("uses CSS root scroll only for missing snapshot offsets, without scaling it twice", () => {
const coordinates = snapshotCoordinates({ scrollOffsetX: 80 }, 2, { x: 999, y: 100 });
expect(snapshotViewportRect([200, 800, 240, 80], { target }, coordinates)?.rect).toEqual({
x: 60,
y: 300,
width: 120,
height: 40,
});
expect(snapshotCoordinates({ scrollOffsetX: 0 }, 2)).toBeNull();
expect(
snapshotCoordinates({ scrollOffsetX: NaN, scrollOffsetY: 0 }, 2, { x: 0, y: 0 }),
).toBeNull();
});
it("does not infer snapshot units from missing, invalid, or CSS-only metrics", () => {
expect(snapshotLayoutScale({ cssLayoutViewport: { clientWidth: 800 } })).toBeNull();
for (const raw of [0, -1, NaN, Infinity]) {
expect(
snapshotLayoutScale({
visualViewport: { clientWidth: raw },
cssVisualViewport: { clientWidth: 800 },
}),
).toBeNull();
}
expect(
snapshotLayoutScale({
visualViewport: { clientWidth: 0, clientHeight: 600 },
cssVisualViewport: { clientWidth: 0, clientHeight: 300 },
}),
).toBe(2);
expect(snapshotCoordinates({ scrollOffsetX: 0, scrollOffsetY: 0 }, null)).toBeNull();
expect(snapshotViewportRect([10, 20, 120, 40], { target }, null)).toBeNull();
});
it("shares in-flight frame measurements, but never shares them with the next context", async () => {
const cdp = driver();
const context = new GeometryContext(cdp, 4);
@@ -123,20 +61,14 @@ describe("measurement geometry context", () => {
height: 1000,
});
if (projection.status !== "available") throw new Error("expected available projection");
const input = snapshotViewportRect([17, 23, 120, 40], source, {
layoutUnitsPerCssPixel: 1,
scrollCss: { x: 0, y: 0 },
});
const input = snapshotViewportRect([17, 23, 120, 40], source, { x: 0, y: 0 });
expect(projectSnapshotRect(input!, projection.projection)).toEqual({
x: 73.75,
y: 641.25,
width: 150,
height: 50,
});
const scrolled = snapshotViewportRect([17, 23, 120, 40], source, {
layoutUnitsPerCssPixel: 1,
scrollCss: { x: 0, y: 10 },
});
const scrolled = snapshotViewportRect([17, 23, 120, 40], source, { x: 0, y: 10 });
expect(projectSnapshotRect(scrolled!, projection.projection)?.y).toBe(628.75);
await context.snapshotProjection(source, 10, [], { width: 1000, height: 1000 });
expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.resolveNode")).toHaveLength(
@@ -241,7 +173,7 @@ describe("measurement geometry context", () => {
const input = snapshotViewportRect(
[30, 50, 120, 40],
{ target, frameId: "main" },
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: 10, y: 20 } },
{ x: 10, y: 20 },
)!;
const projection = {
source: { target, frameId: "main" },
@@ -262,20 +194,8 @@ describe("measurement geometry context", () => {
source: { target: { tabId: 5 }, frameId: "main" },
}),
).toBeNull();
expect(
snapshotViewportRect(
[0, 0, NaN, 40],
{ target },
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: 0, y: 0 } },
),
).toBeNull();
expect(
snapshotViewportRect(
[0, 0, 10, 40],
{ target },
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: Infinity, y: 0 } },
),
).toBeNull();
expect(snapshotViewportRect([0, 0, NaN, 40], { target }, { x: 0, y: 0 })).toBeNull();
expect(snapshotViewportRect([0, 0, 10, 40], { target }, { x: Infinity, y: 0 })).toBeNull();
});
it("does not turn a border quad or a failed owner read into a content projection", async () => {
@@ -308,7 +228,7 @@ describe("measurement geometry context", () => {
const input = snapshotViewportRect(
[0, 0, 300, 200],
{ target, frameId: "nested" },
{ layoutUnitsPerCssPixel: 1, scrollCss: { x: 0, y: 0 } },
{ x: 0, y: 0 },
);
expect(projectSnapshotRect(input!, projection.projection)).toEqual({
x: 60,
@@ -358,11 +278,7 @@ describe("measurement geometry context", () => {
peak = Math.max(peak, active);
await new Promise<void>((resolve) => release.push(resolve));
active--;
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 100, clientHeight: 100 },
};
return { cssLayoutViewport: { clientWidth: 100, clientHeight: 100 } };
});
const controller = new AbortController();
const context = new GeometryContext(
@@ -93,11 +93,7 @@ function makeFakeCdp(handlers: Record<string, (params?: object) => unknown>) {
sent.push({ method, params });
const handler = handlers[method];
if (!handler && method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 },
};
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
}
if (!handler) throw new Error(`unexpected CDP call ${method}`);
return handler(params);
@@ -2587,11 +2583,7 @@ describe("handleSnapshot", () => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes };
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") throw new Error("snapshot unsupported");
@@ -2678,11 +2670,7 @@ describe("handleSnapshot", () => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
@@ -2690,8 +2678,6 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "root",
nodes: {
parentIndex: [-1, 0],
@@ -2776,11 +2762,7 @@ describe("handleSnapshot", () => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
@@ -2788,8 +2770,6 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "root",
nodes: {
parentIndex: [-1, 0],
@@ -2877,11 +2857,7 @@ describe("handleSnapshot", () => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
@@ -2889,8 +2865,6 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "root",
nodes: {
parentIndex: [-1, 0],
@@ -3049,11 +3023,7 @@ describe("handleSnapshot", () => {
};
}
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
}
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
@@ -3160,8 +3130,6 @@ describe("handleSnapshot", () => {
strings: S,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 2],
@@ -3220,8 +3188,6 @@ describe("handleSnapshot", () => {
strings: S,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 1, 1, 1],
@@ -3272,8 +3238,6 @@ describe("handleSnapshot", () => {
}
const VP_METRICS = {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
@@ -3287,8 +3251,6 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "main",
nodes: {
parentIndex: [-1, 0, 1],
@@ -3309,8 +3271,6 @@ describe("handleSnapshot", () => {
},
},
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "child",
nodes: {
parentIndex: [-1, 0, 1],
@@ -3381,11 +3341,7 @@ describe("handleSnapshot", () => {
});
const sendToTarget = vi.fn(async (_target, method: string) => {
if (method === "Page.getLayoutMetrics" && ownerGeometry !== "unavailable")
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 400, clientHeight: 300 },
};
return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } };
if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot")
return { ...snapshot, documents: [snapshot.documents[1]] };
@@ -3428,8 +3384,6 @@ describe("handleSnapshot", () => {
strings: ["body", "a", "href", "//outside.test/docs", "https://page.test/start"],
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "main",
...(hasSnapshotUrl ? { documentURL: 4 } : {}),
nodes: {
@@ -3551,8 +3505,6 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 1, 3],
@@ -3696,14 +3648,14 @@ describe("handleSnapshot", () => {
maxTokens,
});
expect(result.text.match(/@warning/g)).toHaveLength(1);
expect(result.text).toContain("geometry incomplete");
expect(result.text).toContain("iframe geometry incomplete");
expect(result.text.startsWith("@vom 1\n")).toBe(true);
if (maxTokens === undefined) {
expect(result.truncated).toBe(false);
expect(result.text).toContain("Frame action");
expect(result.matchNodes.find((node) => node.backendNodeId === 22)).toMatchObject({
rect: null,
localRect: null,
localRect: { x: 20, y: 30, w: 120, h: 40 },
});
}
if (maxTokens === 100) expect(Math.ceil(result.text.length / 4)).toBeLessThanOrEqual(maxTokens);
@@ -3792,7 +3744,7 @@ describe("handleSnapshot", () => {
expect(result.matchNodes.find((node) => node.backendNodeId === 22)).toMatchObject({
frameId: "child",
tag: "button",
localRect: null,
localRect: { x: 20, y: 30, w: 120, h: 40 },
});
expect(result.refs.find((ref) => ref.backendNodeId === 22)).toMatchObject({
frameId: "child",
@@ -4136,8 +4088,6 @@ describe("handleSnapshot", () => {
strings: S,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 0, 3, 4, 5],
@@ -1,233 +0,0 @@
// @vitest-environment node
import { describe, expect, it } from "vitest";
import type { CdpFrame, CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
import type { CdpRunner } from "../shared";
import { captureObservationFacts } from "../vom/capture-coordinator";
type Send = <T = Record<string, unknown>>(
method: string,
params?: object,
sessionId?: string,
) => Promise<T>;
type Tree = { frame: { id: string; parentId?: string; name?: string }; childFrames?: Tree[] };
type Rect = { x: number; y: number; w: number; h: number };
type Oracle = { probe: Rect; owners: Record<string, { x: number; y: number; scale: number }> };
// The independent oracle uses DOM border boxes and this fixture's axis-aligned
// iframe transforms. No production snapshot conversion/projection builds expectations.
const oracleExpression = `(() => {
const box = document.querySelector('#probe').getBoundingClientRect();
const owners = {};
for (const frame of document.querySelectorAll('iframe')) {
const rect = frame.getBoundingClientRect();
const style = getComputedStyle(frame);
const scale = rect.width / frame.offsetWidth;
owners[frame.name] = {
x: rect.x + (frame.clientLeft + parseFloat(style.paddingLeft)) * scale,
y: rect.y + (frame.clientTop + parseFloat(style.paddingTop)) * scale,
scale,
};
}
return { probe: { x: box.x, y: box.y, w: box.width, h: box.height }, owners };
})()`;
describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate contract", () => {
it.each([
{ deviceScale: 1, zoom: 1 },
{ deviceScale: 0.8, zoom: 1 },
{ deviceScale: 1, zoom: 1.25 },
{ deviceScale: 2, zoom: 1 },
{ deviceScale: 2, zoom: 0.8 },
])("matches DOM geometry at device scale $deviceScale and zoom $zoom", async (configuration) => {
const evalRoot = new URL("../../../../../evals/browser/", import.meta.url);
const { createEvalServer } = await import(new URL("lib/server.mjs", evalRoot).href);
const { withChrome } = await import(
new URL("cases/regression/snapshot-coordinates/chrome.mjs", evalRoot).href
);
const server = createEvalServer();
const { baseUrl } = await server.start();
try {
await withChrome(
{ executable: process.env.BSK_GEOMETRY_CHROME, ...configuration },
async (send: Send) => {
const { targetId } = await send<{ targetId: string }>("Target.createTarget", {
url: "about:blank",
});
const { sessionId: rootSession } = await send<{ sessionId: string }>(
"Target.attachToTarget",
{ targetId, flatten: true },
);
await send(
"Page.navigate",
{ url: `${baseUrl}/snapshot-coordinates?run=coordinates` },
rootSession,
);
await expect
.poll(
() =>
server
.snapshot("coordinates")
.events.some(
(event: { type: string; data: { root?: boolean } }) =>
event.type === "geometry.ready" && event.data.root,
),
{ timeout: 10_000 },
)
.toBe(true);
const targets = await send<{ targetInfos: { targetId: string; type: string }[] }>(
"Target.getTargets",
);
const sessions = [rootSession];
for (const target of targets.targetInfos.filter((target) => target.type === "iframe")) {
const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", {
targetId: target.targetId,
flatten: true,
});
sessions.push(sessionId);
}
expect(sessions.length).toBe(2); // The cross-site fixture must actually be an OOPIF.
const frames: CdpFrame[] = [];
const names = new Map<string, string>();
const sessionFor = (target: CdpTarget) => target.sessionId ?? rootSession;
for (const sessionId of sessions) {
const { frameTree } = await send<{ frameTree: Tree }>(
"Page.getFrameTree",
{},
sessionId,
);
const target: CdpTarget = {
tabId: 1,
...(sessionId === rootSession ? {} : { sessionId }),
};
const visit = (tree: Tree, parentFrameId = tree.frame.parentId) => {
frames.push({ frameId: tree.frame.id, parentFrameId, target });
names.set(tree.frame.id, tree.frame.name ?? "");
for (const child of tree.childFrames ?? []) visit(child, tree.frame.id);
};
visit(frameTree);
}
expect(frames).toHaveLength(5);
for (const frame of frames) {
if (!frame.parentFrameId) continue;
const parent = frames.find((item) => item.frameId === frame.parentFrameId)!;
const owner = await send<{ backendNodeId: number }>(
"DOM.getFrameOwner",
{ frameId: frame.frameId },
sessionFor(parent.target),
);
frame.ownerBackendNodeId = owner.backendNodeId;
}
const graph: CdpFrameGraph = { rootFrameId: frames[0].frameId, frames };
const calls: string[] = [];
const cdp: CdpRunner = {
send: (tabId, method, params) => {
calls.push(`${tabId}:${method}`);
return send(method, params, rootSession);
},
sendToTarget: (target, method, params) => {
calls.push(`${target.sessionId ?? target.tabId}:${method}`);
return send(method, params, sessionFor(target));
},
getFrameGraph: async () => graph,
};
const oracles = new Map<string, Oracle>();
for (const frame of frames) {
const session = sessionFor(frame.target);
const { executionContextId } = await send<{ executionContextId: number }>(
"Page.createIsolatedWorld",
{ frameId: frame.frameId, worldName: "coordinate-oracle" },
session,
);
await expect
.poll(
async () => {
const ready = await send<{ result: { value: boolean } }>(
"Runtime.evaluate",
{
expression: 'document.documentElement.dataset.geometryReady === "true"',
contextId: executionContextId,
returnByValue: true,
},
session,
);
return ready.result.value;
},
{ timeout: 5_000 },
)
.toBe(true);
// Establish scroll after cross-process target creation/layout settles.
const scroll = frame.parentFrameId
? names.get(frame.frameId) === "nested"
? [10, 20]
: [40, 100]
: [80, 240];
await send(
"Runtime.evaluate",
{
expression: `(async () => { scrollTo(${scroll.join(",")}); await new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))); })()`,
contextId: executionContextId,
awaitPromise: true,
},
session,
);
const reply = await send<{ result: { value: Oracle } }>(
"Runtime.evaluate",
{ expression: oracleExpression, contextId: executionContextId, returnByValue: true },
session,
);
oracles.set(frame.frameId, reply.result.value);
}
const facts = await captureObservationFacts(cdp, 1);
expect(facts.issues).toEqual([]);
for (const frame of frames) {
const node = facts.documents
.find((doc) => doc.frame.frameId === frame.frameId)
?.domNodes.find((node) => node.attrs.id === "probe");
expect(node, `missing probe in ${names.get(frame.frameId) || "root"}`).toBeDefined();
const local = oracles.get(frame.frameId)!.probe;
const top = { ...local };
let current = frame;
while (current.parentFrameId) {
const owner = oracles.get(current.parentFrameId)!.owners[names.get(current.frameId)!];
top.x = owner.x + top.x * owner.scale;
top.y = owner.y + top.y * owner.scale;
top.w *= owner.scale;
top.h *= owner.scale;
current = frames.find((item) => item.frameId === current.parentFrameId)!;
}
expect(
node!.rect,
JSON.stringify({
frame: names.get(frame.frameId),
local,
top,
viewport: facts.viewport,
}),
).not.toBeNull();
for (const key of ["x", "y", "w", "h"] as const) {
expect(
Math.abs(node!.localRect![key] - local[key]),
`${names.get(frame.frameId)} local ${key}`,
).toBeLessThan(2);
expect(
Math.abs(node!.rect![key] - top[key]),
`${names.get(frame.frameId)} top ${key}`,
).toBeLessThan(2);
}
}
expect(calls.filter((call) => call.endsWith(":Page.getLayoutMetrics"))).toHaveLength(
sessions.length,
);
const metrics = await send<{
cssVisualViewport: { zoom: number };
visualViewport: { clientWidth: number };
}>("Page.getLayoutMetrics", {}, rootSession);
expect(metrics.cssVisualViewport.zoom).toBeCloseTo(configuration.zoom, 4);
},
);
} finally {
await server.stop();
}
}, 30_000);
});
@@ -25,12 +25,6 @@ export interface UnavailableFrameProjection {
ownerBackendNodeId: number;
}
export interface UnavailableSnapshotCoordinates {
status: "unavailable";
source: CoordinateOwner;
reason: "snapshot-coordinates-unavailable";
}
export type SnapshotProjectionResult =
| { status: "available"; projection: SnapshotProjection }
| UnavailableFrameProjection;
@@ -38,74 +32,25 @@ export type SnapshotProjectionResult =
/** Descendants inherit the failed boundary without attempting another owner read. */
export type FrameProjectionIssue =
| UnavailableFrameProjection
| UnavailableSnapshotCoordinates
| {
status: "blocked";
source: CoordinateOwner;
cause: UnavailableFrameProjection | UnavailableSnapshotCoordinates;
};
| { status: "blocked"; source: CoordinateOwner; cause: UnavailableFrameProjection };
export type FrameProjectionState = SnapshotProjectionResult | FrameProjectionIssue;
export interface SnapshotCoordinates {
layoutUnitsPerCssPixel: number;
scrollCss: { x: number; y: number };
}
/** Snapshot scroll offsets share the raw layout units of its bounds. Only a
* target's root document may fall back to that target's CSS layout viewport. */
export function snapshotCoordinates(
document: { scrollOffsetX?: number; scrollOffsetY?: number },
layoutUnitsPerCssPixel: number | null,
rootScrollCss?: { x?: number; y?: number },
): SnapshotCoordinates | null {
if (
layoutUnitsPerCssPixel === null ||
!Number.isFinite(layoutUnitsPerCssPixel) ||
layoutUnitsPerCssPixel <= 0
)
return null;
const x =
document.scrollOffsetX === undefined
? rootScrollCss?.x
: document.scrollOffsetX / layoutUnitsPerCssPixel;
const y =
document.scrollOffsetY === undefined
? rootScrollCss?.y
: document.scrollOffsetY / layoutUnitsPerCssPixel;
if (x === undefined || y === undefined || !Number.isFinite(x) || !Number.isFinite(y)) return null;
return { layoutUnitsPerCssPixel, scrollCss: { x, y } };
}
/** Convert raw document-relative Blink layout units exactly once, before any
* CSS viewport clipping or frame projection. This scale is not screenshot zoom. */
/** DOMSnapshot bounds are document-relative CSS pixels, independent of raster DPR. */
export function snapshotViewportRect(
bounds: number[],
owner: CoordinateOwner,
coordinates: SnapshotCoordinates | null,
scroll: { x: number; y: number },
): FrameViewportRect | null {
if (!coordinates || bounds.length < 4 || !bounds.slice(0, 4).every(Number.isFinite)) return null;
const { layoutUnitsPerCssPixel: scale, scrollCss: scroll } = coordinates;
if (bounds.length < 4 || !bounds.slice(0, 4).every(Number.isFinite)) return null;
const [x, y, width, height] = bounds;
if (
!Number.isFinite(scale) ||
scale <= 0 ||
width <= 0 ||
height <= 0 ||
!Number.isFinite(scroll.x) ||
!Number.isFinite(scroll.y)
) {
if (width <= 0 || height <= 0 || !Number.isFinite(scroll.x) || !Number.isFinite(scroll.y)) {
return null;
}
return {
space: "frame-viewport-css",
owner,
rect: {
x: x / scale - scroll.x,
y: y / scale - scroll.y,
width: width / scale,
height: height / scale,
},
rect: { x: x - scroll.x, y: y - scroll.y, width, height },
};
}
@@ -18,8 +18,8 @@ import type { CoordinateOwner, CssViewport, SnapshotProjectionResult } from "./c
import { readSnapshotOwnerSizes } from "./snapshot-owner-sizes";
export interface LayoutMetrics {
cssVisualViewport?: { zoom?: number; clientWidth?: number; clientHeight?: number };
visualViewport?: { zoom?: number; clientWidth?: number; clientHeight?: number };
cssVisualViewport?: { zoom?: number };
visualViewport?: { zoom?: number };
cssLayoutViewport?: {
clientWidth?: number;
clientHeight?: number;
@@ -29,29 +29,6 @@ export interface LayoutMetrics {
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
}
/** Chromium emits these paired floating-point dimensions before/after dividing
* by LayoutZoomFactor. Integer layout viewport dimensions lose precision at
* fractional zoom. Neither visualViewport.scale nor .zoom is this conversion.
* See InspectorPageAgent::getLayoutMetrics in Chromium. */
export function snapshotLayoutScale(metrics: LayoutMetrics): number | null {
for (const dimension of ["clientWidth", "clientHeight"] as const) {
const raw = metrics.visualViewport?.[dimension];
const css = metrics.cssVisualViewport?.[dimension];
if (
raw !== undefined &&
css !== undefined &&
Number.isFinite(raw) &&
Number.isFinite(css) &&
raw > 0 &&
css > 0
) {
const scale = raw / css;
if (Number.isFinite(scale) && scale > 0) return scale;
}
}
return null;
}
/** The legacy field is a compatibility fallback, never a source of raster DPR. */
export function cssViewport(metrics: LayoutMetrics): CssViewport {
const value = metrics.cssLayoutViewport ?? metrics.layoutViewport;
+1 -1
View File
@@ -929,7 +929,7 @@ export async function captureVomObservation(
const notices: string[] = [];
if (facts.issues.some((issue) => issue.stage === "geometry"))
notices.push(
"@warning geometry incomplete: some page or frame content has no top-level coordinates.",
"@warning iframe geometry incomplete: some frame content has no top-level coordinates.",
);
const incompleteStages = ["dom", "ax", "forms", "ownership"].filter((stage) =>
facts.issues.some((issue) => issue.stage === stage),
@@ -5,7 +5,7 @@ import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import type { CdpRunner } from "../../shared";
import { captureObservationFacts, semanticCapture } from "../capture-coordinator";
import { buildSemanticGraph } from "../semantic-graph/build";
import { REQUESTED_STYLES, type SnapshotReply } from "../snapshot";
import { REQUESTED_STYLES } from "../snapshot";
function fixture(
options: { frames?: CdpFrame[]; fail?: string; omitDocument?: string; overlay?: string } = {},
@@ -43,11 +43,7 @@ function fixture(
throw new Error("fixture failure");
let result: unknown = {};
if (method === "Page.getLayoutMetrics")
result = {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
result = { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
if (method === "DOMSnapshot.captureSnapshot") {
expect((params as { computedStyles: unknown }).computedStyles).toEqual(REQUESTED_STYLES);
result = {
@@ -78,8 +74,6 @@ function fixture(
const children = localFrames.filter((f) => f.parentFrameId === frame.frameId);
return {
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: frame.frameId,
nodes: {
backendNodeId: [
@@ -146,35 +140,6 @@ function fixture(
}
describe("captureObservationFacts", () => {
it("normalizes snapshot-owned root scroll instead of mixing in stale CSS scroll", async () => {
const { cdp } = fixture({ frames: [{ frameId: "main", target: { tabId: 4 } }] });
const original = cdp.sendToTarget!;
cdp.sendToTarget = async (target, method, params) => {
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 2000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 999, pageY: 999 },
} as never;
const reply = await original(target, method, params);
if (method === "DOMSnapshot.captureSnapshot") {
const doc = (reply as SnapshotReply).documents![0];
doc.scrollOffsetX = 40;
doc.scrollOffsetY = 200;
doc.layout!.bounds![1] = [200, 800, 240, 80];
}
return reply as never;
};
cdp.send = (tabId, method, params) => cdp.sendToTarget!({ tabId }, method, params);
const facts = await captureObservationFacts(cdp, 4);
expect(facts.documents[0].domNodes.find((node) => node.backendNodeId === 2)).toMatchObject({
localRect: { x: 80, y: 300, w: 120, h: 40 },
rect: { x: 80, y: 300, w: 120, h: 40 },
rendered: true,
});
expect(facts.issues).toEqual([]);
});
it("collects each target once and scopes equal backend IDs", async () => {
const { cdp, logs } = fixture();
const facts = await captureObservationFacts(cdp, 4);
@@ -364,8 +329,6 @@ function childSnapshot(frameId: string, backendNodeId: number) {
strings,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId,
nodes: {
parentIndex: [-1, 0],
@@ -394,11 +357,7 @@ describe("OOPIF capture", () => {
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
const sendToTarget = vi.fn(async (target, method) => {
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 },
};
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } };
}
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
@@ -420,11 +379,7 @@ describe("OOPIF capture", () => {
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
}
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
@@ -502,11 +457,7 @@ describe("OOPIF capture", () => {
};
const reply = async (_target: unknown, method: string) => {
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 300, clientHeight: 200 },
};
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200 } };
if (method === "DOMSnapshot.captureSnapshot")
return typeof _target === "number" ? childSnapshot("main", 11) : snapshot;
if (method === "DOM.getBoxModel")
@@ -564,11 +515,7 @@ describe("OOPIF capture", () => {
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable");
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
@@ -611,7 +558,7 @@ describe("OOPIF capture", () => {
expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }),
]);
expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual(
expect.objectContaining({ rect: null, localRect: null, rendered: true }),
expect.objectContaining({ rect: null, localRect: { x: 10, y: 20, w: 100, h: 40 } }),
);
});
});
@@ -621,8 +568,6 @@ function siblingCaptureFixture(
beforeReply: (method: string, params: Record<string, unknown>) => Promise<void> = async () => {},
) {
const document = (id: number, owners: number[], childIndexes: number[]) => ({
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: `frame-${id}`,
nodes: {
parentIndex: [-1, ...owners.map(() => 0)],
@@ -654,11 +599,7 @@ function siblingCaptureFixture(
await beforeReply(method, args);
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
if (method === "DOM.getBoxModel")
return { model: { content: [0, 0, 200, 0, 200, 100, 0, 100] } };
if (method === "DOM.resolveNode") return { object: { objectId: String(args.backendNodeId) } };
@@ -862,11 +803,7 @@ describe("snapshot document provenance", () => {
(method === "DOMSnapshot.captureSnapshot"
? { ...f.snapshot, documents: [f.snapshot.documents[1]] }
: method === "Page.getLayoutMetrics"
? {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 200, clientHeight: 100 },
}
? { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }
: {}) as T,
};
const facts = await captureObservationFacts(cdp, 4);
@@ -215,11 +215,7 @@ function makeCdp(snapshot: unknown) {
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
}
if (method === "Runtime.evaluate") {
return {
@@ -281,8 +277,6 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return hoverTriggerSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -346,8 +340,6 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return twoHoverTriggerSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -396,8 +388,6 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return nestedHoverTriggerSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -743,8 +733,6 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 200 },
};
}
@@ -757,7 +745,7 @@ describe("snapshot capture and hover", () => {
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
});
it("normalizes raw snapshot bounds before viewport clipping", async () => {
it("keeps CSS snapshot bounds independent of legacy metrics ratio", async () => {
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
const i = (s: string) => S.indexOf(s);
const snapshot = {
@@ -792,8 +780,6 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 2000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
layoutViewport: { clientWidth: 2000, clientHeight: 1600 },
};
@@ -811,8 +797,8 @@ describe("snapshot capture and hover", () => {
expect(nodes.find((n) => n.backendNodeId === 12)?.localRect).toEqual({
x: 0,
y: 0,
w: 1000,
h: 800,
w: 2000,
h: 1600,
});
});
@@ -870,8 +856,6 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -962,8 +946,6 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -1053,8 +1035,6 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -1117,8 +1097,6 @@ describe("snapshot capture and hover", () => {
},
},
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "child",
nodes: {
parentIndex: [-1, 0, 1],
@@ -1141,8 +1119,6 @@ describe("snapshot capture and hover", () => {
},
},
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "nested",
nodes: {
parentIndex: [-1, 0],
@@ -1170,8 +1146,6 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -1251,8 +1225,6 @@ describe("snapshot capture and hover", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 2000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 100 },
layoutViewport: { clientWidth: 2000, clientHeight: 1600 },
};
@@ -1261,7 +1233,7 @@ describe("snapshot capture and hover", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 / 2 - 100);
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 - 100);
});
it("collectOverlayExcludedBackendIds walks the pierced overlay host subtree", async () => {
@@ -77,7 +77,7 @@ describe("document facts", () => {
);
expect(nodes[0].attrs).toEqual({ "aria-hidden": "true", inert: "" });
expect(nodes[0].layout).toEqual({
boundsSpace: "snapshot-document-layout",
boundsSpace: "snapshot-document-css",
bounds: [10, 20, 120, 40],
styles: {
position: "static",
+2 -2
View File
@@ -6,9 +6,9 @@ import { createCaptureCheckpoint } from "./capture-abort";
import type { CapturedNode } from "./capture-types";
import type { FrameOwnedAxNode } from "./frame-document";
/** Bounds retain their raw snapshot document layout units until normalization. */
/** Bounds retain their snapshot document CSS units until normalization. */
export interface SnapshotLayout {
readonly boundsSpace: "snapshot-document-layout";
readonly boundsSpace: "snapshot-document-css";
bounds?: number[];
styles: Readonly<Record<string, string>>;
}
+20 -53
View File
@@ -4,15 +4,9 @@ import { type GeometryProjection, projectRectToViewport } from "../geometry";
import {
type FrameProjectionState,
projectSnapshotRect,
type SnapshotCoordinates,
snapshotCoordinates,
snapshotViewportRect,
} from "../geometry/coordinate-types";
import {
GeometryContext,
type LayoutMetrics,
snapshotLayoutScale,
} from "../geometry/frame-context";
import { cssViewport, GeometryContext } from "../geometry/frame-context";
import {
createCaptureCheckpoint,
isAbortError as isCaptureAbort,
@@ -32,7 +26,8 @@ export interface FrameContext {
projection: FrameProjectionState | null;
targetProjection?: GeometryProjection | null;
target: CdpTarget;
coordinates: SnapshotCoordinates | null;
scrollX: number;
scrollY: number;
}
export interface NormalizedDocument {
@@ -56,11 +51,10 @@ export async function normalizeDocument(
if (pending) await pending;
}
const node = decoded.nodes[i];
const bounds = node.layout?.bounds ?? [];
const input = snapshotViewportRect(
bounds,
node.layout?.bounds ?? [],
{ target: context.target, frameId: context.frameId },
context.coordinates,
{ x: context.scrollX, y: context.scrollY },
);
const local = input?.rect;
let rect =
@@ -83,10 +77,7 @@ export async function normalizeDocument(
localRect: local ? { x: local.x, y: local.y, w: local.width, h: local.height } : null,
rect: rect ? { x: rect.x, y: rect.y, w: rect.width, h: rect.height } : null,
rendered:
bounds.length >= 4 &&
bounds.slice(0, 4).every(Number.isFinite) &&
bounds[2] > 0 &&
bounds[3] > 0 &&
!!local &&
visibility !== "hidden" &&
visibility !== "collapse" &&
(Number.parseFloat(opacity) || 0) > 0,
@@ -141,20 +132,17 @@ export async function normalizeSnapshot(
}
sources.set(id, doc);
}
let metrics: LayoutMetrics = {};
let viewport: Viewport = { width: 0, height: 0 };
let scrollX = 0,
scrollY = 0;
try {
metrics = await geometry.layoutMetrics(target);
const measured = cssViewport(await geometry.layoutMetrics(target));
viewport = { width: measured.width, height: measured.height };
scrollX = measured.scrollX;
scrollY = measured.scrollY;
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
const viewport: Viewport = {
width: metrics.cssLayoutViewport?.clientWidth ?? 0,
height: metrics.cssLayoutViewport?.clientHeight ?? 0,
};
const layoutUnitsPerCssPixel = snapshotLayoutScale(metrics);
const viewportAvailable = [viewport.width, viewport.height].every(
(size) => Number.isFinite(size) && size > 0,
);
const children = new Map<string, CdpFrame[]>();
const pending: CdpFrame[] = [];
@@ -200,24 +188,12 @@ export async function normalizeSnapshot(
const doc = sources.get(frame.frameId);
if (!doc) continue;
const source = { target, frameId: frame.frameId };
const coordinates = snapshotCoordinates(
doc,
layoutUnitsPerCssPixel,
const state: FrameProjectionState | null =
frame.frameId === rootFrameId
? { x: metrics.cssLayoutViewport?.pageX, y: metrics.cssLayoutViewport?.pageY }
: undefined,
);
let state: FrameProjectionState | null =
frame.frameId === rootFrameId
? viewportAvailable && coordinates
? {
status: "available",
projection: {
source,
geometry: { sourceClips: [], edges: [], topViewport: viewport },
},
}
: { status: "unavailable", source, reason: "snapshot-coordinates-unavailable" }
? {
status: "available",
projection: { source, geometry: { sourceClips: [], edges: [], topViewport: viewport } },
}
: (projections.get(frame.frameId) ?? null);
if (!state && frame.frameId !== rootFrameId)
issues.push({
@@ -226,8 +202,6 @@ export async function normalizeSnapshot(
stage: "ownership",
reason: "frame-ownership-unresolved",
});
if (!coordinates && (!state || state.status === "available"))
state = { status: "unavailable", source, reason: "snapshot-coordinates-unavailable" };
projections.set(frame.frameId, state);
const projection = state?.status === "available" ? state.projection : null;
if (!projection || (target.sessionId && !targetProjection))
@@ -272,14 +246,6 @@ export async function normalizeSnapshot(
projections.set(child.frameId, null);
continue;
}
if (!snapshotCoordinates(sources.get(child.frameId)!, layoutUnitsPerCssPixel)) {
projections.set(child.frameId, {
status: "unavailable",
source: childSource,
reason: "snapshot-coordinates-unavailable",
});
continue;
}
try {
projections.set(
child.frameId,
@@ -308,7 +274,8 @@ export async function normalizeSnapshot(
projection: state,
target,
...(target.sessionId ? { targetProjection } : {}),
coordinates,
scrollX: doc.scrollOffsetX ?? (frame.frameId === rootFrameId ? scrollX : 0),
scrollY: doc.scrollOffsetY ?? (frame.frameId === rootFrameId ? scrollY : 0),
},
signal,
);
+1 -1
View File
@@ -175,7 +175,7 @@ export async function decodeDocument(
li === undefined
? undefined
: {
boundsSpace: "snapshot-document-layout" as const,
boundsSpace: "snapshot-document-css" as const,
bounds: dl?.bounds?.[li],
styles,
};
@@ -1,44 +0,0 @@
# DOMSnapshot layout-unit regression (#195)
Source: https://github.com/Tencent/BrowserSkill/pull/195
DOMSnapshot bounds and document scroll offsets retain Blink layout units. Treating them as CSS
pixels changes both positions and sizes under device scaling/browser zoom, before iframe
projection or viewport clipping even starts.
The fixture contains a scrolled root, a scrolled same-process iframe, an OOPIF served from the
other loopback hostname, and a nested iframe in each. Owners have borders, padding and CSS
transforms. The geometry test compares the production capture's local and top-level rectangles
against independent DOM border boxes, with at most 2 CSS pixels of layout rounding. It also
checks that each target reads layout metrics only once. No credentials or personal tabs are used.
Run the geometric assertions with Node 22+ and a local Chrome executable:
```sh
BSK_GEOMETRY_CHROME=/path/to/chrome pnpm --filter @browser-skill/extension exec vitest run \
src/tools/__tests__/snapshot-coordinates.browser.test.ts
```
The test launches and cleans up its own headless Chrome profiles. It explicitly verifies the
requested browser zoom and OOPIF topology, so unsupported setups fail rather than silently
testing a different configuration. It is skipped in ordinary unit runs unless the executable
is specified. No browser download or new package dependency is required.
The corpus smoke case checks fixture readiness and observable semantics through the real CLI:
```sh
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --case snapshot-coordinates --bsk ./target/debug/bsk
```
Smoke alone does **not** certify coordinate accuracy; CLI observations do not expose raw boxes.
Use the geometric test above for the regression's numeric assertions.
## Separate existing boundary
The OOPIF root uses non-occupying scrollbars in the unit-conversion regression. Root and
same-process frames retain normal scrollbars. Append `&classic-scrollbars` to the fixture URL
to reproduce the separate existing OOPIF projection issue: `targetProjection()` maps the CSS
layout viewport (which excludes occupying scrollbars) onto the entire owner content quad.
This inflates the projected coordinates when scrollbars occupy space. Main already has that
mapping; fixing it also changes shared live geometry and is outside this snapshot-unit fix.
The test does not relax its numeric tolerance to absorb that error.
@@ -1,99 +0,0 @@
import { spawn } from "node:child_process";
import { once } from "node:events";
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
// This regression owns its browser/profile. It never attaches to a user's Chrome.
export async function withChrome({ executable, deviceScale, zoom }, run) {
const profile = await mkdtemp(join(tmpdir(), "bsk-snapshot-coordinates-"));
let chrome;
let socket;
const pending = new Map();
try {
await mkdir(join(profile, "Default"));
await writeFile(
join(profile, "Default", "Preferences"),
JSON.stringify({
partition: { default_zoom_level: { x: Math.log(zoom) / Math.log(1.2) } },
}),
);
chrome = spawn(
executable,
[
"--headless=new",
"--no-first-run",
"--no-default-browser-check",
"--disable-extensions",
"--site-per-process",
"--remote-debugging-port=0",
"--window-size=1600,1200",
`--force-device-scale-factor=${deviceScale}`,
`--user-data-dir=${profile}`,
"about:blank",
],
{ stdio: ["ignore", "ignore", "pipe"] },
);
const endpoint = await new Promise((resolve, reject) => {
let output = "";
const timeout = setTimeout(
() => reject(new Error(`Chrome startup timed out: ${output}`)),
15_000,
);
const fail = (error) => {
clearTimeout(timeout);
reject(error);
};
chrome.once("error", fail);
chrome.once("exit", (code) => fail(new Error(`Chrome exited (${code}): ${output}`)));
chrome.stderr.on("data", (chunk) => {
output += chunk;
const match = output.match(/DevTools listening on (ws:\/\/\S+)/);
if (match) {
clearTimeout(timeout);
resolve(match[1]);
}
});
});
socket = new WebSocket(endpoint);
await new Promise((resolve, reject) => {
socket.addEventListener("open", resolve, { once: true });
socket.addEventListener("error", reject, { once: true });
});
socket.addEventListener("message", ({ data }) => {
const reply = JSON.parse(data);
const request = pending.get(reply.id);
if (!request) return;
clearTimeout(request.timeout);
pending.delete(reply.id);
if (reply.error) request.reject(new Error(JSON.stringify(reply.error)));
else request.resolve(reply.result);
});
let sequence = 0;
const send = (method, params = {}, sessionId) =>
new Promise((resolve, reject) => {
const id = ++sequence;
const timeout = setTimeout(() => {
pending.delete(id);
reject(new Error(`CDP timed out: ${method}`));
}, 10_000);
pending.set(id, { resolve, reject, timeout });
socket.send(JSON.stringify({ id, method, params, sessionId }));
});
await run(send);
} finally {
for (const request of pending.values()) {
clearTimeout(request.timeout);
request.reject(new Error("regression browser closed"));
}
socket?.close();
if (chrome && chrome.exitCode === null && chrome.signalCode === null) {
const exited = once(chrome, "exit");
const timeout = setTimeout(() => chrome.kill("SIGKILL"), 5_000);
chrome.kill();
await exited;
clearTimeout(timeout);
}
await rm(profile, { recursive: true, force: true });
}
}
@@ -1,33 +0,0 @@
{
"$schema": "../../../schemas/case.schema.json",
"schemaVersion": 1,
"id": "snapshot-coordinates",
"title": "Scrolled and nested frame coordinate fixture",
"suite": "regression",
"tags": ["geometry", "iframe", "scroll"],
"fixture": { "startPath": "/snapshot-coordinates" },
"prompts": {
"en": "Open {url}, observe the page and its frames, report the GEOMETRY marker, then close the session.",
"zh-CN": "打开 {url},观察页面及其 iframe,报告 GEOMETRY 标记,然后关闭会话。"
},
"coverage": ["session.start", "session.stop", "page.navigate", "inspect.observe"],
"assertions": {
"site": [
{
"label": "root fixture scrolled and settled",
"type": "geometry.ready",
"where": { "data.root": true },
"minCount": 1
}
],
"response": [{ "label": "observed the fixture marker", "includes": "GEOMETRY-195" }],
"adapter": [{ "label": "browser session was closed", "key": "sessionStopped" }]
},
"smoke": {
"steps": [
{ "action": "navigate" },
{ "action": "wait-site-event", "type": "geometry.ready", "where": { "root": true } },
{ "action": "observe" }
]
}
}
@@ -1,52 +0,0 @@
import { page, withRun } from "../../../lib/fixtures.mjs";
export default {
id: "snapshot-coordinates",
routes: ["/snapshot-coordinates", "/snapshot-coordinates/frame", "/snapshot-coordinates/nested"],
render({ pathname, runId, query }) {
const root = pathname === "/snapshot-coordinates";
const nested = pathname.endsWith("/nested");
return page({
title: "Snapshot coordinate regression",
body: `
<style>
main { margin: 0; width: auto; }
body { width: 2000px; height: 2400px; }
html { overflow-anchor: none; }
${query.get("scrollbars") === "none" ? "html { scrollbar-width: none; }" : ""}
button { position: absolute; box-sizing: border-box; padding: 0; border: 2px solid black; border-radius: 0; }
#probe { left: ${root ? 180 : 100}px; top: ${root ? 360 : 160}px; width: 120px; height: 40px; }
iframe { position: absolute; box-sizing: content-box; width: 320px; height: 220px;
border: 6px solid black; padding: 8px; transform: scale(1.25); transform-origin: 0 0; }
#same { left: 340px; top: 400px; }
#cross { left: 820px; top: 400px; }
#nested { left: 150px; top: 210px; width: 130px; height: 70px; border-width: 2px; padding: 4px; transform: scale(0.8); }
${nested ? "#probe { left: 30px; top: 40px; width: 60px; height: 20px; }" : ""}
</style>
<button id="probe" data-geometry-probe>GEOMETRY-195</button>
${root ? `<iframe id="same" name="same" title="same-process frame" src="${withRun("/snapshot-coordinates/frame", runId)}"></iframe><iframe id="cross" name="cross" title="cross-site frame"></iframe>` : ""}
${!root && !nested ? `<iframe id="nested" name="nested" title="nested frame" src="${withRun("/snapshot-coordinates/nested", runId)}"></iframe>` : ""}
`,
script: `
history.scrollRestoration = "manual";
const cross = document.querySelector("#cross");
if (cross) {
const url = new URL(${JSON.stringify(withRun("/snapshot-coordinates/frame", runId))}, location.href);
url.hostname = location.hostname === "127.0.0.1" ? "localhost" : "127.0.0.1";
// Isolate unit conversion from the pre-existing OOPIF scrollbar-width
// projection issue. ?classic-scrollbars retains that separate reproduction.
if (!new URLSearchParams(location.search).has("classic-scrollbars")) url.searchParams.set("scrollbars", "none");
cross.src = url.href;
}
window.addEventListener("load", () => {
scrollTo(${root ? "80, 240" : nested ? "10, 20" : "40, 100"});
requestAnimationFrame(() => requestAnimationFrame(() => {
window.geometryReady = true;
document.documentElement.dataset.geometryReady = "true";
browserEval.send("geometry.ready", { root: ${root} });
}));
});
`,
});
},
};
+4 -9
View File
@@ -41,22 +41,17 @@ test("case manifests are discovered, ordered, and grouped into suites", () => {
"diagnostics",
"mobile-emulation",
"generated-form",
"snapshot-coordinates",
],
);
assert.deepEqual(repositorySummary(cases, fixtureRegistry).suites, {
core: 6,
matrix: 1,
regression: 1,
});
assert.deepEqual(repositorySummary(cases, fixtureRegistry).suites, { core: 6, matrix: 1 });
});
test("repository validation links every case to a fixture and valid workflow evidence", () => {
assert.deepEqual(validateRepositoryCases(cases, fixtureRegistry), []);
const summary = repositorySummary(cases, fixtureRegistry);
assert.equal(summary.cases, 8);
assert.equal(summary.fixtureModules, 9);
assert.equal(summary.fixtureRoutes, 15);
assert.equal(summary.cases, 7);
assert.equal(summary.fixtureModules, 8);
assert.equal(summary.fixtureRoutes, 12);
});
test("manifest validation rejects unknown operations and incomplete workflow steps", () => {
@@ -1,7 +1,7 @@
{
"name": "@wxg-prc-cpg/browser-skill-dsh-plugin",
"description": "DeepSeek Harness tool plugin that exposes BrowserSkill browser automation (browser_* tools) to the model",
"version": "0.2.1",
"version": "0.2.0",
"license": "MIT",
"publishConfig": {
"access": "public",
+3 -9
View File
@@ -174,22 +174,16 @@ function updateChangelog(version) {
const today = new Date().toISOString().slice(0, 10);
const escapedVer = version.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const versionHeading = new RegExp(`^## \\[${escapedVer}\\](?:\\s|$)`, "m");
// First: fill in any date-placeholder line for this version (e.g. "## [0.2.0] - 2026-09-XX")
const versionPlaceholder = new RegExp(`^## \\[${escapedVer}\\]\\s*-\\s*\\d{4}-\\d{2}-XX`, "m");
if (versionPlaceholder.test(content)) {
content = content.replace(versionPlaceholder, `## [${version}] - ${today}`);
} else if (!versionHeading.test(content)) {
// Preserve prepared release entries; new versions go before the release history.
const heading = `## [${version}] - ${today}`;
} else {
// Otherwise: insert a new section after [Unreleased]
const unreleased = /^## \[Unreleased\]/m;
if (unreleased.test(content)) {
content = content.replace(unreleased, `## [Unreleased]\n\n${heading}`);
} else if (/^## /m.test(content)) {
content = content.replace(/^## /m, `${heading}\n\n## `);
} else {
content = `${content.trimEnd()}\n\n${heading}\n`;
content = content.replace(unreleased, `## [Unreleased]\n\n## [${version}] - ${today}`);
}
}