Compare commits

..
10 Commits
25 changed files with 2536 additions and 892 deletions
@@ -1,5 +1,5 @@
import { describe, expect, it, vi } from "vitest"; import { describe, expect, it, vi } from "vitest";
import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry"; import { resolveNodeGeometry } from "../frame-geometry";
import { import {
clipPolygon, clipPolygon,
polygonArea, polygonArea,
@@ -10,6 +10,7 @@ import {
rectPolygon, rectPolygon,
regionBounds, regionBounds,
} from "../geometry"; } from "../geometry";
import { GeometryContext } from "../geometry/frame-context";
import type { CdpRunner } from "../shared"; import type { CdpRunner } from "../shared";
describe("frame geometry projection", () => { describe("frame geometry projection", () => {
@@ -82,28 +83,24 @@ describe("frame geometry projection", () => {
throw new Error(`unexpected child command ${method}`); throw new Error(`unexpected child command ${method}`);
}) as CdpRunner["sendToTarget"], }) as CdpRunner["sendToTarget"],
}; };
const projection = await resolveFrameProjection( const projection = await new GeometryContext(cdp, 4, {
cdp, rootFrameId: "main",
{ frames: [
rootFrameId: "main", { frameId: "main", target: { tabId: 4 } },
frames: [ {
{ frameId: "main", target: { tabId: 4 } }, frameId: "same-process-parent",
{ parentFrameId: "main",
frameId: "same-process-parent", ownerBackendNodeId: 10,
parentFrameId: "main", target: { tabId: 4 },
ownerBackendNodeId: 10, },
target: { tabId: 4 }, {
}, frameId: "oopif",
{ parentFrameId: "same-process-parent",
frameId: "oopif", ownerBackendNodeId: 20,
parentFrameId: "same-process-parent", target: { tabId: 4, sessionId: "oopif-session" },
ownerBackendNodeId: 20, },
target: { tabId: 4, sessionId: "oopif-session" }, ],
}, }).targetProjection("oopif");
],
},
"oopif",
);
expect(projection?.edges).toEqual([ expect(projection?.edges).toEqual([
{ {
@@ -0,0 +1,380 @@
import { describe, expect, it, vi } from "vitest";
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
import { resolveNodeGeometry } from "../frame-geometry";
import { rectPolygon } from "../geometry";
import {
projectSnapshotRect,
screenshotPageRect,
snapshotViewportRect,
} from "../geometry/coordinate-types";
import { GeometryContext } from "../geometry/frame-context";
import type { CdpRunner } from "../shared";
const target = { tabId: 4 };
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
{ frameId: "main", target },
{ frameId: "child", parentFrameId: "main", ownerBackendNodeId: 10, target },
],
};
function driver() {
const send = vi.fn(async (_tab: number, method: string): Promise<object> => {
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 1000 } };
if (method === "DOM.getBoxModel")
return { model: { content: [52.5, 612.5, 427.5, 612.5, 427.5, 862.5, 52.5, 862.5] } };
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 300, height: 200 } } };
if (method === "Runtime.releaseObject") return {};
throw new Error(method);
});
return { send: send as CdpRunner["send"], getFrameGraph: vi.fn(async () => graph), calls: send };
}
describe("measurement geometry context", () => {
it("shares in-flight frame measurements, but never shares them with the next context", async () => {
const cdp = driver();
const context = new GeometryContext(cdp, 4);
const results = await Promise.all(
Array.from({ length: 20 }, () => context.targetProjection("child")),
);
expect(results.every((value) => value === results[0])).toBe(true);
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1);
expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([
"Page.getLayoutMetrics",
"DOM.getBoxModel",
]);
await new GeometryContext(cdp, 4).targetProjection("child");
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(2);
expect(cdp.calls.mock.calls).toHaveLength(4);
});
it("projects frame-local snapshot CSS bounds through the scaled owner content quad", async () => {
const cdp = driver();
const context = new GeometryContext(cdp, 4);
const source = { target, frameId: "child" };
const projection = await context.snapshotProjection(source, 10, [], {
width: 1000,
height: 1000,
});
if (projection.status !== "available") throw new Error("expected available projection");
const input = snapshotViewportRect([17, 23, 120, 40], source, { x: 0, y: 0 });
expect(projectSnapshotRect(input!, projection.projection)).toEqual({
x: 73.75,
y: 641.25,
width: 150,
height: 50,
});
const scrolled = snapshotViewportRect([17, 23, 120, 40], source, { x: 0, y: 10 });
expect(projectSnapshotRect(scrolled!, projection.projection)?.y).toBe(628.75);
await context.snapshotProjection(source, 10, [], { width: 1000, height: 1000 });
expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.resolveNode")).toHaveLength(
1,
);
expect(
cdp.calls.mock.calls.filter(([, method]) => method === "Runtime.releaseObject"),
).toHaveLength(1);
});
it("falls back only for missing or invalid batch entries and does not reuse sizes across contexts", async () => {
const cdp = driver();
const original = cdp.send;
const calls: string[] = [];
cdp.send = async (tabId, method, params) => {
calls.push(method);
if (method === "Runtime.evaluate")
return {
result: {
deepSerializedValue: {
type: "array",
value: [
{
type: "array",
value: [
{ type: "node", value: { backendNodeId: 10 } },
{ type: "string", value: '{"width":300,"height":200}' },
],
},
{
type: "array",
value: [
{ type: "node", value: { backendNodeId: 11 } },
{ type: "string", value: '{"width":0,"height":200}' },
],
},
],
},
},
} as never;
if (method === "Runtime.releaseObjectGroup") return {} as never;
return original(tabId, method, params);
};
const context = new GeometryContext(cdp, 4);
context.registerSnapshotOwners(target, new Set([10, 11]));
const values = await Promise.all(
[10, 11].map((id) =>
context.snapshotProjection({ target, frameId: String(id) }, id, [], {
width: 1000,
height: 1000,
}),
),
);
expect(values.every((value) => value.status === "available")).toBe(true);
expect(calls.filter((method) => method === "DOM.resolveNode")).toHaveLength(1);
expect(calls.filter((method) => method === "Runtime.evaluate")).toHaveLength(1);
await new GeometryContext(cdp, 4).snapshotProjection({ target, frameId: "child" }, 10, [], {
width: 1000,
height: 1000,
});
expect(calls.filter((method) => method === "DOM.resolveNode")).toHaveLength(2);
});
it("waits for batch object cleanup and propagates cancellation", async () => {
const cdp = driver();
const original = cdp.send;
const controller = new AbortController();
let release!: () => void;
let cleanup!: () => void;
const started = new Promise<void>((resolve) => {
cleanup = resolve;
});
cdp.send = async (tabId, method, params) => {
if (method === "Runtime.evaluate") return {} as never;
if (method === "Runtime.releaseObjectGroup") {
cleanup();
await new Promise<void>((resolve) => {
release = resolve;
});
return {} as never;
}
return original(tabId, method, params);
};
const context = new GeometryContext(cdp, 4, undefined, controller.signal);
context.registerSnapshotOwners(target, new Set([10, 11]));
let settled = false;
const operation = context
.snapshotProjection({ target, frameId: "child" }, 10, [], { width: 1000, height: 1000 })
.finally(() => {
settled = true;
});
const rejected = expect(operation).rejects.toMatchObject({ name: "AbortError" });
await started;
controller.abort();
expect(settled).toBe(false);
release();
await rejected;
expect(cdp.calls.mock.calls.some(([, method]) => method === "DOM.resolveNode")).toBe(false);
});
it("rejects owner mismatches and invalid numeric bounds", () => {
const input = snapshotViewportRect(
[30, 50, 120, 40],
{ target, frameId: "main" },
{ x: 10, y: 20 },
)!;
const projection = {
source: { target, frameId: "main" },
geometry: { sourceClips: [], edges: [], topViewport: { width: 800, height: 600 } },
};
expect(projectSnapshotRect(input, projection)).toEqual({
x: 20,
y: 30,
width: 120,
height: 40,
});
expect(
projectSnapshotRect(input, { ...projection, source: { target, frameId: "other" } }),
).toBeNull();
expect(
projectSnapshotRect(input, {
...projection,
source: { target: { tabId: 5 }, frameId: "main" },
}),
).toBeNull();
expect(snapshotViewportRect([0, 0, NaN, 40], { target }, { x: 0, y: 0 })).toBeNull();
expect(snapshotViewportRect([0, 0, 10, 40], { target }, { x: Infinity, y: 0 })).toBeNull();
});
it("does not turn a border quad or a failed owner read into a content projection", async () => {
const cdp: CdpRunner = {
send: vi.fn(async () => ({
model: { border: [0, 0, 100, 0, 100, 100, 0, 100] },
})) as CdpRunner["send"],
};
expect(
await new GeometryContext(cdp, 4).snapshotProjection({ target, frameId: "child" }, 10, [], {
width: 100,
height: 100,
}),
).toEqual({
status: "unavailable",
source: { target, frameId: "child" },
ownerBackendNodeId: 10,
});
});
it("keeps ancestor clips in target coordinates instead of applying nested offsets twice", async () => {
const cdp = driver();
const projection = await new GeometryContext(cdp, 4).snapshotProjection(
{ target, frameId: "nested" },
10,
[rectPolygon({ x: 60, y: 620, w: 100, h: 100 })],
{ width: 1000, height: 1000 },
);
if (projection.status !== "available") throw new Error("expected available projection");
const input = snapshotViewportRect(
[0, 0, 300, 200],
{ target, frameId: "nested" },
{ x: 0, y: 0 },
);
expect(projectSnapshotRect(input!, projection.projection)).toEqual({
x: 60,
y: 620,
width: 100,
height: 100,
});
});
it("rejects malformed frame ancestry without looping", async () => {
const cdp = driver();
const cycle: CdpFrameGraph = {
rootFrameId: "a",
frames: [
{ frameId: "a", parentFrameId: "b", target },
{ frameId: "b", parentFrameId: "a", target },
],
};
expect(await new GeometryContext(cdp, 4, cycle).targetProjection("a")).toBeNull();
expect(cdp.calls).not.toHaveBeenCalled();
const orphan: CdpFrameGraph = {
rootFrameId: "a",
frames: [{ frameId: "a", parentFrameId: "missing", target }],
};
expect(await new GeometryContext(cdp, 4, orphan).targetProjection("a")).toBeNull();
});
it("rejects a live node whose frame belongs to another target before any DOM input", async () => {
const cdp = driver();
expect(
await resolveNodeGeometry(
cdp,
4,
{ target: { tabId: 4, sessionId: "wrong" }, frameId: "child", backendNodeId: 20 },
{ scrollIntoView: true },
),
).toMatchObject({ code: "cdp_failed" });
expect(cdp.calls).not.toHaveBeenCalled();
});
it("bounds concurrent measurements and does not dispatch queued reads after cancellation", async () => {
let active = 0;
let peak = 0;
const release: Array<() => void> = [];
const send = vi.fn(async () => {
active++;
peak = Math.max(peak, active);
await new Promise<void>((resolve) => release.push(resolve));
active--;
return { cssLayoutViewport: { clientWidth: 100, clientHeight: 100 } };
});
const controller = new AbortController();
const context = new GeometryContext(
{ send: send as CdpRunner["send"] },
4,
undefined,
controller.signal,
);
const tasks = Array.from({ length: 8 }, (_, index) =>
context.viewport({ tabId: 4, sessionId: String(index) }),
);
const settled = Promise.allSettled(tasks);
expect(send).toHaveBeenCalledTimes(4);
controller.abort();
for (const done of release) done();
const results = await settled;
expect(peak).toBe(4);
expect(send).toHaveBeenCalledTimes(4);
expect(results.filter((item) => item.status === "rejected")).toHaveLength(4);
});
it("releases an owner object even when cancellation arrives before its viewport read", async () => {
const controller = new AbortController();
const cdp = driver();
const original = cdp.calls.getMockImplementation()!;
cdp.calls.mockImplementation(async (tab, method) => {
const result = await original(tab, method);
if (method === "DOM.resolveNode") controller.abort();
return result;
});
const context = new GeometryContext(cdp, 4, undefined, controller.signal);
await expect(
context.snapshotProjection({ target, frameId: "child" }, 10, [], {
width: 1000,
height: 1000,
}),
).rejects.toMatchObject({ name: "AbortError" });
expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([
"DOM.getBoxModel",
"DOM.resolveNode",
"Runtime.releaseObject",
]);
});
it("reads live owner geometry only after scrolling", async () => {
const cdp = driver();
const original = cdp.calls.getMockImplementation()!;
let scrolled = false;
cdp.calls.mockImplementation(async (tab, method) => {
if (method === "DOM.scrollIntoViewIfNeeded") {
scrolled = true;
return {};
}
if (method === "DOM.getContentQuads") {
expect(scrolled).toBe(true);
return { quads: [[60, 620, 100, 620, 100, 640, 60, 640]] };
}
if (method === "DOM.getBoxModel") expect(scrolled).toBe(true);
return original(tab, method);
});
expect(
await resolveNodeGeometry(
cdp,
4,
{ target, frameId: "child", backendNodeId: 20 },
{ scrollIntoView: true },
),
).toMatchObject({ topBounds: { x: 60, y: 620, width: 40, height: 20 } });
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1);
expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.getBoxModel")).toHaveLength(
1,
);
});
it("adapts viewport CSS bounds to page DIP with scroll and browser zoom, never raster DPR", () => {
const viewport = { width: 1600, height: 1000, scrollX: 10, scrollY: 100, cssToDip: 0.9 };
const clip = screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, viewport);
expect(clip).toEqual({ space: "page-dip", rect: { x: 27, y: 117, width: 108, height: 36 } });
expect(
screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: NaN }),
).toBeNull();
expect(
screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: 0 }),
).toBeNull();
});
it("retries a rejected measurement only in a new operation", async () => {
const cdp = driver();
cdp.calls.mockRejectedValueOnce(new Error("unavailable"));
const context = new GeometryContext(cdp, 4);
await expect(context.viewport(target)).rejects.toThrow("unavailable");
await expect(context.viewport(target)).rejects.toThrow("unavailable");
expect(cdp.calls).toHaveBeenCalledTimes(1);
expect(await new GeometryContext(cdp, 4).viewport(target)).toEqual({
width: 1000,
height: 1000,
});
});
});
@@ -266,7 +266,15 @@ describe("handleScreenshot", () => {
expect(sent.find((c) => c.method === "Page.captureScreenshot")).toBeUndefined(); expect(sent.find((c) => c.method === "Page.captureScreenshot")).toBeUndefined();
}); });
it("captures a clipped PNG when ref is given", async () => { it.each([
{ zoom: 1, scroll: 0, expected: { x: 10, y: 20, width: 100, height: 40 } },
{ zoom: 0.9, scroll: 100, expected: { x: 9, y: 108, width: 90, height: 36 } },
{ zoom: 1.25, scroll: 100, expected: { x: 12.5, y: 150, width: 125, height: 50 } },
])("captures a ref in page DIP at zoom $zoom and scroll $scroll", async ({
zoom,
scroll,
expected,
}) => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11"); const ctx = await sm.start("aa11");
ctx.refStore.set("e5", 999, { tabId: 7 }); ctx.refStore.set("e5", 999, { tabId: 7 });
@@ -274,6 +282,10 @@ describe("handleScreenshot", () => {
"DOM.scrollIntoViewIfNeeded": () => ({}), "DOM.scrollIntoViewIfNeeded": () => ({}),
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }), "DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
"Page.captureScreenshot": () => ({ data: TINY_PNG }), "Page.captureScreenshot": () => ({ data: TINY_PNG }),
"Page.getLayoutMetrics": () => ({
cssLayoutViewport: { clientWidth: 1280, clientHeight: 720, pageX: 0, pageY: scroll },
cssVisualViewport: { zoom },
}),
}); });
const captureVisibleTab = vi.fn(); const captureVisibleTab = vi.fn();
const res = await handleScreenshot( const res = await handleScreenshot(
@@ -297,7 +309,8 @@ describe("handleScreenshot", () => {
clip?: { x: number; y: number; width: number; height: number }; clip?: { x: number; y: number; width: number; height: number };
} }
)?.clip; )?.clip;
expect(clip).toMatchObject({ x: 10, y: 20, width: 100, height: 40 }); expect(clip).toMatchObject(expected);
expect(sent.filter((c) => c.method === "Page.getLayoutMetrics")).toHaveLength(1);
}); });
it("returns not_found for unknown ref", async () => { it("returns not_found for unknown ref", async () => {
@@ -655,6 +668,60 @@ describe("buildVomScene", () => {
expect(nestedInput?.parentId).toBe(result.nodes.find((item) => item.backendNodeId === 210)?.id); expect(nestedInput?.parentId).toBe(result.nodes.find((item) => item.backendNodeId === 210)?.id);
}); });
it("excludes an overlay only in its owning document when target backend ids collide", () => {
const owner = (id: number): CapturedNode => ({
backendNodeId: id,
parentBackendNodeId: null,
frameId: "main",
tag: "iframe",
attrs: {},
rect: { x: id * 10, y: 0, w: 200, h: 100 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
});
const owners = [owner(10), owner(20)];
const result = buildFrameVomScene(
[
{
frameId: "main",
contextScopeId: "main",
target: { tabId: 7 },
domNodes: owners,
axNodes: [],
},
...["left", "right"].map((frameId, i) => ({
frameId,
contextScopeId: frameId,
parentFrameId: "main",
ownerBackendNodeId: owners[i].backendNodeId,
target: { tabId: 7, sessionId: frameId },
excludedBackendNodeIds: new Set(i === 0 ? [7] : []),
domNodes: [],
axNodes: [
{
nodeId: "button",
frameId,
backendDOMNodeId: 7,
role: { type: "role", value: "button" },
name: { type: "computedString", value: frameId + " action" },
},
],
})),
],
{
nodes: owners,
viewport: { width: 800, height: 600 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
},
);
const rendered = renderVom(result);
expect(rendered.text).not.toContain("left action");
expect(rendered.text).toContain("right action");
expect(rendered.refs.find((ref) => ref.backendNodeId === 7)?.frameId).toBe("right");
});
it("does not place a child document at the page root when its frame boundary is unresolved", () => { it("does not place a child document at the page root when its frame boundary is unresolved", () => {
const childNode: CapturedNode = { const childNode: CapturedNode = {
backendNodeId: 101, backendNodeId: 101,
@@ -3134,7 +3201,9 @@ describe("handleSnapshot", () => {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
}; };
function makeFrameAwareDeps() { function makeFrameAwareDeps(
ownerGeometry: "unavailable" | "available" | "clipped" = "unavailable",
) {
const strings = ["html", "body", "iframe", "button", "title", "Remote", "static", "auto"]; const strings = ["html", "body", "iframe", "button", "title", "Remote", "static", "auto"];
const i = (value: string) => strings.indexOf(value); const i = (value: string) => strings.indexOf(value);
const styles = [i("static"), i("auto"), i("auto")]; const styles = [i("static"), i("auto"), i("auto")];
@@ -3214,6 +3283,16 @@ describe("handleSnapshot", () => {
]; ];
const send = vi.fn(async (_tabId: number, method: string) => { const send = vi.fn(async (_tabId: number, method: string) => {
if (method === "Page.getLayoutMetrics") return VP_METRICS; if (method === "Page.getLayoutMetrics") return VP_METRICS;
if (ownerGeometry !== "unavailable") {
if (method === "DOM.getBoxModel") {
const x = ownerGeometry === "clipped" ? 10000 : 100;
return { model: { content: [x, 100, x + 400, 100, x + 400, 400, x, 400] } };
}
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 400, height: 300 } } };
if (method === "Runtime.releaseObject") return {};
}
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {}; if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return snapshot; if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Accessibility.getFullAXTree") return { nodes: mainAx }; if (method === "Accessibility.getFullAXTree") return { nodes: mainAx };
@@ -3454,6 +3533,43 @@ describe("handleSnapshot", () => {
expect(result.text).toContain("•••"); expect(result.text).toContain("•••");
}); });
it.each([
undefined,
100,
0,
])("reports unavailable iframe geometry within the observation budget %s", async (maxTokens) => {
const result = await captureVomObservation(makeFrameAwareDeps().cdp, 4, "https://example.com", {
maxTokens,
});
expect(result.text.match(/@warning/g)).toHaveLength(1);
expect(result.text).toContain("iframe geometry incomplete");
expect(result.text.startsWith("@vom 1\n")).toBe(true);
if (maxTokens === undefined) {
expect(result.truncated).toBe(false);
expect(result.text).toContain("Frame action");
expect(result.matchNodes.find((node) => node.backendNodeId === 22)).toMatchObject({
rect: null,
localRect: { x: 20, y: 30, w: 120, h: 40 },
});
}
if (maxTokens === 100) expect(Math.ceil(result.text.length / 4)).toBeLessThanOrEqual(maxTokens);
});
it.each([
"available",
"clipped",
] as const)("does not report successful %s geometry as an observation failure", async (ownerGeometry) => {
const result = await captureVomObservation(
makeFrameAwareDeps(ownerGeometry).cdp,
4,
"https://example.com",
);
expect(result.text).not.toContain("@warning");
const node = result.matchNodes.find((item) => item.backendNodeId === 22);
if (ownerGeometry === "clipped") expect(node?.rect).toBeNull();
else expect(node?.rect).toEqual({ x: 120, y: 130, w: 120, h: 40 });
});
it("keeps frames and rendered refs on the same frame identity", async () => { it("keeps frames and rendered refs on the same frame identity", async () => {
const deps = makeFrameAwareDeps(); const deps = makeFrameAwareDeps();
+46 -151
View File
@@ -1,24 +1,21 @@
import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph"; import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
import type { RpcError } from "@/transport/types"; import type { RpcError } from "@/transport/types";
import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry"; import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry";
import { import {
clipPolygon, clipPolygon,
type GeometryProjection,
type Point, type Point,
type Polygon, type Polygon,
type ProjectiveEdge,
parseCdpQuad,
polygonArea, polygonArea,
polygonCentroid, polygonCentroid,
projectRegionToViewport, projectRegionToViewport,
type Quad,
type Region, type Region,
rectPolygon, rectPolygon,
regionBounds, regionBounds,
type Size,
type ViewportRect, type ViewportRect,
} from "./geometry"; } from "./geometry";
import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared"; import type { CssViewport } from "./geometry/coordinate-types";
import { cssViewport, GeometryContext } from "./geometry/frame-context";
import { type CdpRunner, cdpRunnerForTarget, isRpcError } from "./shared";
export interface NodeAddress { export interface NodeAddress {
target: CdpTarget; target: CdpTarget;
@@ -27,6 +24,8 @@ export interface NodeAddress {
} }
export interface ResolvedNodeGeometry { export interface ResolvedNodeGeometry {
/** The same top-level measurement used by projection, for screenshot coordinate adaptation. */
topViewport: CssViewport;
topVisibleRegions: Region; topVisibleRegions: Region;
topBounds: ViewportRect; topBounds: ViewportRect;
/** Point in the top-level tab viewport, used by root-target input events. */ /** Point in the top-level tab viewport, used by root-target input events. */
@@ -39,145 +38,16 @@ function geometryError(message: string): RpcError {
return { code: "cdp_failed", message }; return { code: "cdp_failed", message };
} }
function frameMap(graph: CdpFrameGraph): Map<string, CdpFrame> {
return new Map(graph.frames.map((frame) => [frame.frameId, frame]));
}
function targetRootFrame(byId: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
const seen = new Set<string>();
let current = frame;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
const parent = byId.get(current.parentFrameId);
if (!parent || parent.target.sessionId !== current.target.sessionId) break;
current = parent;
}
return current;
}
function frameAncestry(graph: CdpFrameGraph, frameId: string): CdpFrame[] | null {
const byId = frameMap(graph);
const path: CdpFrame[] = [];
const seen = new Set<string>();
let current = byId.get(frameId);
if (!current) return null;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
path.push(current);
const parent = byId.get(current.parentFrameId);
if (!parent) return null;
current = parent;
}
return path;
}
async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise<Size | null> {
const metrics = await sendToCdpTarget<{
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
layoutViewport?: { clientWidth?: number; clientHeight?: number };
}>(cdp, target, "Page.getLayoutMetrics", {});
const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
const width = viewport.clientWidth ?? 0;
const height = viewport.clientHeight ?? 0;
return width > 0 && height > 0 ? { width, height } : null;
}
async function ownerContentQuad(
cdp: CdpRunner,
parent: CdpFrame,
ownerBackendNodeId: number,
): Promise<Quad | null> {
const result = await sendToCdpTarget<{ model?: { content?: number[] } }>(
cdp,
parent.target,
"DOM.getBoxModel",
{ backendNodeId: ownerBackendNodeId },
);
return parseCdpQuad(result.model?.content);
}
async function sameTargetFrameClips(
cdp: CdpRunner,
byId: Map<string, CdpFrame>,
frame: CdpFrame,
targetRoot: CdpFrame,
): Promise<Polygon[] | null> {
const clips: Polygon[] = [];
let current = frame;
const seen = new Set<string>();
while (current.frameId !== targetRoot.frameId) {
if (seen.has(current.frameId) || !current.parentFrameId) return null;
seen.add(current.frameId);
const parent = byId.get(current.parentFrameId);
if (!parent || current.ownerBackendNodeId === undefined) return null;
const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId);
if (!clip) return null;
clips.push(clip);
current = parent;
}
return clips;
}
export async function resolveFrameProjection(
cdp: CdpRunner,
graph: CdpFrameGraph,
frameId: string,
): Promise<GeometryProjection | null> {
const byId = frameMap(graph);
const frame = byId.get(frameId);
if (!frame) return null;
let targetRoot = targetRootFrame(byId, frame);
if (!targetRoot) return null;
const sourceViewport = await targetViewport(cdp, targetRoot.target);
if (!sourceViewport) return null;
const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot);
if (!sourceClips) return null;
const edges: ProjectiveEdge[] = [];
while (targetRoot.parentFrameId) {
const parent = byId.get(targetRoot.parentFrameId);
if (!parent || targetRoot.ownerBackendNodeId === undefined) return null;
const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId);
if (!destinationQuad) return null;
const source =
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
if (!source) return null;
const parentTargetRoot = targetRootFrame(byId, parent);
if (!parentTargetRoot) return null;
const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot);
if (!destinationClips) return null;
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
targetRoot = parentTargetRoot;
}
const topViewport =
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
return topViewport ? { sourceClips, edges, topViewport } : null;
}
async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
if (!cdp.getFrameGraph) return null;
try {
return await cdp.getFrameGraph(tabId);
} catch (error) {
console.debug("[bsk frame-geometry] frame graph resolution failed", error);
return null;
}
}
async function scrollFrameOwners( async function scrollFrameOwners(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
graph: CdpFrameGraph, context: GeometryContext,
frameId: string, frameId: string,
): Promise<RpcError | null> { ): Promise<RpcError | null> {
const byId = frameMap(graph); const ancestry = await context.ancestry(frameId);
const ancestry = frameAncestry(graph, frameId);
if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`); if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`);
for (const child of [...ancestry].reverse()) { for (const child of [...ancestry].reverse()) {
const parent = child.parentFrameId ? byId.get(child.parentFrameId) : undefined; const parent = child.parentFrameId ? await context.frame(child.parentFrameId) : undefined;
if (!parent || child.ownerBackendNodeId === undefined) { if (!parent || child.ownerBackendNodeId === undefined) {
return geometryError(`could not resolve frame owner for ${child.frameId}`); return geometryError(`could not resolve frame owner for ${child.frameId}`);
} }
@@ -191,17 +61,20 @@ async function scrollFrameOwners(
return null; return null;
} }
async function scrollElementWithFrameGraph( async function scrollElementWithContext(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
target: CdpTarget, target: CdpTarget,
backendNodeId: number, backendNodeId: number,
frameId: string | undefined, frameId: string | undefined,
graph: CdpFrameGraph | null, context: GeometryContext,
): Promise<RpcError | null> { ): Promise<RpcError | null> {
if (frameId) { if (frameId) {
if (!graph) return geometryError(`could not resolve frame graph for ${frameId}`); const frame = await context.frame(frameId);
const error = await scrollFrameOwners(cdp, tabId, graph, frameId); if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(target)) {
return geometryError("node frame does not belong to its target");
}
const error = await scrollFrameOwners(cdp, tabId, context, frameId);
if (error) return error; if (error) return error;
} else if (target.sessionId) { } else if (target.sessionId) {
return geometryError("an OOPIF node address requires frameId"); return geometryError("an OOPIF node address requires frameId");
@@ -216,8 +89,15 @@ export async function scrollElementAndFramesIntoView(
backendNodeId: number, backendNodeId: number,
frameId?: string, frameId?: string,
): Promise<RpcError | null> { ): Promise<RpcError | null> {
const graph = frameId ? await loadFrameGraph(cdp, tabId) : null; if (target.tabId !== tabId) return geometryError("node target belongs to another tab");
return scrollElementWithFrameGraph(cdp, tabId, target, backendNodeId, frameId, graph); return scrollElementWithContext(
cdp,
tabId,
target,
backendNodeId,
frameId,
new GeometryContext(cdp, tabId),
);
} }
function largestRegion(regions: Region): Polygon | null { function largestRegion(regions: Region): Polygon | null {
@@ -240,23 +120,32 @@ export async function resolveNodeGeometry(
if (address.target.sessionId && !address.frameId) { if (address.target.sessionId && !address.frameId) {
return geometryError("an OOPIF node address requires frameId"); return geometryError("an OOPIF node address requires frameId");
} }
const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null; if (address.target.tabId !== tabId) return geometryError("node target belongs to another tab");
const context = new GeometryContext(cdp, tabId);
const graph = address.frameId ? await context.graph() : null;
if (address.frameId && graph) {
const frame = await context.frame(address.frameId);
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(address.target)) {
return geometryError("node frame does not belong to its target");
}
}
if (address.frameId && !graph) { if (address.frameId && !graph) {
return geometryError(`could not resolve frame graph for ${address.frameId}`); return geometryError(`could not resolve frame graph for ${address.frameId}`);
} }
if (options.scrollIntoView) { if (options.scrollIntoView) {
const scrollError = await scrollElementWithFrameGraph( const scrollError = await scrollElementWithContext(
cdp, cdp,
tabId, tabId,
address.target, address.target,
address.backendNodeId, address.backendNodeId,
address.frameId, address.frameId,
graph, context,
); );
if (scrollError) return scrollError; if (scrollError) return scrollError;
} }
// Only topology was read above. Begin live measurements after scrolling.
const localRegion = await nodeContentRegion( const localRegion = await nodeContentRegion(
cdpRunnerForTarget(cdp, address.target), cdpRunnerForTarget(cdp, address.target),
tabId, tabId,
@@ -267,7 +156,7 @@ export async function resolveNodeGeometry(
let topVisibleRegions: Region; let topVisibleRegions: Region;
let targetActionPoint: Point | null = null; let targetActionPoint: Point | null = null;
if (address.frameId && graph) { if (address.frameId && graph) {
const projection = await resolveFrameProjection(cdp, graph, address.frameId); const projection = await context.targetProjection(address.frameId);
if (!projection) if (!projection)
return geometryError(`could not resolve frame geometry for ${address.frameId}`); return geometryError(`could not resolve frame geometry for ${address.frameId}`);
topVisibleRegions = projectRegionToViewport(localRegion, projection); topVisibleRegions = projectRegionToViewport(localRegion, projection);
@@ -290,7 +179,7 @@ export async function resolveNodeGeometry(
targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null; targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null;
} }
} else { } else {
const viewport = await targetViewport(cdp, address.target); const viewport = await context.viewport(address.target);
if (!viewport) return geometryError("could not resolve top viewport geometry"); if (!viewport) return geometryError("could not resolve top viewport geometry");
topVisibleRegions = localRegion topVisibleRegions = localRegion
.map((polygon) => .map((polygon) =>
@@ -309,7 +198,13 @@ export async function resolveNodeGeometry(
return { code: "permission_denied", message: "element not visible in its target" }; return { code: "permission_denied", message: "element not visible in its target" };
} }
targetActionPoint ??= actionPoint; targetActionPoint ??= actionPoint;
return { topVisibleRegions, topBounds, actionPoint, targetActionPoint }; return {
topVisibleRegions,
topBounds,
actionPoint,
targetActionPoint,
topViewport: cssViewport(await context.layoutMetrics({ tabId })),
};
} catch (error) { } catch (error) {
return geometryError(error instanceof Error ? error.message : String(error)); return geometryError(error instanceof Error ? error.message : String(error));
} }
-19
View File
@@ -245,22 +245,3 @@ export function projectRectToViewport(
if (!rect) return null; if (!rect) return null;
return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection)); return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection));
} }
export function childFrameProjection(
parent: GeometryProjection,
ownerRectInParent: { x: number; y: number; w: number; h: number },
): GeometryProjection {
const destinationQuad = rectPolygon(ownerRectInParent);
return {
sourceClips: [],
edges: [
{
sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
destinationQuad,
destinationClips: parent.sourceClips,
},
...parent.edges,
],
topViewport: parent.topViewport,
};
}
@@ -0,0 +1,102 @@
import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
import type { GeometryProjection, ViewportRect } from "../geometry";
import { projectRectToViewport } from "../geometry";
export interface CoordinateOwner {
target: CdpTarget;
frameId?: string;
}
export interface FrameViewportRect {
space: "frame-viewport-css";
owner: CoordinateOwner;
rect: ViewportRect;
}
export interface SnapshotProjection {
source: CoordinateOwner;
geometry: GeometryProjection;
}
/** A failed owner read is different from a successfully projected, clipped-out rect. */
export interface UnavailableFrameProjection {
status: "unavailable";
source: CoordinateOwner;
ownerBackendNodeId: number;
}
export type SnapshotProjectionResult =
| { status: "available"; projection: SnapshotProjection }
| UnavailableFrameProjection;
/** Descendants inherit the failed boundary without attempting another owner read. */
export type FrameProjectionIssue =
| UnavailableFrameProjection
| { status: "blocked"; source: CoordinateOwner; cause: UnavailableFrameProjection };
export type FrameProjectionState = SnapshotProjectionResult | FrameProjectionIssue;
/** DOMSnapshot bounds are document-relative CSS pixels, independent of raster DPR. */
export function snapshotViewportRect(
bounds: number[],
owner: CoordinateOwner,
scroll: { x: number; y: number },
): FrameViewportRect | null {
if (bounds.length < 4 || !bounds.slice(0, 4).every(Number.isFinite)) return null;
const [x, y, width, height] = bounds;
if (width <= 0 || height <= 0 || !Number.isFinite(scroll.x) || !Number.isFinite(scroll.y)) {
return null;
}
return {
space: "frame-viewport-css",
owner,
rect: { x: x - scroll.x, y: y - scroll.y, width, height },
};
}
export function projectSnapshotRect(
input: FrameViewportRect,
projection: SnapshotProjection,
): ViewportRect | null {
if (
input.owner.frameId !== projection.source.frameId ||
cdpTargetKey(input.owner.target) !== cdpTargetKey(projection.source.target)
) {
return null;
}
const { x, y, width: w, height: h } = input.rect;
return projectRectToViewport({ x, y, w, h }, projection.geometry);
}
/** CSS viewport metadata; browser zoom is distinct from raster devicePixelRatio. */
export interface CssViewport {
width: number;
height: number;
scrollX: number;
scrollY: number;
cssToDip: number;
}
/** Page.captureScreenshot takes page-relative DIP, not viewport-relative CSS pixels. */
export function screenshotPageRect(
rect: ViewportRect,
viewport: CssViewport,
): { space: "page-dip"; rect: ViewportRect } | null {
const { cssToDip, scrollX, scrollY } = viewport;
if (
![rect.x, rect.y, rect.width, rect.height, cssToDip, scrollX, scrollY].every(Number.isFinite) ||
cssToDip <= 0 ||
rect.width <= 0 ||
rect.height <= 0
)
return null;
return {
space: "page-dip",
rect: {
x: (rect.x + scrollX) * cssToDip,
y: (rect.y + scrollY) * cssToDip,
width: rect.width * cssToDip,
height: rect.height * cssToDip,
},
};
}
@@ -0,0 +1,333 @@
import {
type CdpFrame,
type CdpFrameGraph,
type CdpTarget,
cdpTargetKey,
} from "@/browser-driver/frame-graph";
import {
type GeometryProjection,
type Polygon,
type ProjectiveEdge,
parseCdpQuad,
type Quad,
type Size,
} from "../geometry";
import { type CdpRunner, sendToCdpTarget } from "../shared";
import type { CoordinateOwner, CssViewport, SnapshotProjectionResult } from "./coordinate-types";
import { readSnapshotOwnerSizes } from "./snapshot-owner-sizes";
export interface LayoutMetrics {
cssVisualViewport?: { zoom?: number };
visualViewport?: { zoom?: number };
cssLayoutViewport?: {
clientWidth?: number;
clientHeight?: number;
pageX?: number;
pageY?: number;
};
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
}
/** The legacy field is a compatibility fallback, never a source of raster DPR. */
export function cssViewport(metrics: LayoutMetrics): CssViewport {
const value = metrics.cssLayoutViewport ?? metrics.layoutViewport;
return {
width: value?.clientWidth ?? 0,
height: value?.clientHeight ?? 0,
scrollX: value?.pageX ?? 0,
scrollY: value?.pageY ?? 0,
cssToDip: metrics.cssVisualViewport?.zoom ?? metrics.visualViewport?.zoom ?? 1,
};
}
/** One read-only measurement phase. Discard after scrolling or any later operation. */
export class GeometryContext {
private readonly metrics = new Map<string, Promise<LayoutMetrics>>();
private readonly owners = new Map<string, Promise<Quad | null>>();
private readonly projections = new Map<string, Promise<GeometryProjection | null>>();
private readonly snapshotOwners = new Map<string, Promise<{ quad: Quad; size: Size } | null>>();
private readonly sizeBatches = new Map<string, () => Promise<Map<number, Size>>>();
private active = 0;
private readonly waiting: Array<() => void> = [];
private graphPromise?: Promise<CdpFrameGraph | null>;
private framesPromise?: Promise<Map<string, CdpFrame>>;
constructor(
private readonly cdp: CdpRunner,
private readonly tabId: number,
graph?: CdpFrameGraph,
private readonly signal?: AbortSignal,
) {
if (graph) this.graphPromise = Promise.resolve(graph);
}
private async request<T>(
target: CdpTarget,
method: string,
params: object,
cleanup = false,
): Promise<T> {
if (!cleanup && this.signal?.aborted) throw new DOMException("geometry aborted", "AbortError");
if (this.active >= 4) await new Promise<void>((resolve) => this.waiting.push(resolve));
else this.active++;
try {
if (!cleanup && this.signal?.aborted)
throw new DOMException("geometry aborted", "AbortError");
return await sendToCdpTarget<T>(this.cdp, target, method, params);
} finally {
const next = this.waiting.shift();
if (next) next();
else this.active--;
}
}
graph(): Promise<CdpFrameGraph | null> {
this.graphPromise ??=
this.cdp.getFrameGraph?.(this.tabId).catch(() => null) ?? Promise.resolve(null);
return this.graphPromise;
}
private frames(): Promise<Map<string, CdpFrame>> {
this.framesPromise ??= this.graph().then(
(graph) => new Map(graph?.frames.map((frame) => [frame.frameId, frame]) ?? []),
);
return this.framesPromise;
}
async frame(frameId: string): Promise<CdpFrame | undefined> {
return (await this.frames()).get(frameId);
}
/** Topology only: safe to resolve before the caller scrolls frame owners. */
async ancestry(frameId: string): Promise<CdpFrame[] | null> {
const frames = await this.frames();
const path: CdpFrame[] = [];
const seen = new Set<string>();
let current = frames.get(frameId);
if (!current) return null;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
path.push(current);
const parent = frames.get(current.parentFrameId);
if (!parent) return null;
current = parent;
}
return path;
}
layoutMetrics(target: CdpTarget): Promise<LayoutMetrics> {
const key = cdpTargetKey(target);
let promise = this.metrics.get(key);
if (!promise) {
promise = this.request<LayoutMetrics>(target, "Page.getLayoutMetrics", {});
this.metrics.set(key, promise);
}
return promise;
}
async viewport(target: CdpTarget): Promise<Size | null> {
const { width, height } = cssViewport(await this.layoutMetrics(target));
return Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0
? { width, height }
: null;
}
ownerContent(target: CdpTarget, backendNodeId: number): Promise<Quad | null> {
const key = `${cdpTargetKey(target)}:${backendNodeId}`;
let promise = this.owners.get(key);
if (!promise) {
promise = this.request<{ model?: { content?: number[] } }>(target, "DOM.getBoxModel", {
backendNodeId,
}).then((reply) => parseCdpQuad(reply.model?.content));
this.owners.set(key, promise);
}
return promise;
}
/** Same-target owner quads are already target-relative, including nested owners. */
async snapshotProjection(
source: CoordinateOwner,
ownerBackendNodeId: number,
ancestorClips: Polygon[],
viewport: Size,
): Promise<SnapshotProjectionResult> {
const key = `${cdpTargetKey(source.target)}:${ownerBackendNodeId}`;
let promise = this.snapshotOwners.get(key);
if (!promise) {
promise = this.snapshotOwner(source.target, ownerBackendNodeId);
this.snapshotOwners.set(key, promise);
}
let owner: { quad: Quad; size: Size } | null;
try {
owner = await promise;
} catch (error) {
if (error && typeof error === "object" && "name" in error && error.name === "AbortError")
throw error;
console.debug("[bsk geometry] frame owner unavailable", error);
owner = null;
}
return owner
? {
status: "available",
projection: {
source,
geometry: {
sourceClips: [],
edges: [
{
sourceViewport: owner.size,
destinationQuad: owner.quad,
destinationClips: ancestorClips,
},
],
topViewport: viewport,
},
},
}
: { status: "unavailable", source, ownerBackendNodeId };
}
/** Register one target's snapshot owners; the batch starts only when a
* projection actually needs a size. Single owners keep the cheaper old path. */
registerSnapshotOwners(target: CdpTarget, owners: ReadonlySet<number>): void {
const key = cdpTargetKey(target);
if (owners.size < 2 || this.sizeBatches.has(key)) return;
let pending: Promise<Map<number, Size>> | undefined;
this.sizeBatches.set(key, () => {
pending ??= readSnapshotOwnerSizes(
(method, params, cleanup) => this.request(target, method, params, cleanup),
owners,
).catch((error) => {
if (error && typeof error === "object" && "name" in error && error.name === "AbortError")
throw error;
return new Map<number, Size>();
});
return pending;
});
}
private async snapshotOwner(
target: CdpTarget,
backendNodeId: number,
): Promise<{ quad: Quad; size: Size } | null> {
const quad = await this.ownerContent(target, backendNodeId);
if (!quad) return null;
const batchSize = (await this.sizeBatches.get(cdpTargetKey(target))?.())?.get(backendNodeId);
if (this.signal?.aborted) throw new DOMException("geometry aborted", "AbortError");
if (batchSize) return { quad, size: batchSize };
const resolved = await this.request<{ object?: { objectId?: string } }>(
target,
"DOM.resolveNode",
{ backendNodeId },
);
const objectId = resolved.object?.objectId;
if (!objectId) return null;
try {
const reply = await this.request<{ result?: { value?: Size }; exceptionDetails?: unknown }>(
target,
"Runtime.callFunctionOn",
{
objectId,
functionDeclaration: `function() {
if (!this.isConnected) return null;
const style = getComputedStyle(this);
return {
width: this.clientWidth - parseFloat(style.paddingLeft) - parseFloat(style.paddingRight),
height: this.clientHeight - parseFloat(style.paddingTop) - parseFloat(style.paddingBottom)
};
}`,
returnByValue: true,
},
);
const size = reply.result?.value;
return !reply.exceptionDetails &&
size &&
Number.isFinite(size.width) &&
Number.isFinite(size.height) &&
size.width > 0 &&
size.height > 0
? { quad, size }
: null;
} finally {
// Cleanup must still run if cancellation arrived after resolving the object.
await this.request(target, "Runtime.releaseObject", { objectId }, true).catch(() => {});
}
}
private targetRoot(frames: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
const seen = new Set<string>();
let current = frame;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
const parent = frames.get(current.parentFrameId);
if (!parent) return null;
if (cdpTargetKey(parent.target) !== cdpTargetKey(current.target)) break;
current = parent;
}
return current;
}
private async clips(
frames: Map<string, CdpFrame>,
frame: CdpFrame,
root: CdpFrame,
): Promise<Polygon[] | null> {
const clips: Polygon[] = [];
let current = frame;
const seen = new Set<string>();
while (current.frameId !== root.frameId) {
if (seen.has(current.frameId) || !current.parentFrameId) return null;
seen.add(current.frameId);
const parent = frames.get(current.parentFrameId);
if (!parent || current.ownerBackendNodeId === undefined) return null;
const quad = await this.ownerContent(parent.target, current.ownerBackendNodeId);
if (!quad) return null;
clips.push(quad);
current = parent;
}
return clips;
}
/** Input is already in the frame's CDP target viewport, not its local viewport. */
targetProjection(frameId: string): Promise<GeometryProjection | null> {
let promise = this.projections.get(frameId);
if (!promise) {
promise = this.buildTargetProjection(frameId);
this.projections.set(frameId, promise);
}
return promise;
}
private async buildTargetProjection(frameId: string): Promise<GeometryProjection | null> {
const frames = await this.frames();
const frame = frames.get(frameId);
if (!frame) return null;
let root = this.targetRoot(frames, frame);
if (!root) return null;
const sourceViewport = await this.viewport(root.target);
if (!sourceViewport) return null;
const sourceClips = await this.clips(frames, frame, root);
if (!sourceClips) return null;
const edges: ProjectiveEdge[] = [];
const seen = new Set<string>();
while (root.parentFrameId) {
if (seen.has(root.frameId)) return null;
seen.add(root.frameId);
const parent = frames.get(root.parentFrameId);
if (!parent || root.ownerBackendNodeId === undefined) return null;
const destinationQuad = await this.ownerContent(parent.target, root.ownerBackendNodeId);
const source = await this.viewport(root.target);
const parentRoot = this.targetRoot(frames, parent);
if (!destinationQuad || !source || !parentRoot) return null;
const destinationClips = await this.clips(frames, parent, parentRoot);
if (!destinationClips) return null;
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
root = parentRoot;
}
const topViewport = await this.viewport(root.target);
return topViewport ? { sourceClips, edges, topViewport } : null;
}
}
@@ -0,0 +1,74 @@
import type { Size } from "../geometry";
interface SerializedValue {
type: string;
value?: unknown;
}
type Send = <T>(method: string, params: object, cleanup?: boolean) => Promise<T>;
/** Batch only accessible frame owners. Missing/shadow/cross-origin owners keep
* the existing node-scoped read; no page-wide style columns are requested. */
export async function readSnapshotOwnerSizes(
send: Send,
owners: ReadonlySet<number>,
): Promise<Map<number, Size>> {
const sizes = new Map<number, Size>();
const objectGroup = `bsk-frame-sizes-${crypto.randomUUID()}`;
try {
const reply = await send<{ result?: { deepSerializedValue?: SerializedValue } }>(
"Runtime.evaluate",
{
expression: `(() => {
const result = [];
const documents = [document];
for (let i = 0; i < documents.length && result.length < ${owners.size}; i++) {
for (const owner of documents[i].querySelectorAll('iframe,frame')) {
if (result.length >= ${owners.size}) break;
if (!owner.isConnected) continue;
const style = getComputedStyle(owner);
result.push([owner, JSON.stringify({
width: owner.clientWidth - parseFloat(style.paddingLeft) - parseFloat(style.paddingRight),
height: owner.clientHeight - parseFloat(style.paddingTop) - parseFloat(style.paddingBottom)
})]);
try { if (owner.contentDocument) documents.push(owner.contentDocument); } catch {}
}
}
return result;
})()`,
objectGroup,
serializationOptions: {
serialization: "deep",
maxDepth: 3,
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
},
},
);
const result = reply.result?.deepSerializedValue;
if (result?.type !== "array" || !Array.isArray(result.value)) return sizes;
for (const entry of result.value as SerializedValue[]) {
if (entry.type !== "array" || !Array.isArray(entry.value)) continue;
const [node, data] = entry.value as SerializedValue[];
if (node?.type !== "node" || data?.type !== "string" || typeof data.value !== "string")
continue;
const id = (node.value as { backendNodeId?: number } | undefined)?.backendNodeId;
if (id === undefined || !owners.has(id)) continue;
try {
const size = JSON.parse(data.value) as Size;
if (
size &&
Number.isFinite(size.width) &&
Number.isFinite(size.height) &&
size.width > 0 &&
size.height > 0
)
sizes.set(id, size);
} catch {
/* An invalid entry uses the existing individual read. */
}
}
} finally {
await send("Runtime.releaseObjectGroup", { objectGroup }, true).catch(() => {});
}
return sizes;
}
+33 -20
View File
@@ -34,6 +34,8 @@ import type {
import { attachDialogs, markDialogCursor } from "./dialogs"; import { attachDialogs, markDialogCursor } from "./dialogs";
import { rpcError } from "./errors"; import { rpcError } from "./errors";
import { resolveNodeGeometry } from "./frame-geometry"; import { resolveNodeGeometry } from "./frame-geometry";
import { screenshotPageRect } from "./geometry/coordinate-types";
import { cssViewport, GeometryContext } from "./geometry/frame-context";
import { import {
type ChromeTabsApi, type ChromeTabsApi,
enforceToolTargetScope, enforceToolTargetScope,
@@ -197,15 +199,14 @@ async function captureElementScreenshot(
if (isRpcError(geometry)) return geometry; if (isRpcError(geometry)) return geometry;
if (signal?.aborted) return cancelled("screenshot"); if (signal?.aborted) return cancelled("screenshot");
const rect = geometry.topBounds; const rect = geometry.topBounds;
const clip = screenshotPageRect(rect, geometry.topViewport);
if (!clip) return { code: "cdp_failed", message: "invalid screenshot coordinate space" };
try { try {
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", { const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
format: "png", format: "png",
clip: { clip: {
x: rect.x, ...clip.rect,
y: rect.y,
width: rect.width,
height: rect.height,
scale: 1, scale: 1,
}, },
}); });
@@ -675,6 +676,7 @@ function legacyFrameDocuments(
...(pageUrl ? { url: pageUrl } : {}), ...(pageUrl ? { url: pageUrl } : {}),
axNodes: [], axNodes: [],
domNodes: captured.nodes.map((node) => ({ ...node, frameId: node.frameId ?? rootFrameId })), domNodes: captured.nodes.map((node) => ({ ...node, frameId: node.frameId ?? rootFrameId })),
excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(rootFrameId),
}, },
]; ];
const pending = [...captured.iframeNodes.entries()]; const pending = [...captured.iframeNodes.entries()];
@@ -698,6 +700,7 @@ function legacyFrameDocuments(
target: parent.target, target: parent.target,
axNodes: [], axNodes: [],
domNodes: domNodes.map((node) => ({ ...node, frameId: node.frameId ?? frameId })), domNodes: domNodes.map((node) => ({ ...node, frameId: node.frameId ?? frameId })),
excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(frameId),
}); });
pending.splice(index, 1); pending.splice(index, 1);
progress = true; progress = true;
@@ -984,26 +987,19 @@ function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedVie
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() }; return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
} }
interface LayoutMetricsViewportReply {
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
layoutViewport?: { clientWidth?: number; clientHeight?: number };
}
async function fallbackCapturedViewModel( async function fallbackCapturedViewModel(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
geometry: GeometryContext,
signal?: AbortSignal, signal?: AbortSignal,
): Promise<CapturedViewModel> { ): Promise<CapturedViewModel> {
throwIfAborted(signal, "observation"); throwIfAborted(signal, "observation");
let viewport = { width: 0, height: 0 }; let viewport = { width: 0, height: 0 };
try { try {
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {}); const metrics = await geometry.layoutMetrics({ tabId });
throwIfAborted(signal, "observation"); throwIfAborted(signal, "observation");
const source = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {}; const source = cssViewport(metrics);
viewport = { viewport = { width: source.width, height: source.height };
width: source.clientWidth ?? 0,
height: source.clientHeight ?? 0,
};
} catch (error) { } catch (error) {
if (isAbortError(error)) throw error; if (isAbortError(error)) throw error;
} }
@@ -1016,12 +1012,13 @@ async function captureForVom(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
options: CaptureVomObservationOptions, options: CaptureVomObservationOptions,
geometry: GeometryContext,
): Promise<CapturedViewModel> { ): Promise<CapturedViewModel> {
try { try {
return await captureViewModel(cdp, tabId, { signal: options.signal }); return await captureViewModel(cdp, tabId, { signal: options.signal, geometry });
} catch (error) { } catch (error) {
if (isAbortError(error)) throw error; if (isAbortError(error)) throw error;
return fallbackCapturedViewModel(cdp, tabId, options.signal); return fallbackCapturedViewModel(cdp, tabId, geometry, options.signal);
} }
} }
@@ -1096,9 +1093,16 @@ export async function captureVomObservation(
throwIfAborted(options.signal, "observation"); throwIfAborted(options.signal, "observation");
await cdp.ensureAttachedToUrl?.(tabId, url); await cdp.ensureAttachedToUrl?.(tabId, url);
throwIfAborted(options.signal, "observation"); throwIfAborted(options.signal, "observation");
const captured = await captureForVom(cdp, tabId, options); const geometry = new GeometryContext(cdp, tabId, undefined, options.signal);
const captured = await captureForVom(cdp, tabId, options, geometry);
throwIfAborted(options.signal, "observation"); throwIfAborted(options.signal, "observation");
const documents = await captureFrameData<CdpAxNode>(cdp, tabId, captured, options.signal); const documents = await captureFrameData<CdpAxNode>(
cdp,
tabId,
captured,
options.signal,
geometry,
);
throwIfAborted(options.signal, "observation"); throwIfAborted(options.signal, "observation");
const normalizedDocuments = const normalizedDocuments =
documents.length === 1 && captured.iframeNodes.size > 0 documents.length === 1 && captured.iframeNodes.size > 0
@@ -1131,12 +1135,21 @@ export async function captureVomObservation(
captured, captured,
hoverProbes.surfaceProbes, hoverProbes.surfaceProbes,
); );
// Reserve space using the renderer's character-based token estimate. Like VOM
// headers, this integrity notice remains visible even under a tiny token budget.
const geometryNotice = captured.frameGeometryIssues?.length
? "@warning iframe geometry incomplete: some frame content has no top-level coordinates.\n"
: "";
const rendered = renderVom(decoratedScene, { const rendered = renderVom(decoratedScene, {
maxDepth: options.maxDepth, maxDepth: options.maxDepth,
maxTokens: options.maxTokens, maxTokens:
!geometryNotice || options.maxTokens === undefined
? options.maxTokens
: Math.max(0, options.maxTokens - Math.ceil(geometryNotice.length / 4)),
redactValues: options.redactValues, redactValues: options.redactValues,
activeRegionPolicy: options.activeRegionPolicy, activeRegionPolicy: options.activeRegionPolicy,
}); });
if (geometryNotice) rendered.text += `\n${geometryNotice.trimEnd()}`;
throwIfAborted(options.signal, "observation"); throwIfAborted(options.signal, "observation");
return projectRecordSafeObservation({ return projectRecordSafeObservation({
rootFrameId: captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root", rootFrameId: captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root",
@@ -1,5 +1,6 @@
import { describe, expect, it, vi } from "vitest"; import { describe, expect, it, vi } from "vitest";
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge"; import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
import type { CdpRunner } from "../../shared";
import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture"; import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
// Minimal but format-accurate captureSnapshot reply: a body with one // Minimal but format-accurate captureSnapshot reply: a body with one
@@ -527,6 +528,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "main",
nodes: { nodes: {
parentIndex: [-1, 0, 1, 0, 3, 4], parentIndex: [-1, 0, 1, 0, 3, 4],
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")], nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
@@ -557,8 +559,12 @@ describe("captureViewModel", () => {
], ],
}; };
const { excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4); const { excludedBackendNodeIds, frameExcludedBackendNodeIds } = await captureViewModel(
makeCdp(snapshot),
4,
);
expect(excludedBackendNodeIds).toEqual(new Set([13, 14, 15])); expect(excludedBackendNodeIds).toEqual(new Set([13, 14, 15]));
expect(frameExcludedBackendNodeIds?.get("main")).toEqual(excludedBackendNodeIds);
}); });
it("excludes overlay host matched by tag name only", async () => { it("excludes overlay host matched by tag name only", async () => {
@@ -754,7 +760,7 @@ describe("captureViewModel", () => {
expect(div?.rect).toMatchObject({ y: 0, h: 600 }); expect(div?.rect).toMatchObject({ y: 0, h: 600 });
}); });
it("normalizes device-pixel bounds by devicePixelRatio", async () => { it("keeps CSS snapshot bounds independent of legacy metrics ratio", async () => {
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"]; const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
const i = (s: string) => S.indexOf(s); const i = (s: string) => S.indexOf(s);
const snapshot = { const snapshot = {
@@ -802,6 +808,12 @@ describe("captureViewModel", () => {
w: 1000, w: 1000,
h: 800, h: 800,
}); });
expect(nodes.find((n) => n.backendNodeId === 12)?.localRect).toEqual({
x: 0,
y: 0,
w: 2000,
h: 1600,
});
}); });
it("extracts textContent from #text child nodes via nodeValue", async () => { it("extracts textContent from #text child nodes via nodeValue", async () => {
@@ -950,6 +962,12 @@ describe("captureViewModel", () => {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
}; };
} }
if (method === "DOM.getBoxModel")
return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } };
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 800, height: 400 } } };
if (method === "Runtime.releaseObject") return {};
throw new Error(method); throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
@@ -979,7 +997,7 @@ describe("captureViewModel", () => {
expect(frameParentIds?.get("child-frame")).toBe("main-frame"); expect(frameParentIds?.get("child-frame")).toBe("main-frame");
}); });
it("captures iframe documents without requiring owner-frame geometry", async () => { it("projects iframe documents independently of owner snapshot geometry", async () => {
const S = [ const S = [
"html", "html",
"body", "body",
@@ -1040,6 +1058,12 @@ describe("captureViewModel", () => {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
}; };
} }
if (method === "DOM.getBoxModel")
return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } };
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 800, height: 400 } } };
if (method === "Runtime.releaseObject") return {};
throw new Error(method); throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
@@ -1048,7 +1072,7 @@ describe("captureViewModel", () => {
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101); const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 }); expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
expect(input?.rect).toBeNull(); expect(input?.rect).toEqual({ x: 120, y: 320, w: 120, h: 60 });
}); });
it("recursively normalizes nested iframe sub-documents", async () => { it("recursively normalizes nested iframe sub-documents", async () => {
@@ -1127,16 +1151,43 @@ describe("captureViewModel", () => {
], ],
}; };
const cdp = { const cdp = {
send: vi.fn(async (_t: number, method: string) => { send: vi.fn(
if (method === "DOMSnapshot.enable") return {}; async (
if (method === "DOMSnapshot.captureSnapshot") return snapshot; _t: number,
if (method === "Page.getLayoutMetrics") { method: string,
return { params?: { backendNodeId?: number; objectId?: string },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, ) => {
}; if (method === "DOMSnapshot.enable") return {};
} if (method === "DOMSnapshot.captureSnapshot") return snapshot;
throw new Error(method); if (method === "Page.getLayoutMetrics") {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, return {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "DOM.getBoxModel")
return {
model: {
content:
params?.backendNodeId === 13
? [10, 20, 310, 20, 310, 220, 10, 220]
: [15, 26, 115, 26, 115, 106, 15, 106],
},
};
if (method === "DOM.resolveNode")
return { object: { objectId: String(params?.backendNodeId) } };
if (method === "Runtime.callFunctionOn")
return {
result: {
value:
params?.objectId === "13"
? { width: 300, height: 200 }
: { width: 100, height: 80 },
},
};
if (method === "Runtime.releaseObject") return {};
throw new Error(method);
},
) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { iframeNodes } = await captureViewModel(cdp, 4); const { iframeNodes } = await captureViewModel(cdp, 4);
@@ -1150,7 +1201,7 @@ describe("captureViewModel", () => {
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 }); expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
}); });
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => { it("subtracts CSS scroll once without dividing snapshot bounds by a metrics ratio", async () => {
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"]; const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
const i = (s: string) => S.indexOf(s); const i = (s: string) => S.indexOf(s);
const snapshot = { const snapshot = {
@@ -1192,7 +1243,7 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { nodes } = await captureViewModel(cdp, 4); const { nodes } = await captureViewModel(cdp, 4);
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 / 2 - 100); expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 - 100);
}); });
it("collectOverlayExcludedBackendIds walks the pierced overlay host subtree", async () => { it("collectOverlayExcludedBackendIds walks the pierced overlay host subtree", async () => {
@@ -1217,3 +1268,194 @@ describe("captureViewModel", () => {
expect(excluded).toEqual(new Set([200, 201, 202])); expect(excluded).toEqual(new Set([200, 201, 202]));
}); });
}); });
// Six sibling frames and one nested frame exercise scheduling through the real capture entry.
function siblingCaptureFixture(
beforeReply: (method: string, params: Record<string, unknown>) => Promise<void> = async () => {},
) {
const document = (id: number, owners: number[], childIndexes: number[]) => ({
frameId: `frame-${id}`,
nodes: {
parentIndex: [-1, ...owners.map(() => 0)],
nodeName: [0, ...owners.map(() => 1)],
backendNodeId: [1000 + id, ...owners],
attributes: [[], ...owners.map(() => [])],
contentDocumentIndex: { index: owners.map((_, i) => i + 1), value: childIndexes },
},
layout: {
nodeIndex: [0, ...owners.map((_, i) => i + 1)],
bounds: [[0, 0, 200, 100], ...owners.map(() => [0, 0, 200, 100])],
},
});
const snapshot = {
strings: ["body", "iframe"],
documents: [
document(0, [100, 101, 102, 103, 104, 105], [1, 2, 3, 4, 5, 6]),
document(1, [200], [7]),
...Array.from({ length: 6 }, (_, i) => document(i + 2, [], [])),
],
};
let active = 0;
let peak = 0;
const send = vi.fn(async (_tabId: number, method: string, params: object = {}) => {
const args = params as Record<string, unknown>;
active++;
peak = Math.max(peak, active);
try {
await beforeReply(method, args);
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
if (method === "DOM.getBoxModel")
return { model: { content: [0, 0, 200, 0, 200, 100, 0, 100] } };
if (method === "DOM.resolveNode") return { object: { objectId: String(args.backendNodeId) } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 200, height: 100 } } };
return {};
} finally {
active--;
}
});
return {
cdp: { send: send as CdpRunner["send"] },
send,
peak: () => peak,
active: () => active,
measured: () =>
send.mock.calls
.filter(([, method]) => method === "DOM.getBoxModel")
.map(([, , params]) => (params as { backendNodeId: number }).backendNodeId),
};
}
describe("sibling frame measurement scheduling", () => {
it("batches same-target owner sizes without changing projected results", async () => {
const fixture = siblingCaptureFixture();
const original = fixture.cdp.send;
const methods: string[] = [];
fixture.cdp.send = async (tabId, method, params) => {
methods.push(method);
if (method === "Runtime.evaluate")
return {
result: {
deepSerializedValue: {
type: "array",
value: [100, 101, 102, 103, 104, 105, 200].map((backendNodeId) => ({
type: "array",
value: [
{ type: "node", value: { backendNodeId } },
{ type: "string", value: JSON.stringify({ width: 200, height: 100 }) },
],
})),
},
},
} as never;
return original(tabId, method, params);
};
const captured = await captureViewModel(fixture.cdp, 4);
expect(captured.frameGeometryIssues).toEqual([]);
expect([...captured.iframeNodes.keys()]).toEqual([100, 200, 101, 102, 103, 104, 105]);
expect(captured.iframeNodes.get(200)?.[0].rect).toEqual({ x: 0, y: 0, w: 200, h: 100 });
expect(methods.filter((method) => method === "DOM.getBoxModel")).toHaveLength(7);
expect(methods.filter((method) => method === "Runtime.evaluate")).toHaveLength(1);
expect(methods.filter((method) => method === "Runtime.releaseObjectGroup")).toHaveLength(1);
expect(methods).not.toContain("DOM.resolveNode");
expect(methods).not.toContain("Runtime.callFunctionOn");
});
it("bounds concurrent reads, fills free slots and preserves depth-first output despite reordered replies", async () => {
const pending = new Map<number, () => void>();
const fixture = siblingCaptureFixture(async (method, params) => {
if (method === "DOM.getBoxModel" && Number(params.backendNodeId) < 200)
await new Promise<void>((resolve) => pending.set(Number(params.backendNodeId), resolve));
});
const capture = captureViewModel(fixture.cdp, 4);
await vi.waitFor(() => expect(pending.size).toBe(4));
expect(fixture.measured()).toEqual([100, 101, 102, 103]);
pending.get(103)!();
await vi.waitFor(() => expect(pending.has(104)).toBe(true));
pending.get(104)!();
await vi.waitFor(() => expect(pending.has(105)).toBe(true));
expect(fixture.measured()).not.toContain(200);
for (const id of [105, 102, 101, 100]) pending.get(id)!();
const captured = await capture;
expect(captured.frameGeometryIssues).toEqual([]);
expect(fixture.peak()).toBe(4);
expect(fixture.active()).toBe(0);
expect(fixture.measured()).toEqual([100, 101, 102, 103, 104, 105, 200]);
expect([...captured.iframeNodes.keys()]).toEqual([100, 200, 101, 102, 103, 104, 105]);
expect(captured.iframeNodes.get(200)?.[0].rect).toEqual({ x: 0, y: 0, w: 200, h: 100 });
for (const method of [
"DOM.getBoxModel",
"DOM.resolveNode",
"Runtime.callFunctionOn",
"Runtime.releaseObject",
])
expect(fixture.send.mock.calls.filter(([, name]) => name === method)).toHaveLength(7);
});
it("does not measure descendants of a failed owner and retains sibling geometry and local nodes", async () => {
const fixture = siblingCaptureFixture(async (method, params) => {
if (method === "DOM.resolveNode" && params.backendNodeId === 100)
throw new Error("owner replaced");
});
const captured = await captureViewModel(fixture.cdp, 4);
expect(captured.frameGeometryIssues).toEqual([
{
status: "unavailable",
source: { target: { tabId: 4 }, frameId: "frame-1" },
ownerBackendNodeId: 100,
},
{
status: "blocked",
source: { target: { tabId: 4 }, frameId: "frame-7" },
cause: captured.frameGeometryIssues?.[0],
},
]);
expect(fixture.measured()).not.toContain(200);
expect(captured.iframeNodes.get(200)?.[0]).toMatchObject({
tag: "body",
rect: null,
localRect: { x: 0, y: 0, w: 200, h: 100 },
});
expect(captured.iframeNodes.get(101)?.[0].rect).toEqual({ x: 0, y: 0, w: 200, h: 100 });
expect([...captured.iframeNodes.keys()]).toEqual([100, 200, 101, 102, 103, 104, 105]);
});
it("stops scheduling on cancellation and waits for resolved objects to be released", async () => {
const controller = new AbortController();
const resolutions = new Map<string, () => void>();
const releases = new Map<string, () => void>();
const fixture = siblingCaptureFixture(async (method, params) => {
if (method === "DOM.resolveNode")
await new Promise<void>((resolve) =>
resolutions.set(String(params.backendNodeId), resolve),
);
if (method === "Runtime.releaseObject")
await new Promise<void>((resolve) => releases.set(String(params.objectId), resolve));
});
let settled = false;
const capture = captureViewModel(fixture.cdp, 4, { signal: controller.signal });
const rejected = expect(capture).rejects.toMatchObject({ name: "AbortError" });
void capture.then(
() => {
settled = true;
},
() => {
settled = true;
},
);
await vi.waitFor(() => expect(resolutions.size).toBe(4));
controller.abort();
for (const resolve of resolutions.values()) resolve();
await vi.waitFor(() => expect(releases.size).toBe(4));
expect(settled).toBe(false);
expect(fixture.measured()).toEqual([100, 101, 102, 103]);
for (const release of releases.values()) release();
await rejected;
expect(fixture.active()).toBe(0);
expect(fixture.send.mock.calls.some(([, method]) => method === "Runtime.callFunctionOn")).toBe(
false,
);
});
});
@@ -0,0 +1,127 @@
import { describe, expect, it, vi } from "vitest";
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import { createCaptureCheckpoint } from "../capture-abort";
import { buildDocumentIndex, type DecodedNode } from "../facts";
import { decodeDocument, REQUESTED_STYLES } from "../snapshot";
function node(id: number, parent: number | null): DecodedNode {
return {
backendNodeId: id,
parentBackendNodeId: parent,
tag: "div",
attrs: {},
paintOrder: 0,
position: "static",
pointerEvents: "auto",
};
}
describe("document facts", () => {
it.each([
1_000, 10_000, 100_000,
])("indexes deep and wide inputs of %i without copying ancestors", async (count) => {
for (const wide of [false, true]) {
let reads = 0;
const input = Array.from({ length: count }, (_, i) => {
const item = node(i, i ? (wide ? 0 : i - 1) : null);
if (i === 0) item.attrs[OVERLAY_HOST_MARKER_ATTR] = "";
Object.defineProperty(item, "parentBackendNodeId", {
get: () => {
reads++;
return i ? (wide ? 0 : i - 1) : null;
},
});
return item;
}).reverse(); // parent need not precede child
const index = await buildDocumentIndex(input);
expect(index.nodes.size).toBe(count);
expect(index.excludedBackendNodeIds.size).toBe(count);
expect(reads).toBeLessThanOrEqual(count * 8);
expect(index.nodes.get(count - 1)).toBe(input[0]);
}
});
it("handles cycles and orphans and propagates overlay through shadow roots", async () => {
const host = { ...node(1, null), attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } };
const input = [
node(3, 2),
{ ...node(2, 1), tag: "#document-fragment" },
host,
node(4, 99),
node(5, 6),
node(6, 5),
node(7, 6),
];
const index = await buildDocumentIndex(input);
expect([...index.excludedBackendNodeIds].sort()).toEqual([1, 2, 3]);
expect(index.nodes.size).toBe(input.length);
});
it("does not propagate a descendant overlay backwards into a cycle", async () => {
const overlay = { ...node(3, 1), attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } };
const index = await buildDocumentIndex([overlay, node(1, 2), node(2, 1)]);
expect([...index.excludedBackendNodeIds]).toEqual([3]);
});
it("decodes current observation bounds, styles and attributes without projecting coordinates", async () => {
const strings = ["div", "aria-hidden", "true", "inert", "", "visible", "0", "static", "auto"];
const styles = REQUESTED_STYLES.map((style) =>
style === "visibility" ? 5 : style === "opacity" ? 6 : style === "position" ? 7 : 8,
);
const { nodes } = await decodeDocument(
{
nodes: { backendNodeId: [1], parentIndex: [-1], nodeName: [0], attributes: [[1, 2, 3, 4]] },
layout: { nodeIndex: [0], bounds: [[10, 20, 120, 40]], styles: [styles] },
},
strings,
);
expect(nodes[0].attrs).toEqual({ "aria-hidden": "true", inert: "" });
expect(nodes[0].layout).toEqual({
boundsSpace: "snapshot-document-css",
bounds: [10, 20, 120, 40],
styles: {
position: "static",
"pointer-events": "auto",
cursor: "auto",
visibility: "visible",
opacity: "0",
},
});
expect(nodes[0]).not.toHaveProperty("rect");
expect(nodes[0]).not.toHaveProperty("excluded");
});
it("keeps short blocks synchronous and checks cancellation after yielding", async () => {
const clock = vi.spyOn(performance, "now").mockReturnValue(0);
try {
const controller = new AbortController();
const checkpoint = createCaptureCheckpoint(controller.signal);
for (let i = 0; i < 20; i++) expect(checkpoint()).toBeUndefined();
clock.mockReturnValue(9);
const pending = checkpoint();
expect(pending).toBeInstanceOf(Promise);
controller.abort();
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
} finally {
clock.mockRestore();
}
});
it("checks cancellation at bounded indexing blocks", async () => {
const controller = new AbortController();
const input = Array.from({ length: 100_000 }, (_, i) => node(i, i ? i - 1 : null));
let reads = 0;
for (const item of input)
Object.defineProperty(item, "attrs", {
get: () => {
reads++;
if (reads === 100) controller.abort();
return {};
},
});
await expect(buildDocumentIndex(input, controller.signal)).rejects.toMatchObject({
name: "AbortError",
});
expect(reads).toBeLessThanOrEqual(356);
});
});
@@ -1,6 +1,7 @@
import { describe, expect, it, vi } from "vitest"; import { describe, expect, it, vi } from "vitest";
import { type CdpRunner, cdpRunnerForTarget } from "../../shared"; import { type CdpRunner, cdpRunnerForTarget } from "../../shared";
import { captureViewModel } from "../capture"; import { enrichFormControlStates } from "../form-capture";
import { decodeDocument } from "../snapshot";
interface Control { interface Control {
id: number; id: number;
@@ -77,21 +78,32 @@ function reply(entries: Array<[number, unknown]>) {
}; };
} }
function fakeCdp(groups: Control[][], evaluate: (params: Record<string, unknown>) => unknown) { async function fakeCdp(
groups: Control[][],
evaluate: (params: Record<string, unknown>) => unknown,
) {
const snap = snapshot(groups); const snap = snapshot(groups);
const send = vi.fn(async (_tabId: number, method: string, params?: object) => { const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
if (method === "DOMSnapshot.captureSnapshot") return snap;
if (method === "Runtime.evaluate") return evaluate(params as Record<string, unknown>); if (method === "Runtime.evaluate") return evaluate(params as Record<string, unknown>);
return {}; return {};
}); });
return { send, cdp: { send: send as CdpRunner["send"] } }; const nodes = await decodeControls(snap);
return { send, cdp: { send: send as CdpRunner["send"] }, nodes };
}
async function decodeControls(snap: ReturnType<typeof snapshot>) {
return Promise.all(
snap.documents.map(async (doc) =>
(await decodeDocument(doc, snap.strings)).nodes.map((node) => ({ ...node, rect: null })),
),
);
} }
describe("form state identity", () => { describe("form state identity", () => {
it("matches reordered controls by identity and preserves controls missing from the batch", async () => { it("matches reordered controls by identity and preserves controls missing from the batch", async () => {
// The snapshot includes a shadow input. The runtime query skips it, // The snapshot includes a shadow input. The runtime query skips it,
// encounters a new input, and sees the remaining inputs in a new order. // encounters a new input, and sees the remaining inputs in a new order.
const { cdp } = fakeCdp( const { cdp, nodes } = await fakeCdp(
[ [
[ [
{ id: 10, value: "" }, { id: 10, value: "" },
@@ -106,21 +118,21 @@ describe("form state identity", () => {
[11, state("alice@example.com")], [11, state("alice@example.com")],
]), ]),
); );
const captured = await captureViewModel(cdp, 7); await enrichFormControlStates(cdp, 7, nodes);
expect(captured.nodes.find((n) => n.backendNodeId === 10)).toMatchObject({ expect(nodes[0].find((n) => n.backendNodeId === 10)).toMatchObject({
formValue: "", formValue: "",
formState: "empty", formState: "empty",
formPlaceholder: "snapshot placeholder", formPlaceholder: "snapshot placeholder",
}); });
expect(captured.nodes.find((n) => n.backendNodeId === 11)).toMatchObject({ expect(nodes[0].find((n) => n.backendNodeId === 11)).toMatchObject({
formValue: "alice@example.com", formValue: "alice@example.com",
}); });
expect(captured.nodes.find((n) => n.backendNodeId === 12)).toMatchObject({ formValue: "上海" }); expect(nodes[0].find((n) => n.backendNodeId === 12)).toMatchObject({ formValue: "上海" });
expect(captured.nodes.some((n) => n.backendNodeId === 99)).toBe(false); expect(nodes[0].some((n) => n.backendNodeId === 99)).toBe(false);
}); });
it("does not shift state between frames when one frame is absent from the runtime batch", async () => { it("does not shift state between frames when one frame is absent from the runtime batch", async () => {
const { cdp } = fakeCdp( const { cdp, nodes } = await fakeCdp(
[ [
[], [],
[{ id: 11, value: "inaccessible frame" }], [{ id: 11, value: "inaccessible frame" }],
@@ -133,8 +145,8 @@ describe("form state identity", () => {
[21, state("second frame")], [21, state("second frame")],
]), ]),
); );
const captured = await captureViewModel(cdp, 7); await enrichFormControlStates(cdp, 7, nodes);
const controls = [...captured.iframeNodes.values()].flat(); const controls = nodes.slice(1).flat();
expect(controls.find((n) => n.backendNodeId === 11)?.formValue).toBe("inaccessible frame"); expect(controls.find((n) => n.backendNodeId === 11)?.formValue).toBe("inaccessible frame");
expect(controls.find((n) => n.backendNodeId === 21)?.formValue).toBe("second frame"); expect(controls.find((n) => n.backendNodeId === 21)?.formValue).toBe("second frame");
expect(controls.find((n) => n.backendNodeId === 31)?.formValue).toBe("third frame"); expect(controls.find((n) => n.backendNodeId === 31)?.formValue).toBe("third frame");
@@ -142,7 +154,6 @@ describe("form state identity", () => {
it("keeps identical backend ids in different OOPIF targets separate", async () => { it("keeps identical backend ids in different OOPIF targets separate", async () => {
const sendToTarget = vi.fn(async (target, method) => { const sendToTarget = vi.fn(async (target, method) => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot([[{ id: 11, value: "old" }]]);
if (method === "Runtime.evaluate") return reply([[11, state(target.sessionId)]]); if (method === "Runtime.evaluate") return reply([[11, state(target.sessionId)]]);
return {}; return {};
}); });
@@ -151,8 +162,9 @@ describe("form state identity", () => {
}); });
const cdp = { send, sendToTarget } as unknown as CdpRunner; const cdp = { send, sendToTarget } as unknown as CdpRunner;
for (const sessionId of ["left-frame", "right-frame"]) { for (const sessionId of ["left-frame", "right-frame"]) {
const captured = await captureViewModel(cdpRunnerForTarget(cdp, { tabId: 7, sessionId }), 7); const nodes = await decodeControls(snapshot([[{ id: 11, value: "old" }]]));
expect(captured.nodes.find((n) => n.backendNodeId === 11)?.formValue).toBe(sessionId); await enrichFormControlStates(cdpRunnerForTarget(cdp, { tabId: 7, sessionId }), 7, nodes);
expect(nodes[0].find((n) => n.backendNodeId === 11)?.formValue).toBe(sessionId);
expect(sendToTarget).toHaveBeenCalledWith( expect(sendToTarget).toHaveBeenCalledWith(
{ tabId: 7, sessionId }, { tabId: 7, sessionId },
"Runtime.releaseObjectGroup", "Runtime.releaseObjectGroup",
@@ -168,7 +180,7 @@ describe("form state identity", () => {
"malformed", "malformed",
"invalid-state", "invalid-state",
])("preserves snapshot data when enrichment is %s", async (mode) => { ])("preserves snapshot data when enrichment is %s", async (mode) => {
const { cdp, send } = fakeCdp([[{ id: 11, value: "snapshot value" }]], () => { const { cdp, send, nodes } = await fakeCdp([[{ id: 11, value: "snapshot value" }]], () => {
if (mode === "unsupported") throw new Error("serialization unsupported"); if (mode === "unsupported") throw new Error("serialization unsupported");
if (mode === "legacy") return { result: { value: { controls: [state("wrong value")] } } }; if (mode === "legacy") return { result: { value: { controls: [state("wrong value")] } } };
const result = reply([ const result = reply([
@@ -178,8 +190,8 @@ describe("form state identity", () => {
result.result.deepSerializedValue.value[0].value[1].value = "not json"; result.result.deepSerializedValue.value[0].value[1].value = "not json";
return result; return result;
}); });
const captured = await captureViewModel(cdp, 7); expect(await enrichFormControlStates(cdp, 7, nodes)).toBe(mode !== "unsupported");
expect(captured.nodes.find((n) => n.backendNodeId === 11)).toMatchObject({ expect(nodes[0].find((n) => n.backendNodeId === 11)).toMatchObject({
formValue: "snapshot value", formValue: "snapshot value",
formPlaceholder: "snapshot placeholder", formPlaceholder: "snapshot placeholder",
}); });
@@ -188,18 +200,34 @@ describe("form state identity", () => {
it("releases the batch's remote objects when capture is cancelled", async () => { it("releases the batch's remote objects when capture is cancelled", async () => {
const controller = new AbortController(); const controller = new AbortController();
const { cdp, send } = fakeCdp([[{ id: 11, value: "old" }]], () => { const { cdp, send, nodes } = await fakeCdp([[{ id: 11, value: "old" }]], () => {
controller.abort(); controller.abort();
return reply([[11, state("new")]]); return reply([[11, state("new")]]);
}); });
await expect(captureViewModel(cdp, 7, { signal: controller.signal })).rejects.toMatchObject({ await expect(enrichFormControlStates(cdp, 7, nodes, controller.signal)).rejects.toMatchObject({
name: "AbortError", name: "AbortError",
}); });
expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything()); expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
}); });
it("does not report success when cancelled during object cleanup", async () => {
const controller = new AbortController();
const nodes = await decodeControls(snapshot([[{ id: 11, value: "old" }]]));
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method) => {
if (method === "Runtime.evaluate") return reply([[11, state("new")]]);
if (method === "Runtime.releaseObjectGroup") controller.abort();
return {};
}) as CdpRunner["send"],
};
await expect(enrichFormControlStates(cdp, 7, nodes, controller.signal)).rejects.toMatchObject({
name: "AbortError",
});
expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
});
it("skips entries without node identity while still applying valid entries", async () => { it("skips entries without node identity while still applying valid entries", async () => {
const { cdp } = fakeCdp( const { cdp, nodes } = await fakeCdp(
[ [
[ [
{ id: 11, value: "snapshot value" }, { id: 11, value: "snapshot value" },
@@ -216,20 +244,20 @@ describe("form state identity", () => {
return result; return result;
}, },
); );
const captured = await captureViewModel(cdp, 7); await enrichFormControlStates(cdp, 7, nodes);
expect(captured.nodes.find((n) => n.backendNodeId === 11)?.formValue).toBe("snapshot value"); expect(nodes[0].find((n) => n.backendNodeId === 11)?.formValue).toBe("snapshot value");
expect(captured.nodes.find((n) => n.backendNodeId === 12)?.formValue).toBe("live value"); expect(nodes[0].find((n) => n.backendNodeId === 12)?.formValue).toBe("live value");
}); });
it.each([ it.each([
"password", "password",
"text", "text",
])("redacts a sensitive %s control after identity matching", async (type) => { ])("redacts a sensitive %s control after identity matching", async (type) => {
const { cdp } = fakeCdp([[{ id: 11, value: "secret", type }]], () => const { cdp, nodes } = await fakeCdp([[{ id: 11, value: "secret", type }]], () =>
reply([[11, { ...state("secret"), sensitive: true }]]), reply([[11, { ...state("secret"), sensitive: true }]]),
); );
const captured = await captureViewModel(cdp, 7); await enrichFormControlStates(cdp, 7, nodes);
const control = captured.nodes.find((n) => n.backendNodeId === 11); const control = nodes[0].find((n) => n.backendNodeId === 11);
expect(control?.formState).toBe("filled"); expect(control?.formState).toBe("filled");
expect(control?.formValue).toBeUndefined(); expect(control?.formValue).toBeUndefined();
expect(control?.formDefaultValue).toBeUndefined(); expect(control?.formDefaultValue).toBeUndefined();
@@ -246,7 +274,7 @@ describe("form state identity", () => {
defaultValue: "", defaultValue: "",
placeholder: "", placeholder: "",
})); }));
const { cdp, send } = fakeCdp([controls], (params) => { const { cdp, send, nodes } = await fakeCdp([controls], (params) => {
const doc = { const doc = {
querySelectorAll: (selector: string) => querySelectorAll: (selector: string) =>
selector === "input,textarea,select" ? elements : [], selector === "input,textarea,select" ? elements : [],
@@ -257,9 +285,9 @@ describe("form state identity", () => {
expect(entries).toHaveLength(250); expect(entries).toHaveLength(250);
return reply(entries.map(([element, json]) => [element.backendNodeId, JSON.parse(json)])); return reply(entries.map(([element, json]) => [element.backendNodeId, JSON.parse(json)]));
}); });
const captured = await captureViewModel(cdp, 7); await enrichFormControlStates(cdp, 7, nodes);
expect(captured.nodes.find((n) => n.backendNodeId === 250)?.formValue).toBe("live 250"); expect(nodes[0].find((n) => n.backendNodeId === 250)?.formValue).toBe("live 250");
expect(captured.nodes.find((n) => n.backendNodeId === 251)?.formValue).toBe("old"); expect(nodes[0].find((n) => n.backendNodeId === 251)?.formValue).toBe("old");
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1); expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1);
expect( expect(
send.mock.calls.filter(([, method]) => method === "Runtime.releaseObjectGroup"), send.mock.calls.filter(([, method]) => method === "Runtime.releaseObjectGroup"),
@@ -1,4 +1,5 @@
import { describe, expect, it, vi } from "vitest"; import { describe, expect, it, vi } from "vitest";
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import type { CdpRunner } from "../../shared"; import type { CdpRunner } from "../../shared";
import type { CapturedNode, CapturedViewModel } from "../capture"; import type { CapturedNode, CapturedViewModel } from "../capture";
import { captureFrameData } from "../frame-capture"; import { captureFrameData } from "../frame-capture";
@@ -30,7 +31,7 @@ function childSnapshot(frameId: string, backendNodeId: number) {
parentIndex: [-1, 0], parentIndex: [-1, 0],
nodeName: [1, 2], nodeName: [1, 2],
backendNodeId: [backendNodeId - 1, backendNodeId], backendNodeId: [backendNodeId - 1, backendNodeId],
attributes: [[], []], attributes: [[], []] as number[][],
}, },
layout: { layout: {
nodeIndex: [0, 1], nodeIndex: [0, 1],
@@ -50,6 +51,68 @@ function childSnapshot(frameId: string, backendNodeId: number) {
} }
describe("captureFrameData", () => { describe("captureFrameData", () => {
it("retains nested owner failures when merging a captured OOPIF", async () => {
const owner = ownerNode(10, 50);
const captured: CapturedViewModel = {
nodes: [owner],
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
frameNodes: new Map([["main", [owner]]]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const child = childSnapshot("child", 101);
const nested = childSnapshot("nested", 201);
const document = child.documents[0];
const snapshot = {
strings: child.strings,
documents: [
{
...document,
nodes: { ...document.nodes, contentDocumentIndex: { index: [1], value: [1] } },
},
nested.documents[0],
],
};
const reply = async (_target: unknown, method: string) => {
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200 } };
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "DOM.getBoxModel")
return { model: { content: [50, 100, 350, 100, 350, 300, 50, 300] } };
if (method === "DOM.resolveNode") throw new Error("nested owner replaced");
return {};
};
const cdp: CdpRunner = {
send: vi.fn(reply) as CdpRunner["send"],
sendToTarget: vi.fn(reply) as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
})),
};
await captureFrameData(cdp, 4, captured);
expect(captured.frameGeometryIssues).toEqual([
{
status: "unavailable",
source: { target: { tabId: 4, sessionId: "child-session" }, frameId: "nested" },
ownerBackendNodeId: 101,
},
]);
expect(captured.frameNodes?.get("nested")?.[1]).toMatchObject({
rect: null,
localRect: { x: 10, y: 20, w: 100, h: 40 },
});
});
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => { it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
const leftOwner = ownerNode(10, 50); const leftOwner = ownerNode(10, 50);
const rightOwner = ownerNode(20, 500); const rightOwner = ownerNode(20, 500);
@@ -68,9 +131,15 @@ describe("captureFrameData", () => {
} }
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {}; if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") { if (method === "DOMSnapshot.captureSnapshot") {
return target.sessionId === "left-session" if (target.sessionId !== "left-session") return childSnapshot("right", 201);
? childSnapshot("left", 101) const snapshot = childSnapshot("left", 101);
: childSnapshot("right", 201); const marker = snapshot.strings.push(OVERLAY_HOST_MARKER_ATTR) - 1;
const nodes = snapshot.documents[0].nodes;
nodes.backendNodeId.push(201); // Same id as a real button in the other target.
nodes.parentIndex.push(0);
nodes.nodeName.push(2);
nodes.attributes.push([marker, -1]);
return snapshot;
} }
if (method === "Accessibility.getFullAXTree") return { nodes: [] }; if (method === "Accessibility.getFullAXTree") return { nodes: [] };
throw new Error(`unexpected ${method}`); throw new Error(`unexpected ${method}`);
@@ -112,6 +181,10 @@ describe("captureFrameData", () => {
const trees = await captureFrameData(cdp, 4, captured); const trees = await captureFrameData(cdp, 4, captured);
expect(trees.find((tree) => tree.frameId === "left")?.excludedBackendNodeIds).toEqual(
new Set([201]),
);
expect(trees.find((tree) => tree.frameId === "right")?.excludedBackendNodeIds?.size).toBe(0);
expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]); expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]);
expect( expect(
captured.frameNodes?.get("left")?.find((node) => node.backendNodeId === 101)?.rect, captured.frameNodes?.get("left")?.find((node) => node.backendNodeId === 101)?.rect,
@@ -29,7 +29,7 @@ function captured(frameNodes: Map<string, CapturedNode[]>): CapturedViewModel {
} }
describe("buildFrameDocuments", () => { describe("buildFrameDocuments", () => {
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", () => { it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", async () => {
const graph: CdpFrameGraph = { const graph: CdpFrameGraph = {
rootFrameId: "main", rootFrameId: "main",
frames: [ frames: [
@@ -54,7 +54,7 @@ describe("buildFrameDocuments", () => {
}, },
]; ];
const documents = buildFrameDocuments( const documents = await buildFrameDocuments(
graph, graph,
batches, batches,
captured( captured(
@@ -72,7 +72,7 @@ describe("buildFrameDocuments", () => {
expect(childNodes.find((node) => node.nodeId === "virtual")?.frameId).toBe("child"); expect(childNodes.find((node) => node.nodeId === "virtual")?.frameId).toBe("child");
}); });
it("deduplicates overlapping same-target AX results using the strongest ownership", () => { it("deduplicates overlapping same-target AX results using the strongest ownership", async () => {
const graph: CdpFrameGraph = { const graph: CdpFrameGraph = {
rootFrameId: "main", rootFrameId: "main",
frames: [ frames: [
@@ -92,7 +92,7 @@ describe("buildFrameDocuments", () => {
}, },
]; ];
const documents = buildFrameDocuments( const documents = await buildFrameDocuments(
graph, graph,
batches, batches,
captured( captured(
@@ -107,7 +107,7 @@ describe("buildFrameDocuments", () => {
expect(documents.find((document) => document.frameId === "child")?.axNodes).toHaveLength(1); expect(documents.find((document) => document.frameId === "child")?.axNodes).toHaveLength(1);
}); });
it("uses target-scoped backend ownership when OOPIF backend ids collide", () => { it("uses target-scoped backend ownership when OOPIF backend ids collide", async () => {
const graph: CdpFrameGraph = { const graph: CdpFrameGraph = {
rootFrameId: "main", rootFrameId: "main",
frames: [ frames: [
@@ -129,7 +129,7 @@ describe("buildFrameDocuments", () => {
nodes: [{ nodeId: `${frame.frameId}-node`, backendDOMNodeId: 7 }], nodes: [{ nodeId: `${frame.frameId}-node`, backendDOMNodeId: 7 }],
})); }));
const documents = buildFrameDocuments( const documents = await buildFrameDocuments(
graph, graph,
batches, batches,
captured( captured(
@@ -149,7 +149,7 @@ describe("buildFrameDocuments", () => {
); );
}); });
it("uses captured frame parent identity instead of guessing from backend ids", () => { it("uses captured frame parent identity instead of guessing from backend ids", async () => {
const frameNodes = new Map([ const frameNodes = new Map([
["main", [domNode("main", 1)]], ["main", [domNode("main", 1)]],
["left", [domNode("left", 7)]], ["left", [domNode("left", 7)]],
@@ -164,11 +164,25 @@ describe("buildFrameDocuments", () => {
nodes: [], nodes: [],
})); }));
const documents = buildFrameDocuments(null, frames, capture); const documents = await buildFrameDocuments(null, frames, capture);
expect(documents.find((document) => document.frameId === "child")).toMatchObject({ expect(documents.find((document) => document.frameId === "child")).toMatchObject({
parentFrameId: "left", parentFrameId: "left",
ownerBackendNodeId: 7, ownerBackendNodeId: 7,
}); });
}); });
it("resolves a reverse-ordered deep AX parent chain without recursion", async () => {
const frame = { frameId: "main", target: { tabId: 4 } };
const nodes: FrameOwnedAxNode[] = Array.from({ length: 10000 }, (_, i) => ({
nodeId: String(i),
...(i ? { parentId: String(i - 1) } : { frameId: "main" }),
})).reverse();
const documents = await buildFrameDocuments(
{ rootFrameId: "main", frames: [frame] },
[{ frame, nodes }],
captured(new Map()),
);
expect(documents[0].axNodes).toHaveLength(10000);
expect(documents[0].axNodes[0].frameId).toBe("main");
});
}); });
@@ -0,0 +1,40 @@
function captureAbortError(): Error {
const error = new Error("observation aborted");
error.name = "AbortError";
return error;
}
export function isAbortError(error: unknown): boolean {
return (
typeof error === "object" &&
error !== null &&
(error as { name?: string }).name === "AbortError"
);
}
export function throwIfAborted(signal: AbortSignal | undefined): void {
if (signal?.aborted) throw captureAbortError();
}
/** Check at bounded work blocks; yield only after a processing slice is spent.
* Each caller owns its deadline, so unrelated captures do not share state. */
export function createCaptureCheckpoint(signal?: AbortSignal): () => Promise<void> | undefined {
throwIfAborted(signal);
let deadline = performance.now() + 8;
return () => {
throwIfAborted(signal);
if (performance.now() < deadline) return;
return new Promise<void>((resolve) => {
const channel = new MessageChannel();
channel.port1.onmessage = () => {
channel.port1.close();
channel.port2.close();
resolve();
};
channel.port2.postMessage(null);
}).then(() => {
throwIfAborted(signal);
deadline = performance.now() + 8;
});
};
}
@@ -0,0 +1,37 @@
import type { Rect } from "@browser-skill/vom";
export interface CapturedNode {
backendNodeId: number;
parentBackendNodeId: number | null;
frameId?: string;
/** Owning iframe backend node id; `null` for the top-level document. */
ownerFrameBackendNodeId?: number | null;
tag: string;
attrs: Record<string, string>;
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
rect: Rect | null;
/** Frame-local viewport-relative CSS px before top-level projection. */
localRect?: Rect | null;
paintOrder: number;
position: string;
pointerEvents: string;
/**
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
* is actually an interactive control — used by the adapter to surface
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
* `textContent`: the live parser always sets it, hand-built fixtures may not.
*/
cursor?: string;
/**
* Whether the live DOM snapshot provides a painted, non-hidden box for this
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
* nodes remain authoritative even when they are outside the viewport.
*/
rendered?: boolean;
textContent?: string;
formState?: "empty" | "filled" | "default";
formValue?: string;
formDefaultValue?: string;
formPlaceholder?: string;
}
+155 -550
View File
@@ -1,56 +1,27 @@
// CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics // Coordinate DOMSnapshot acquisition, frame geometry and hover probes.
// → CapturedNode[] + Viewport. This is the ONLY VOM module that touches // Snapshot decoding and form enrichment live in their own modules.
// raw CDP. The captureSnapshot reply is columnar (parallel arrays + a
// shared string table); requested computed styles are decoded through
// STYLE_COL so capture and visibility policy share one explicit contract.
import type { Rect, Viewport } from "@browser-skill/vom"; import type { Viewport } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy"; import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge"; import { OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry"; import { type FrameProjectionIssue, type FrameProjectionState } from "../geometry/coordinate-types";
import { cssViewport, GeometryContext, type LayoutMetrics } from "../geometry/frame-context";
import type { CdpRunner } from "../shared"; import type { CdpRunner } from "../shared";
import { isAbortError, throwIfAborted } from "./capture-abort";
import type { CapturedNode } from "./capture-types";
import { enrichFormControlStates } from "./form-capture";
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception"; import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
import { type FrameContext, normalizeDocument } from "./normalize";
import {
REQUESTED_STYLES,
type SnapshotDocument,
type SnapshotReply,
snapshotFrameId,
sparseIndexMap,
} from "./snapshot";
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const; export type { CapturedNode } from "./capture-types";
const STYLE_COL = Object.fromEntries(
REQUESTED_STYLES.map((name, index) => [name, index]),
) as Record<(typeof REQUESTED_STYLES)[number], number>;
export interface CapturedNode {
backendNodeId: number;
parentBackendNodeId: number | null;
frameId?: string;
/** Owning iframe backend node id; `null` for the top-level document. */
ownerFrameBackendNodeId?: number | null;
tag: string;
attrs: Record<string, string>;
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
rect: Rect | null;
/** Frame-local viewport-relative CSS px before top-level projection. */
localRect?: Rect | null;
paintOrder: number;
position: string;
pointerEvents: string;
/**
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
* is actually an interactive control — used by the adapter to surface
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
* `textContent`: the live parser always sets it, hand-built fixtures may not.
*/
cursor?: string;
/**
* Whether the live DOM snapshot provides a painted, non-hidden box for this
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
* nodes remain authoritative even when they are outside the viewport.
*/
rendered?: boolean;
textContent?: string;
formState?: "empty" | "filled" | "default";
formValue?: string;
formDefaultValue?: string;
formPlaceholder?: string;
}
export type CapturedIframeNodes = Map<number, CapturedNode[]>; export type CapturedIframeNodes = Map<number, CapturedNode[]>;
@@ -67,266 +38,26 @@ export interface CapturedViewModel {
viewport: Viewport; viewport: Viewport;
iframeNodes: CapturedIframeNodes; iframeNodes: CapturedIframeNodes;
frameNodes?: Map<string, CapturedNode[]>; frameNodes?: Map<string, CapturedNode[]>;
frameExcludedBackendNodeIds?: Map<string, ReadonlySet<number>>;
frameOwnerBackendNodeIds?: Map<string, number>; frameOwnerBackendNodeIds?: Map<string, number>;
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */ /** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
frameParentIds?: Map<string, string>; frameParentIds?: Map<string, string>;
rootFrameId?: string; rootFrameId?: string;
/** Failed boundaries and descendants blocked by them; clipping is not a failure. */
frameGeometryIssues?: FrameProjectionIssue[];
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */ /** Backend node ids belonging to the agent overlay host + its shadow subtree. */
excludedBackendNodeIds: Set<number>; excludedBackendNodeIds: Set<number>;
} }
export interface CaptureViewModelOptions { export interface CaptureViewModelOptions {
signal?: AbortSignal; signal?: AbortSignal;
} geometry?: GeometryContext;
target?: CdpTarget;
function captureAbortError(): Error {
const error = new Error("observation aborted");
error.name = "AbortError";
return error;
}
function isAbortError(error: unknown): boolean {
return (
typeof error === "object" &&
error !== null &&
(error as { name?: string }).name === "AbortError"
);
}
function throwIfAborted(signal: AbortSignal | undefined): void {
if (signal?.aborted) throw captureAbortError();
}
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
interface SparseArray {
index: number[];
value: number[];
}
interface RareBooleanData {
index: number[];
}
interface SnapshotDocument {
frameId?: string | number;
scrollOffsetX?: number;
scrollOffsetY?: number;
nodes?: {
parentIndex?: number[];
nodeName?: number[];
backendNodeId?: number[];
attributes?: number[][];
/**
* Per-node text value (index into strings), set for `#text` / CDATA nodes.
* Element nodes carry -1. Same length as `backendNodeId`.
*/
nodeValue?: number[];
/** Maps node array index → index into `documents[]` for frame content. */
contentDocumentIndex?: SparseArray;
inputValue?: SparseArray;
textValue?: SparseArray;
inputChecked?: RareBooleanData;
optionSelected?: RareBooleanData;
};
layout?: {
nodeIndex?: number[];
styles?: number[][];
bounds?: number[][];
paintOrders?: number[];
};
}
function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
if (typeof document.frameId === "string") return document.frameId || undefined;
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
return undefined;
}
interface SnapshotReply {
strings?: string[];
documents?: SnapshotDocument[];
}
interface LayoutMetricsReply {
cssLayoutViewport?: {
clientWidth?: number;
clientHeight?: number;
pageX?: number;
pageY?: number;
};
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
}
function isFormControlTag(tag: string): boolean {
return tag === "input" || tag === "textarea" || tag === "select";
}
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
}
function snapshotFormState(
value: string | undefined,
defaultValue: string,
hasDefaultValue: boolean,
sensitive: boolean,
): CapturedNode["formState"] {
if (sensitive) {
if (value === undefined && !hasDefaultValue) return undefined;
return (value ?? defaultValue) === "" ? "empty" : "filled";
}
if (value === undefined) return undefined;
if (value === "") return "empty";
return value === defaultValue ? "default" : "filled";
}
const MAX_FORM_ENRICH_CONTROLS = 250;
interface CapturedFormState {
value?: string;
defaultValue?: string;
placeholder?: string;
state?: "empty" | "filled" | "default";
sensitive?: boolean;
}
interface DeepSerializedValue {
type: string;
value?: unknown;
}
function formStateBatchExpression(maxControls: number): string {
return `(() => {
const maxControls = ${JSON.stringify(maxControls)};
let remaining = maxControls;
const controlSelector = "input,textarea,select";
const controlState = (el) => {
const tag = el.tagName.toLowerCase();
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
const sensitive = type === "password" || type === "credit-card";
const rawValue = typeof el.value === "string" ? el.value : "";
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
return {
state,
sensitive,
placeholder,
...(sensitive ? {} : { value: rawValue, defaultValue }),
};
};
const controls = [];
const collect = (doc) => {
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
if (remaining <= 0) break;
// Deep serialization supplies the node's backend id. Keep the
// state as JSON so decoding needs no general-purpose V8 deserializer.
controls.push([el, JSON.stringify(controlState(el))]);
remaining -= 1;
}
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
if (remaining <= 0) break;
let childDoc = null;
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
if (childDoc) collect(childDoc);
}
};
collect(document);
return controls;
})()`;
}
function formStatesByBackendId(result: RuntimeEvaluateReply): Map<number, CapturedFormState> {
const states = new Map<number, CapturedFormState>();
const serialized = result.result?.deepSerializedValue;
if (serialized?.type !== "array" || !Array.isArray(serialized.value)) return states;
for (const entry of serialized.value as DeepSerializedValue[]) {
if (entry?.type !== "array" || !Array.isArray(entry.value)) continue;
const [element, json] = entry.value as DeepSerializedValue[];
if (element?.type !== "node" || json?.type !== "string" || typeof json.value !== "string") {
continue;
}
const backendNodeId = (element.value as { backendNodeId?: number } | undefined)?.backendNodeId;
if (typeof backendNodeId !== "number" || !Number.isSafeInteger(backendNodeId)) continue;
try {
const state = JSON.parse(json.value) as CapturedFormState | null;
if (
!state ||
!["empty", "filled", "default"].includes(state.state ?? "") ||
typeof state.sensitive !== "boolean" ||
typeof state.placeholder !== "string" ||
(!state.sensitive &&
(typeof state.value !== "string" || typeof state.defaultValue !== "string"))
) {
continue;
}
states.set(backendNodeId, state);
} catch {
// A malformed entry must not overwrite the snapshot's own state.
}
}
return states;
}
function applyFormStates(nodes: CapturedNode[], states: Map<number, CapturedFormState>): void {
for (const node of nodes) {
if (!isFormControlTag(node.tag)) continue;
const state = states.get(node.backendNodeId);
if (!state) continue;
node.formState = state.state;
node.formPlaceholder = state.placeholder ?? "";
if (state.sensitive || isSensitiveFormControl(node)) {
delete node.formValue;
delete node.formDefaultValue;
delete node.attrs.value;
} else {
node.formDefaultValue = state.defaultValue ?? "";
if (state.value !== undefined) node.formValue = state.value;
}
}
}
async function enrichFormControlStates(
cdp: CdpRunner,
tabId: number,
frameNodeGroups: CapturedNode[][],
signal?: AbortSignal,
): Promise<void> {
const hasControls = frameNodeGroups.some((nodes) =>
nodes.some((node) => isFormControlTag(node.tag)),
);
if (!hasControls) return;
const objectGroup = `bsk-vom-forms-${crypto.randomUUID()}`;
try {
throwIfAborted(signal);
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
objectGroup,
serializationOptions: {
serialization: "deep",
maxDepth: 3,
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
},
});
throwIfAborted(signal);
// Backend ids are scoped to this capture's CDP target. OOPIF targets
// use their own captureViewModel call and never share this lookup.
const states = formStatesByBackendId(result);
for (const nodes of frameNodeGroups) {
applyFormStates(nodes, states);
}
} catch (error) {
if (isAbortError(error)) throw error;
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
} finally {
await cdp.send(tabId, "Runtime.releaseObjectGroup", { objectGroup }).catch(() => undefined);
}
} }
interface RuntimeEvaluateReply { interface RuntimeEvaluateReply {
result?: { result?: {
value?: unknown; value?: unknown;
deepSerializedValue?: DeepSerializedValue;
}; };
} }
@@ -345,44 +76,6 @@ interface CdpDomNode {
shadowRoots?: CdpDomNode[]; shadowRoots?: CdpDomNode[];
} }
interface ParseDocumentResult {
nodes: CapturedNode[];
excludedBackendNodeIds: Set<number>;
}
interface FrameContext {
frameId?: string;
ownerFrameBackendNodeId: number | null;
projection: GeometryProjection | null;
scrollX: number;
scrollY: number;
}
function str(strings: string[], idx: number | undefined): string {
if (idx === undefined || idx < 0) return "";
return strings[idx] ?? "";
}
function devicePixelRatio(metrics: LayoutMetricsReply): number {
const layoutW = metrics.layoutViewport?.clientWidth ?? 0;
const cssW = metrics.cssLayoutViewport?.clientWidth ?? 0;
if (!layoutW || !cssW) return 1;
const dpr = layoutW / cssW;
if (!Number.isFinite(dpr) || dpr <= 0) return 1;
return dpr >= 1 ? dpr : 1;
}
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
const out = new Map<number, number>();
if (!sparse?.index || !sparse.value) return out;
for (let i = 0; i < sparse.index.length; i++) {
const nodeIndex = sparse.index[i];
const docIndex = sparse.value[i];
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
}
return out;
}
function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<number>): void { function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<number>): void {
if (!node) return; if (!node) return;
if (typeof node.backendNodeId === "number") { if (typeof node.backendNodeId === "number") {
@@ -792,230 +485,108 @@ export async function collectOverlayExcludedBackendIds(
return excluded; return excluded;
} }
/**
* Parse one DOMSnapshot document into CapturedNode[].
*
* @param doc - the raw DOMSnapshot document object
* @param strings - the shared string table for the whole snapshot
* @param dpr - device-pixel-ratio from Page.getLayoutMetrics
* @param context - frame coordinate context; output rects are top viewport-relative
*/
function parseDocumentNodes(
doc: SnapshotDocument,
strings: string[],
dpr: number,
context: FrameContext,
): ParseDocumentResult {
const dn = doc.nodes;
const dl = doc.layout;
if (!dn?.backendNodeId) {
return { nodes: [], excludedBackendNodeIds: new Set() };
}
const count = dn.backendNodeId.length;
const layoutByNode = new Map<number, number>();
(dl?.nodeIndex ?? []).forEach((nodeIdx, layoutIdx) => layoutByNode.set(nodeIdx, layoutIdx));
// The agent's own overlay (WXT shadow host, marked with OVERLAY_HOST_MARKER_ATTR)
// is injected into the page and its open shadow root
// is inlined by DOMSnapshot as descendants of the host. Its fixed
// full-viewport click-blocker would otherwise dominate occlusion detection
// and hide the real page, so mark the host + its whole subtree excluded.
// parentIndex always references an earlier node, so one forward pass suffices.
const excluded = new Array<boolean>(count).fill(false);
const excludedBackendNodeIds = new Set<number>();
for (let n = 0; n < count; n++) {
const parentIdx = dn.parentIndex?.[n] ?? -1;
const inherited = parentIdx >= 0 && excluded[parentIdx];
let isOverlayHost = inherited;
if (!isOverlayHost) {
const pairs = dn.attributes?.[n] ?? [];
const attrNames: string[] = [];
for (let a = 0; a + 1 < pairs.length; a += 2) {
attrNames.push(str(strings, pairs[a]));
}
isOverlayHost = isOverlayHostNode(str(strings, dn.nodeName?.[n]), attrNames);
}
if (!isOverlayHost) continue;
excluded[n] = true;
const backendNodeId = dn.backendNodeId[n];
if (backendNodeId !== undefined) {
excludedBackendNodeIds.add(backendNodeId);
}
}
const posCol = STYLE_COL.position;
const peCol = STYLE_COL["pointer-events"];
const cursorCol = STYLE_COL.cursor;
const visibilityCol = STYLE_COL.visibility;
const opacityCol = STYLE_COL.opacity;
const inputValues = sparseIndexMap(dn.inputValue);
const textValues = sparseIndexMap(dn.textValue);
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
// Collect visible text from #text child nodes so element CapturedNodes
// carry a textContent value usable as a button/link label fallback.
// nodeValue is a parallel array: string index for text nodes, -1 otherwise.
const nodeTextContent = new Map<number, string>();
if (dn.nodeValue) {
for (let n = 0; n < count; n++) {
const nvIdx = dn.nodeValue[n] ?? -1;
if (nvIdx < 0) continue;
const text = str(strings, nvIdx).trim();
if (!text) continue;
const parentIdx = dn.parentIndex?.[n] ?? -1;
if (parentIdx >= 0) {
const existing = nodeTextContent.get(parentIdx);
nodeTextContent.set(parentIdx, existing ? `${existing} ${text}` : text);
}
}
}
// Build CapturedNodes.
const nodes: CapturedNode[] = [];
for (let n = 0; n < count; n++) {
if (excluded[n]) continue;
const backendNodeId = dn.backendNodeId[n];
const parentIdx = dn.parentIndex?.[n] ?? -1;
const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null;
const tag = str(strings, dn.nodeName?.[n]).toLowerCase();
const attrs: Record<string, string> = {};
const pairs = dn.attributes?.[n] ?? [];
for (let a = 0; a + 1 < pairs.length; a += 2) {
attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]);
}
let rect: Rect | null = null;
let localRect: Rect | null = null;
let paintOrder = 0;
let position = "static";
let pointerEvents = "auto";
let cursor = "auto";
let rendered = false;
const li = layoutByNode.get(n);
if (li !== undefined) {
const b = dl?.bounds?.[li];
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
localRect = {
x: b[0] / dpr - context.scrollX,
y: b[1] / dpr - context.scrollY,
w: b[2] / dpr,
h: b[3] / dpr,
};
const bounds = context.projection
? projectRectToViewport(localRect, context.projection)
: null;
rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
}
paintOrder = dl?.paintOrders?.[li] ?? 0;
const styleRow = dl?.styles?.[li] ?? [];
position = str(strings, styleRow[posCol]) || "static";
pointerEvents = str(strings, styleRow[peCol]) || "auto";
cursor = str(strings, styleRow[cursorCol]) || "auto";
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
const opacity = str(strings, styleRow[opacityCol]) || "1";
rendered =
localRect !== null &&
visibility !== "hidden" &&
visibility !== "collapse" &&
(Number.parseFloat(opacity) || 0) > 0;
}
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
// geometry and are never queried by callers.
if (tag.startsWith("#")) continue;
const textContent = nodeTextContent.get(n);
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
const rawFormValue =
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
const sensitive = isSensitiveFormControl({ tag, attrs });
const formDefaultValue = attrs.value ?? "";
const formValue = sensitive ? undefined : rawFormValue;
const formState = snapshotFormState(
rawFormValue,
formDefaultValue,
Object.prototype.hasOwnProperty.call(attrs, "value"),
sensitive,
);
if (sensitive) delete attrs.value;
nodes.push({
backendNodeId,
parentBackendNodeId,
...(context.frameId ? { frameId: context.frameId } : {}),
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
tag,
attrs,
rect,
localRect,
paintOrder,
position,
pointerEvents,
cursor,
rendered,
textContent,
...(formValue !== undefined ? { formValue } : {}),
...(tag === "input" || tag === "textarea"
? {
formPlaceholder: attrs.placeholder ?? "",
...(!sensitive ? { formDefaultValue } : {}),
...(formState ? { formState } : {}),
}
: {}),
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
});
}
return { nodes, excludedBackendNodeIds };
}
interface ParseFrameDocumentsResult { interface ParseFrameDocumentsResult {
frameGeometryIssues: FrameProjectionIssue[];
iframeNodes: CapturedIframeNodes; iframeNodes: CapturedIframeNodes;
frameOwnerBackendNodeIds: Map<string, number>; frameOwnerBackendNodeIds: Map<string, number>;
frameParentIds: Map<string, string>; frameParentIds: Map<string, string>;
frameExcludedBackendNodeIds: Map<string, ReadonlySet<number>>;
excludedBackendNodeIds: Set<number>; excludedBackendNodeIds: Set<number>;
} }
function parseChildFrameDocuments( async function parseChildFrameDocuments(
documents: SnapshotDocument[], documents: SnapshotDocument[],
strings: string[], strings: string[],
dpr: number,
parentDocIndex: number, parentDocIndex: number,
parentNodes: CapturedNode[],
parentContext: FrameContext, parentContext: FrameContext,
geometry: GeometryContext,
signal?: AbortSignal,
visited = new Set<number>(), visited = new Set<number>(),
): ParseFrameDocumentsResult { ): Promise<ParseFrameDocumentsResult> {
const frameGeometryIssues: FrameProjectionIssue[] = [];
const iframeNodes: CapturedIframeNodes = new Map(); const iframeNodes: CapturedIframeNodes = new Map();
const frameOwnerBackendNodeIds = new Map<string, number>(); const frameOwnerBackendNodeIds = new Map<string, number>();
const frameParentIds = new Map<string, string>(); const frameParentIds = new Map<string, string>();
const frameExcludedBackendNodeIds = new Map<string, ReadonlySet<number>>();
const excludedBackendNodeIds = new Set<number>(); const excludedBackendNodeIds = new Set<number>();
const parentDoc = documents[parentDocIndex]; const parentDoc = documents[parentDocIndex];
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex); const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
if (cdi.size === 0) { if (cdi.size === 0) {
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds }; return {
iframeNodes,
frameOwnerBackendNodeIds,
frameParentIds,
frameExcludedBackendNodeIds,
excludedBackendNodeIds,
frameGeometryIssues,
};
} }
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? []; const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
for (const [nodeArrayIdx, childDocIndex] of cdi) { const children = [...cdi].flatMap(([nodeArrayIdx, childDocIndex]) => {
if (visited.has(childDocIndex)) continue;
const childDoc = documents[childDocIndex]; const childDoc = documents[childDocIndex];
if (!childDoc) continue;
const iframeBackendId = parentBackendIds[nodeArrayIdx]; const iframeBackendId = parentBackendIds[nodeArrayIdx];
if (iframeBackendId === undefined) continue; if (visited.has(childDocIndex) || !childDoc || iframeBackendId === undefined) return [];
const iframeNode = parentNodeByBackendId.get(iframeBackendId); const source = { target: parentContext.target, frameId: snapshotFrameId(childDoc, strings) };
const projection = const parent = parentContext.projection;
parentContext.projection && iframeNode?.localRect const projection: FrameProjectionState =
? childFrameProjection(parentContext.projection, iframeNode.localRect) parent.status === "available"
: null; ? { status: "unavailable", source, ownerBackendNodeId: iframeBackendId }
: { status: "blocked", source, cause: parent.status === "blocked" ? parent.cause : parent };
return [
{
childDocIndex,
childDoc,
iframeBackendId,
source,
projection: projection as FrameProjectionState,
},
];
});
const parentProjection =
parentContext.projection.status === "available"
? parentContext.projection.projection.geometry
: null;
if (parentProjection) {
// Measure only direct siblings. Recursion starts after these workers finish,
// so descendants never hold or recursively acquire a measurement slot.
const edge = parentProjection.edges[0];
const clips = edge
? [edge.destinationQuad, ...(edge.destinationClips ?? [])]
: parentProjection.sourceClips;
let cursor = 0;
let failure: unknown;
await Promise.all(
Array.from({ length: Math.min(4, children.length) }, async () => {
while (cursor < children.length && !signal?.aborted && !failure) {
const child = children[cursor++];
try {
// Owner quads are target-relative; do not apply the parent transform twice.
child.projection = await geometry.snapshotProjection(
child.source,
child.iframeBackendId,
clips,
parentProjection.topViewport,
);
} catch (error) {
failure ??= error;
}
}
}),
);
// Join all active reads, including object cleanup, before propagating abort.
if (failure) throw failure;
}
throwIfAborted(signal);
// Completion order must not change document traversal or result insertion order.
for (const { childDocIndex, childDoc, iframeBackendId, source, projection } of children) {
throwIfAborted(signal);
if (projection.status !== "available") frameGeometryIssues.push(projection);
const childContext: FrameContext = { const childContext: FrameContext = {
frameId: snapshotFrameId(childDoc, strings), frameId: source.frameId,
target: source.target,
ownerFrameBackendNodeId: iframeBackendId, ownerFrameBackendNodeId: iframeBackendId,
projection, projection,
scrollX: childDoc.scrollOffsetX ?? 0, scrollX: childDoc.scrollOffsetX ?? 0,
@@ -1023,23 +594,25 @@ function parseChildFrameDocuments(
}; };
const nextVisited = new Set(visited); const nextVisited = new Set(visited);
nextVisited.add(childDocIndex); nextVisited.add(childDocIndex);
const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext); const parsed = await normalizeDocument(childDoc, strings, childContext, signal);
iframeNodes.set(iframeBackendId, parsed.nodes); iframeNodes.set(iframeBackendId, parsed.nodes);
if (childContext.frameId) { if (childContext.frameId) {
frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId); frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId);
frameExcludedBackendNodeIds.set(childContext.frameId, parsed.index.excludedBackendNodeIds);
if (parentContext.frameId) frameParentIds.set(childContext.frameId, parentContext.frameId); if (parentContext.frameId) frameParentIds.set(childContext.frameId, parentContext.frameId);
} }
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id); for (const id of parsed.index.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
const nested = parseChildFrameDocuments( const nested = await parseChildFrameDocuments(
documents, documents,
strings, strings,
dpr,
childDocIndex, childDocIndex,
parsed.nodes,
childContext, childContext,
geometry,
signal,
nextVisited, nextVisited,
); );
frameGeometryIssues.push(...nested.frameGeometryIssues);
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) { for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
iframeNodes.set(nestedFrameId, nestedNodes); iframeNodes.set(nestedFrameId, nestedNodes);
} }
@@ -1049,10 +622,19 @@ function parseChildFrameDocuments(
for (const [frameId, parentFrameId] of nested.frameParentIds) { for (const [frameId, parentFrameId] of nested.frameParentIds) {
frameParentIds.set(frameId, parentFrameId); frameParentIds.set(frameId, parentFrameId);
} }
for (const [frameId, excluded] of nested.frameExcludedBackendNodeIds)
frameExcludedBackendNodeIds.set(frameId, excluded);
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id); for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
} }
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds }; return {
iframeNodes,
frameOwnerBackendNodeIds,
frameParentIds,
frameExcludedBackendNodeIds,
excludedBackendNodeIds,
frameGeometryIssues,
};
} }
export async function captureViewModel( export async function captureViewModel(
@@ -1061,22 +643,20 @@ export async function captureViewModel(
options: CaptureViewModelOptions = {}, options: CaptureViewModelOptions = {},
): Promise<CapturedViewModel> { ): Promise<CapturedViewModel> {
throwIfAborted(options.signal); throwIfAborted(options.signal);
let metrics: LayoutMetricsReply = {}; const target = options.target ?? { tabId };
const geometry = options.geometry ?? new GeometryContext(cdp, tabId, undefined, options.signal);
let metrics: LayoutMetrics = {};
try { try {
metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {}); metrics = await geometry.layoutMetrics(target);
} catch (error) { } catch (error) {
if (isAbortError(error)) throw error; if (isAbortError(error)) throw error;
console.debug("[bsk capture] layout metrics unavailable", error); console.debug("[bsk capture] layout metrics unavailable", error);
} }
throwIfAborted(options.signal); throwIfAborted(options.signal);
const dpr = devicePixelRatio(metrics); const measured = cssViewport(metrics);
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {}; const viewport: Viewport = { width: measured.width, height: measured.height };
const viewport: Viewport = { const scrollX = measured.scrollX;
width: vpSrc.clientWidth ?? 0, const scrollY = measured.scrollY;
height: vpSrc.clientHeight ?? 0,
};
const scrollX = vpSrc.pageX ?? 0;
const scrollY = vpSrc.pageY ?? 0;
await cdp.send(tabId, "DOMSnapshot.enable", {}); await cdp.send(tabId, "DOMSnapshot.enable", {});
throwIfAborted(options.signal); throwIfAborted(options.signal);
@@ -1102,20 +682,43 @@ export async function captureViewModel(
const topContext: FrameContext = { const topContext: FrameContext = {
frameId: snapshotFrameId(doc0, strings), frameId: snapshotFrameId(doc0, strings),
ownerFrameBackendNodeId: null, ownerFrameBackendNodeId: null,
target,
projection: { projection: {
sourceClips: [], status: "available",
edges: [], projection: {
topViewport: viewport, source: { target, frameId: snapshotFrameId(doc0, strings) },
geometry: { sourceClips: [], edges: [], topViewport: viewport },
},
}, },
scrollX, scrollX,
scrollY, scrollY,
}; };
const mainParsed = parseDocumentNodes(doc0, strings, dpr, topContext); const mainParsed = await normalizeDocument(doc0, strings, topContext, options.signal);
const nodes = mainParsed.nodes; const nodes = mainParsed.nodes;
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds); const excludedBackendNodeIds = new Set(mainParsed.index.excludedBackendNodeIds);
const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext); const ownerIds = new Set<number>();
for (const document of documents) {
for (const index of document.nodes?.contentDocumentIndex?.index ?? []) {
const id = document.nodes?.backendNodeId?.[index];
if (id !== undefined) ownerIds.add(id);
}
}
geometry.registerSnapshotOwners(target, ownerIds);
const frameParsed = await parseChildFrameDocuments(
documents,
strings,
0,
topContext,
geometry,
options.signal,
);
const iframeNodes = frameParsed.iframeNodes; const iframeNodes = frameParsed.iframeNodes;
if (topContext.frameId)
frameParsed.frameExcludedBackendNodeIds.set(
topContext.frameId,
mainParsed.index.excludedBackendNodeIds,
);
for (const id of frameParsed.excludedBackendNodeIds) { for (const id of frameParsed.excludedBackendNodeIds) {
excludedBackendNodeIds.add(id); excludedBackendNodeIds.add(id);
} }
@@ -1140,7 +743,9 @@ export async function captureViewModel(
viewport, viewport,
iframeNodes, iframeNodes,
frameNodes, frameNodes,
frameGeometryIssues: frameParsed.frameGeometryIssues,
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds, frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
frameExcludedBackendNodeIds: frameParsed.frameExcludedBackendNodeIds,
frameParentIds: frameParsed.frameParentIds, frameParentIds: frameParsed.frameParentIds,
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}), ...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
excludedBackendNodeIds, excludedBackendNodeIds,
+94
View File
@@ -0,0 +1,94 @@
import { isOverlayHostNode } from "@/lib/overlay-bridge";
import { createCaptureCheckpoint } from "./capture-abort";
import type { CapturedNode } from "./capture-types";
/** Bounds retain their snapshot document CSS units until normalization. */
export interface SnapshotLayout {
readonly boundsSpace: "snapshot-document-css";
bounds?: number[];
styles: Readonly<Record<string, string>>;
}
export interface DecodedNode extends Omit<CapturedNode, "rect" | "localRect" | "rendered"> {
nodeType?: number;
layout?: SnapshotLayout;
}
export interface NodeFacts extends CapturedNode {
nodeType?: number;
layout?: SnapshotLayout;
}
export interface DocumentIndex<T extends DecodedNode = NodeFacts> {
readonly nodes: ReadonlyMap<number, T>;
readonly excludedBackendNodeIds: ReadonlySet<number>;
}
export interface DecodedDocument {
nodes: DecodedNode[];
}
/** All snapshot nodes participate, including document and shadow roots. This
* preserves overlay propagation when the semantic adapter omits those nodes. */
export async function buildDocumentIndex<T extends DecodedNode>(
input: readonly T[],
signal?: AbortSignal,
): Promise<DocumentIndex<T>> {
const checkpoint = createCaptureCheckpoint(signal);
const nodes = new Map<number, T>();
const overlayByNode = new Map<number, boolean>();
const excludedBackendNodeIds = new Set<number>();
for (let i = 0; i < input.length; i++) {
if (i % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const node = input[i];
nodes.set(node.backendNodeId, node);
}
let work = 0;
for (const node of input) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
if (overlayByNode.has(node.backendNodeId)) continue;
const path: DecodedNode[] = [];
const visiting = new Set<number>();
let current: DecodedNode | undefined = node;
let overlay = false;
while (current && !overlayByNode.has(current.backendNodeId)) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
if (visiting.has(current.backendNodeId)) {
// Only an overlay inside the cycle may mark the cycle and its descendants.
overlay = path
.slice(path.findIndex((item) => item.backendNodeId === current!.backendNodeId))
.some((item) => isOverlayHostNode(item.tag, Object.keys(item.attrs)));
break;
}
visiting.add(current.backendNodeId);
path.push(current);
if (current.parentBackendNodeId === null) {
current = undefined;
break;
}
current = nodes.get(current.parentBackendNodeId);
}
if (current && overlayByNode.has(current.backendNodeId))
overlay = overlayByNode.get(current.backendNodeId)!;
for (let i = path.length - 1; i >= 0; i--) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const item = path[i];
overlay = overlay || isOverlayHostNode(item.tag, Object.keys(item.attrs));
overlayByNode.set(item.backendNodeId, overlay);
if (overlay) excludedBackendNodeIds.add(item.backendNodeId);
}
}
return { nodes, excludedBackendNodeIds };
}
@@ -0,0 +1,155 @@
import type { CdpRunner } from "../shared";
import { isAbortError, throwIfAborted } from "./capture-abort";
import type { CapturedNode } from "./capture-types";
interface RuntimeEvaluateReply {
result?: { deepSerializedValue?: DeepSerializedValue };
}
const MAX_FORM_ENRICH_CONTROLS = 250;
interface CapturedFormState {
value?: string;
defaultValue?: string;
placeholder?: string;
state?: "empty" | "filled" | "default";
sensitive?: boolean;
}
interface DeepSerializedValue {
type: string;
value?: unknown;
}
function formStateBatchExpression(maxControls: number): string {
return `(() => {
const maxControls = ${JSON.stringify(maxControls)};
let remaining = maxControls;
const controlSelector = "input,textarea,select";
const controlState = (el) => {
const tag = el.tagName.toLowerCase();
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
const sensitive = type === "password" || type === "credit-card";
const rawValue = typeof el.value === "string" ? el.value : "";
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
return {
state,
sensitive,
placeholder,
...(sensitive ? {} : { value: rawValue, defaultValue }),
};
};
const controls = [];
const collect = (doc) => {
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
if (remaining <= 0) break;
// Deep serialization supplies the node's backend id. Keep the
// state as JSON so decoding needs no general-purpose V8 deserializer.
controls.push([el, JSON.stringify(controlState(el))]);
remaining -= 1;
}
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
if (remaining <= 0) break;
let childDoc = null;
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
if (childDoc) collect(childDoc);
}
};
collect(document);
return controls;
})()`;
}
function formStatesByBackendId(result: RuntimeEvaluateReply): Map<number, CapturedFormState> {
const states = new Map<number, CapturedFormState>();
const serialized = result.result?.deepSerializedValue;
if (serialized?.type !== "array" || !Array.isArray(serialized.value)) return states;
for (const entry of serialized.value as DeepSerializedValue[]) {
if (entry?.type !== "array" || !Array.isArray(entry.value)) continue;
const [element, json] = entry.value as DeepSerializedValue[];
if (element?.type !== "node" || json?.type !== "string" || typeof json.value !== "string") {
continue;
}
const backendNodeId = (element.value as { backendNodeId?: number } | undefined)?.backendNodeId;
if (typeof backendNodeId !== "number" || !Number.isSafeInteger(backendNodeId)) continue;
try {
const state = JSON.parse(json.value) as CapturedFormState | null;
if (
!state ||
!["empty", "filled", "default"].includes(state.state ?? "") ||
typeof state.sensitive !== "boolean" ||
typeof state.placeholder !== "string" ||
(!state.sensitive &&
(typeof state.value !== "string" || typeof state.defaultValue !== "string"))
) {
continue;
}
states.set(backendNodeId, state);
} catch {
// A malformed entry must not overwrite the snapshot's own state.
}
}
return states;
}
function applyFormStates(nodes: CapturedNode[], states: Map<number, CapturedFormState>): void {
for (const node of nodes) {
if (!["input", "textarea", "select"].includes(node.tag)) continue;
const state = states.get(node.backendNodeId);
if (!state) continue;
node.formState = state.state;
node.formPlaceholder = state.placeholder ?? "";
if (
state.sensitive ||
(node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password")
) {
delete node.formValue;
delete node.formDefaultValue;
delete node.attrs.value;
} else {
node.formDefaultValue = state.defaultValue ?? "";
if (state.value !== undefined) node.formValue = state.value;
}
}
}
export async function enrichFormControlStates(
cdp: CdpRunner,
tabId: number,
frameNodeGroups: CapturedNode[][],
signal?: AbortSignal,
): Promise<boolean> {
const hasControls = frameNodeGroups.some((nodes) =>
nodes.some((node) => ["input", "textarea", "select"].includes(node.tag)),
);
if (!hasControls) return true;
const objectGroup = `bsk-vom-forms-${crypto.randomUUID()}`;
try {
throwIfAborted(signal);
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
objectGroup,
serializationOptions: {
serialization: "deep",
maxDepth: 3,
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
},
});
throwIfAborted(signal);
// Backend ids are scoped to this capture's CDP target. OOPIF targets
// have separate enrichment calls and never share this lookup.
const states = formStatesByBackendId(result);
for (const nodes of frameNodeGroups) {
applyFormStates(nodes, states);
}
return true;
} catch (error) {
if (isAbortError(error)) throw error;
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
return false;
} finally {
await cdp.send(tabId, "Runtime.releaseObjectGroup", { objectGroup }).catch(() => undefined);
throwIfAborted(signal);
}
}
+17 -15
View File
@@ -1,6 +1,6 @@
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph"; import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
import { resolveFrameProjection } from "../frame-geometry";
import { type GeometryProjection, projectRectToViewport } from "../geometry"; import { type GeometryProjection, projectRectToViewport } from "../geometry";
import { GeometryContext } from "../geometry/frame-context";
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared"; import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture"; import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
import { import {
@@ -21,16 +21,6 @@ function isAbortError(err: unknown): boolean {
return err instanceof Error && err.name === "AbortError"; return err instanceof Error && err.name === "AbortError";
} }
async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
if (!cdp.getFrameGraph) return null;
try {
return await cdp.getFrameGraph(tabId);
} catch (err) {
console.debug("[bsk observation] frame graph capture failed", err);
return null;
}
}
function transformFrameNodes( function transformFrameNodes(
nodes: CapturedNode[], nodes: CapturedNode[],
projection: GeometryProjection | null, projection: GeometryProjection | null,
@@ -51,6 +41,7 @@ async function captureMissingFrameDocuments(
tabId: number, tabId: number,
graph: CdpFrameGraph, graph: CdpFrameGraph,
captured: CapturedViewModel, captured: CapturedViewModel,
geometry: GeometryContext,
signal?: AbortSignal, signal?: AbortSignal,
): Promise<void> { ): Promise<void> {
if (!captured.frameNodes) captured.frameNodes = new Map(); if (!captured.frameNodes) captured.frameNodes = new Map();
@@ -62,10 +53,20 @@ async function captureMissingFrameDocuments(
try { try {
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, { const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
signal, signal,
geometry,
target: frame.target,
}); });
if (child.frameExcludedBackendNodeIds?.size) {
captured.frameExcludedBackendNodeIds ??= new Map();
for (const [id, excluded] of child.frameExcludedBackendNodeIds)
captured.frameExcludedBackendNodeIds.set(id, excluded);
}
if (child.frameGeometryIssues?.length) {
(captured.frameGeometryIssues ??= []).push(...child.frameGeometryIssues);
}
let projection: GeometryProjection | null = null; let projection: GeometryProjection | null = null;
try { try {
projection = await resolveFrameProjection(cdp, graph, frame.frameId); projection = await geometry.targetProjection(frame.frameId);
} catch (err) { } catch (err) {
if (isAbortError(err)) throw err; if (isAbortError(err)) throw err;
console.debug("[bsk observation] child frame geometry projection failed", { console.debug("[bsk observation] child frame geometry projection failed", {
@@ -171,11 +172,12 @@ export async function captureFrameData<T extends FrameAxNode>(
tabId: number, tabId: number,
captured: CapturedViewModel, captured: CapturedViewModel,
signal?: AbortSignal, signal?: AbortSignal,
geometry = new GeometryContext(cdp, tabId, undefined, signal),
): Promise<CapturedFrameDocument<T>[]> { ): Promise<CapturedFrameDocument<T>[]> {
const graph = await discoverFrameGraph(cdp, tabId); const graph = await geometry.graph();
const frames = graph?.frames ?? []; const frames = graph?.frames ?? [];
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal); if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, geometry, signal);
throwIfAborted(signal); throwIfAborted(signal);
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal); const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
return buildFrameDocuments(graph, batches, captured); return buildFrameDocuments(graph, batches, captured, signal);
} }
+80 -46
View File
@@ -4,7 +4,8 @@ import {
type CdpTarget, type CdpTarget,
cdpTargetKey, cdpTargetKey,
} from "@/browser-driver/frame-graph"; } from "@/browser-driver/frame-graph";
import type { CapturedNode, CapturedViewModel } from "./capture"; import { createCaptureCheckpoint } from "./capture-abort";
import type { CapturedNode } from "./capture-types";
export interface FrameOwnedAxNode { export interface FrameOwnedAxNode {
nodeId: string; nodeId: string;
@@ -23,6 +24,17 @@ export interface FrameDocument<T extends FrameOwnedAxNode> extends CdpFrame {
contextScopeId: string; contextScopeId: string;
axNodes: T[]; axNodes: T[];
domNodes: CapturedNode[]; domNodes: CapturedNode[];
excludedBackendNodeIds?: ReadonlySet<number>;
}
/** Explicit snapshot document data used to join DOM and AX ownership. */
export interface FrameDomInput {
nodes: CapturedNode[];
rootFrameId?: string;
frameNodes?: ReadonlyMap<string, CapturedNode[]>;
frameOwnerBackendNodeIds?: ReadonlyMap<string, number>;
frameParentIds?: ReadonlyMap<string, string>;
frameExcludedBackendNodeIds?: ReadonlyMap<string, ReadonlySet<number>>;
} }
interface Ownership { interface Ownership {
@@ -47,7 +59,7 @@ function targetBackendKey(target: CdpTarget, backendNodeId: number): string {
function frameList<T extends FrameOwnedAxNode>( function frameList<T extends FrameOwnedAxNode>(
graph: CdpFrameGraph | null, graph: CdpFrameGraph | null,
batches: FrameAxBatch<T>[], batches: FrameAxBatch<T>[],
captured: CapturedViewModel, captured: FrameDomInput,
): CdpFrame[] { ): CdpFrame[] {
const frames = new Map<string, CdpFrame>(); const frames = new Map<string, CdpFrame>();
for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame); for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame);
@@ -70,11 +82,14 @@ function frameList<T extends FrameOwnedAxNode>(
}); });
} }
export function buildFrameDocuments<T extends FrameOwnedAxNode>( export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
graph: CdpFrameGraph | null, graph: CdpFrameGraph | null,
batches: FrameAxBatch<T>[], batches: FrameAxBatch<T>[],
captured: CapturedViewModel, captured: FrameDomInput,
): FrameDocument<T>[] { signal?: AbortSignal,
): Promise<FrameDocument<T>[]> {
const checkpoint = createCaptureCheckpoint(signal);
let work = 0;
const frames = frameList(graph, batches, captured); const frames = frameList(graph, batches, captured);
const frameById = new Map(frames.map((frame) => [frame.frameId, frame])); const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId; const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId;
@@ -83,54 +98,68 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
const backendOwner = new Map<string, string>(); const backendOwner = new Map<string, string>();
for (const frame of frames) { for (const frame of frames) {
for (const node of domNodesForFrame(frame.frameId)) { for (const node of domNodesForFrame(frame.frameId)) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId); backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId);
} }
} }
const candidates = new Map<string, OwnedCandidate<T>>(); const candidates = new Map<string, OwnedCandidate<T>>();
for (const batch of batches) { for (const batch of batches) {
const nodeById = new Map(batch.nodes.map((node) => [node.nodeId, node])); const nodeById = new Map<string, T>();
const ownershipByNodeId = new Map<string, Ownership>();
const resolving = new Set<string>();
const resolveOwnership = (node: T): Ownership => {
const cached = ownershipByNodeId.get(node.nodeId);
if (cached) return cached;
if (node.frameId && frameById.has(node.frameId)) {
const ownership = { frameId: node.frameId, strength: 4 };
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
}
if (typeof node.backendDOMNodeId === "number") {
const frameId = backendOwner.get(
targetBackendKey(batch.frame.target, node.backendDOMNodeId),
);
if (frameId) {
const ownership = { frameId, strength: 3 };
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
}
}
if (node.parentId && !resolving.has(node.nodeId)) {
const parent = nodeById.get(node.parentId);
if (parent) {
resolving.add(node.nodeId);
const inherited = resolveOwnership(parent);
resolving.delete(node.nodeId);
const ownership = {
frameId: inherited.frameId,
strength: Math.min(2, inherited.strength),
};
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
}
}
const ownership = { frameId: batch.frame.frameId, strength: 1 };
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
};
for (const node of batch.nodes) { for (const node of batch.nodes) {
const ownership = resolveOwnership(node); if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
nodeById.set(node.nodeId, node);
}
const ownershipByNodeId = new Map<string, Ownership>();
for (const node of batch.nodes) {
const path: T[] = [];
const visiting = new Set<string>();
let current: T | undefined = node;
let ownership: Ownership = { frameId: batch.frame.frameId, strength: 1 };
while (current) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const cached = ownershipByNodeId.get(current.nodeId);
if (cached) {
ownership = cached;
break;
}
if (visiting.has(current.nodeId)) break;
visiting.add(current.nodeId);
if (current.frameId && frameById.has(current.frameId)) {
ownership = { frameId: current.frameId, strength: 4 };
ownershipByNodeId.set(current.nodeId, ownership);
break;
}
const frameId =
typeof current.backendDOMNodeId === "number"
? backendOwner.get(targetBackendKey(batch.frame.target, current.backendDOMNodeId))
: undefined;
if (frameId) {
ownership = { frameId, strength: 3 };
ownershipByNodeId.set(current.nodeId, ownership);
break;
}
path.push(current);
current = current.parentId ? nodeById.get(current.parentId) : undefined;
}
for (let i = path.length - 1; i >= 0; i--) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
ownership = { frameId: ownership.frameId, strength: Math.min(2, ownership.strength) };
ownershipByNodeId.set(path[i].nodeId, ownership);
}
if (!ownershipByNodeId.has(node.nodeId)) ownershipByNodeId.set(node.nodeId, ownership);
const key = targetNodeKey(batch.frame.target, node.nodeId); const key = targetNodeKey(batch.frame.target, node.nodeId);
const existing = candidates.get(key); const existing = candidates.get(key);
if (!existing || ownership.strength > existing.ownership.strength) { if (!existing || ownership.strength > existing.ownership.strength) {
@@ -144,6 +173,10 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
); );
const axNodesByFrame = new Map<string, T[]>(); const axNodesByFrame = new Map<string, T[]>();
for (const candidate of candidates.values()) { for (const candidate of candidates.values()) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const { node, target, ownership } = candidate; const { node, target, ownership } = candidate;
const parentFrameId = node.parentId const parentFrameId = node.parentId
? ownershipByTargetNode.get(targetNodeKey(target, node.parentId)) ? ownershipByTargetNode.get(targetNodeKey(target, node.parentId))
@@ -169,5 +202,6 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
contextScopeId: frame.frameId, contextScopeId: frame.frameId,
axNodes: axNodesByFrame.get(frame.frameId) ?? [], axNodes: axNodesByFrame.get(frame.frameId) ?? [],
domNodes: domNodesForFrame(frame.frameId), domNodes: domNodesForFrame(frame.frameId),
excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(frame.frameId),
})); }));
} }
+73
View File
@@ -0,0 +1,73 @@
import type { CdpTarget } from "@/browser-driver/frame-graph";
import {
type FrameProjectionState,
projectSnapshotRect,
snapshotViewportRect,
} from "../geometry/coordinate-types";
import { createCaptureCheckpoint } from "./capture-abort";
import { buildDocumentIndex, type DocumentIndex, type NodeFacts } from "./facts";
import { decodeDocument, type SnapshotDocument } from "./snapshot";
export interface FrameContext {
frameId?: string;
ownerFrameBackendNodeId: number | null;
projection: FrameProjectionState;
target: CdpTarget;
scrollX: number;
scrollY: number;
}
export interface NormalizedDocument {
nodes: NodeFacts[];
index: DocumentIndex;
}
/** Interpret one document using a supplied projection. No live reads or frame scheduling. */
export async function normalizeDocument(
doc: SnapshotDocument,
strings: string[],
context: FrameContext,
signal?: AbortSignal,
): Promise<NormalizedDocument> {
const decoded = await decodeDocument(doc, strings, signal);
const checkpoint = createCaptureCheckpoint(signal);
const nodes: NodeFacts[] = [];
for (let i = 0; i < decoded.nodes.length; i++) {
if (i % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const node = decoded.nodes[i];
const input = snapshotViewportRect(
node.layout?.bounds ?? [],
{ target: context.target, frameId: context.frameId },
{ x: context.scrollX, y: context.scrollY },
);
const local = input?.rect;
const rect =
input && context.projection.status === "available"
? projectSnapshotRect(input, context.projection.projection)
: null;
const visibility = node.layout?.styles.visibility || "visible";
const opacity = node.layout?.styles.opacity || "1";
nodes.push({
...node,
...(context.frameId ? { frameId: context.frameId } : {}),
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
localRect: local ? { x: local.x, y: local.y, w: local.width, h: local.height } : null,
rect: rect ? { x: rect.x, y: rect.y, w: rect.width, h: rect.height } : null,
rendered:
!!local &&
visibility !== "hidden" &&
visibility !== "collapse" &&
(Number.parseFloat(opacity) || 0) > 0,
});
}
const index = await buildDocumentIndex(nodes, signal);
return {
nodes: nodes.filter(
(node) => !node.tag.startsWith("#") && !index.excludedBackendNodeIds.has(node.backendNodeId),
),
index,
};
}
@@ -2,7 +2,10 @@ import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph"; import type { CdpTarget } from "@/browser-driver/frame-graph";
import { readRecordingDocumentIdentity } from "@/shared/recording-document-identity"; import { readRecordingDocumentIdentity } from "@/shared/recording-document-identity";
import type { CapturedSurfaceProbe } from "./capture"; import type { CapturedSurfaceProbe } from "./capture";
import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture"; import type {
FrameDocument as CapturedFrameDocument,
FrameOwnedAxNode as FrameAxNode,
} from "./frame-document";
/** Frame identity needed to resolve an `@eN` that lives in an iframe. */ /** Frame identity needed to resolve an `@eN` that lives in an iframe. */
export interface CaptureVomFrame { export interface CaptureVomFrame {
@@ -61,8 +61,9 @@ export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGrap
dom, dom,
sourceOrder: index, sourceOrder: index,
excluded: excluded:
document.frameId === rootFrameId && document.excludedBackendNodeIds?.has(dom.backendNodeId) === true ||
input.excludedBackendNodeIds?.has(dom.backendNodeId) === true, (document.frameId === rootFrameId &&
input.excludedBackendNodeIds?.has(dom.backendNodeId) === true),
}; };
nodes.set(id, node); nodes.set(id, node);
frameNodeIds.push(id); frameNodeIds.push(id);
@@ -90,9 +91,10 @@ export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGrap
ax, ax,
sourceOrder: document.domNodes.length + index, sourceOrder: document.domNodes.length + index,
excluded: excluded:
document.frameId === rootFrameId &&
typeof ax.backendDOMNodeId === "number" && typeof ax.backendDOMNodeId === "number" &&
input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true, (document.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true ||
(document.frameId === rootFrameId &&
input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true)),
}); });
frameNodeIds.push(id); frameNodeIds.push(id);
} }
+224
View File
@@ -0,0 +1,224 @@
import { createCaptureCheckpoint } from "./capture-abort";
import type { CapturedNode } from "./capture-types";
import type { DecodedDocument, DecodedNode } from "./facts";
export const REQUESTED_STYLES = [
"position",
"pointer-events",
"cursor",
"visibility",
"opacity",
] as const;
const STYLE_COL = Object.fromEntries(
REQUESTED_STYLES.map((name, index) => [name, index]),
) as Record<(typeof REQUESTED_STYLES)[number], number>;
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
interface SparseArray {
index: number[];
value: number[];
}
interface RareBooleanData {
index: number[];
}
export interface SnapshotDocument {
frameId?: string | number;
scrollOffsetX?: number;
scrollOffsetY?: number;
nodes?: {
parentIndex?: number[];
nodeName?: number[];
nodeType?: number[];
backendNodeId?: number[];
attributes?: number[][];
/**
* Per-node text value (index into strings), set for `#text` / CDATA nodes.
* Element nodes carry -1. Same length as `backendNodeId`.
*/
nodeValue?: number[];
/** Maps node array index → index into `documents[]` for frame content. */
contentDocumentIndex?: SparseArray;
inputValue?: SparseArray;
textValue?: SparseArray;
inputChecked?: RareBooleanData;
optionSelected?: RareBooleanData;
};
layout?: {
nodeIndex?: number[];
styles?: number[][];
bounds?: number[][];
paintOrders?: number[];
};
}
export function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
if (typeof document.frameId === "string") return document.frameId || undefined;
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
return undefined;
}
export interface SnapshotReply {
strings?: string[];
documents?: SnapshotDocument[];
}
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
}
function snapshotFormState(
value: string | undefined,
defaultValue: string,
hasDefaultValue: boolean,
sensitive: boolean,
): CapturedNode["formState"] {
if (sensitive) {
if (value === undefined && !hasDefaultValue) return undefined;
return (value ?? defaultValue) === "" ? "empty" : "filled";
}
if (value === undefined) return undefined;
if (value === "") return "empty";
return value === defaultValue ? "default" : "filled";
}
function str(strings: string[], idx: number | undefined): string {
if (idx === undefined || idx < 0) return "";
return strings[idx] ?? "";
}
export function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
const out = new Map<number, number>();
if (!sparse?.index || !sparse.value) return out;
for (let i = 0; i < sparse.index.length; i++) {
const nodeIndex = sparse.index[i];
const docIndex = sparse.value[i];
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
}
return out;
}
export async function decodeDocument(
doc: SnapshotDocument,
strings: string[],
signal?: AbortSignal,
): Promise<DecodedDocument> {
const checkpoint = createCaptureCheckpoint(signal);
const dn = doc.nodes;
const dl = doc.layout;
if (!dn?.backendNodeId) {
return { nodes: [] };
}
const count = dn.backendNodeId.length;
const layoutByNode = new Map<number, number>();
for (let i = 0; i < (dl?.nodeIndex?.length ?? 0); i++) {
if (i % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
layoutByNode.set(dl!.nodeIndex![i], i);
}
const inputValues = sparseIndexMap(dn.inputValue);
const textValues = sparseIndexMap(dn.textValue);
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
// Collect visible text from #text child nodes so element CapturedNodes
// carry a textContent value usable as a button/link label fallback.
// nodeValue is a parallel array: string index for text nodes, -1 otherwise.
const nodeTextContent = new Map<number, string[]>();
if (dn.nodeValue) {
for (let n = 0; n < count; n++) {
if (n % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const nvIdx = dn.nodeValue[n] ?? -1;
if (nvIdx < 0) continue;
const text = str(strings, nvIdx).trim();
if (!text) continue;
const parentIdx = dn.parentIndex?.[n] ?? -1;
if (parentIdx >= 0) {
const existing = nodeTextContent.get(parentIdx);
if (existing) existing.push(text);
else nodeTextContent.set(parentIdx, [text]);
}
}
}
// Decode fields without assigning a coordinate projection or semantic policy.
const nodes: DecodedNode[] = [];
for (let n = 0; n < count; n++) {
if (n % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const backendNodeId = dn.backendNodeId[n];
const parentIdx = dn.parentIndex?.[n] ?? -1;
const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null;
const tag = str(strings, dn.nodeName?.[n]).toLowerCase();
const attrs: Record<string, string> = {};
const pairs = dn.attributes?.[n] ?? [];
for (let a = 0; a + 1 < pairs.length; a += 2) {
attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]);
}
const li = layoutByNode.get(n);
const styleRow = li === undefined ? [] : (dl?.styles?.[li] ?? []);
const styles: Record<string, string> = {};
for (const name of REQUESTED_STYLES) styles[name] = str(strings, styleRow[STYLE_COL[name]]);
const layout =
li === undefined
? undefined
: {
boundsSpace: "snapshot-document-css" as const,
bounds: dl?.bounds?.[li],
styles,
};
const textContent = nodeTextContent.get(n)?.join(" ");
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
const rawFormValue =
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
const sensitive = isSensitiveFormControl({ tag, attrs });
const formDefaultValue = attrs.value ?? "";
const formValue = sensitive ? undefined : rawFormValue;
const formState = snapshotFormState(
rawFormValue,
formDefaultValue,
Object.prototype.hasOwnProperty.call(attrs, "value"),
sensitive,
);
if (sensitive) delete attrs.value;
nodes.push({
backendNodeId,
nodeType: dn.nodeType?.[n],
parentBackendNodeId,
tag,
attrs,
layout,
paintOrder: li === undefined ? 0 : (dl?.paintOrders?.[li] ?? 0),
position: styles.position || "static",
pointerEvents: styles["pointer-events"] || "auto",
cursor: styles.cursor || "auto",
textContent,
...(formValue !== undefined ? { formValue } : {}),
...(tag === "input" || tag === "textarea"
? {
formPlaceholder: attrs.placeholder ?? "",
...(!sensitive ? { formDefaultValue } : {}),
...(formState ? { formState } : {}),
}
: {}),
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
});
}
return { nodes };
}