Compare commits

...
Author SHA1 Message Date
Ljy-0827 2438b35fc1 fix(observation): preserve snapshot document provenance and URL metadata 2026-09-08 21:42:20 +08:00
Ljy-0827 5fc59b6053 refactor(observation): defer visual facts until visual discovery 2026-09-08 20:42:05 +08:00
Ljy-0827 5fa6b49e91 fix(observation): await capture task cleanup before propagating cancellation 2026-09-08 20:26:52 +08:00
Ljy-0827 b2d2aed754 refactor(observation): organize capture around document facts 2026-09-08 20:09:44 +08:00
Ljy-0827 ad1e44a72b fix(vom): surface incomplete iframe geometry in observations 2026-09-08 20:04:00 +08:00
Ljy-0827 1a86b0aa0e perf(observation): measure sibling iframe geometry with bounded concurrency 2026-09-08 19:51:03 +08:00
Ljy-0827 27dba3c57f fix(geometry): unify frame projection and screenshot coordinates 2026-09-08 17:42:27 +08:00
Zhang GH e8137b42cc Merge pull request #194 from Tencent/fix/dsh-runtime-peer-dependency
fix(dsh-plugin): remove obsolete client runtime host dependency
2026-09-08 17:34:33 +08:00
Zhang GH cb5dc148d5 fix(dsh-plugin): remove obsolete client runtime host dependency
Keep dsh-client-runtime as a development-only type dependency. Remove its peer metadata, client injection entry, and runtime external allowance so newer hosts do not preflight a package they no longer ship.

Fixes #193
2026-09-08 16:58:20 +08:00
27 changed files with 3658 additions and 1873 deletions

No files matched your search

@@ -1,5 +1,5 @@
import { describe, expect, it, vi } from "vitest"; import { describe, expect, it, vi } from "vitest";
import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry"; import { resolveNodeGeometry } from "../frame-geometry";
import { import {
clipPolygon, clipPolygon,
polygonArea, polygonArea,
@@ -10,6 +10,7 @@ import {
rectPolygon, rectPolygon,
regionBounds, regionBounds,
} from "../geometry"; } from "../geometry";
import { GeometryContext } from "../geometry/frame-context";
import type { CdpRunner } from "../shared"; import type { CdpRunner } from "../shared";
describe("frame geometry projection", () => { describe("frame geometry projection", () => {
@@ -82,28 +83,24 @@ describe("frame geometry projection", () => {
throw new Error(`unexpected child command ${method}`); throw new Error(`unexpected child command ${method}`);
}) as CdpRunner["sendToTarget"], }) as CdpRunner["sendToTarget"],
}; };
const projection = await resolveFrameProjection( const projection = await new GeometryContext(cdp, 4, {
cdp, rootFrameId: "main",
{ frames: [
rootFrameId: "main", { frameId: "main", target: { tabId: 4 } },
frames: [ {
{ frameId: "main", target: { tabId: 4 } }, frameId: "same-process-parent",
{ parentFrameId: "main",
frameId: "same-process-parent", ownerBackendNodeId: 10,
parentFrameId: "main", target: { tabId: 4 },
ownerBackendNodeId: 10, },
target: { tabId: 4 }, {
}, frameId: "oopif",
{ parentFrameId: "same-process-parent",
frameId: "oopif", ownerBackendNodeId: 20,
parentFrameId: "same-process-parent", target: { tabId: 4, sessionId: "oopif-session" },
ownerBackendNodeId: 20, },
target: { tabId: 4, sessionId: "oopif-session" }, ],
}, }).targetProjection("oopif");
],
},
"oopif",
);
expect(projection?.edges).toEqual([ expect(projection?.edges).toEqual([
{ {
@@ -0,0 +1,290 @@
import { describe, expect, it, vi } from "vitest";
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
import { resolveNodeGeometry } from "../frame-geometry";
import { rectPolygon } from "../geometry";
import {
projectSnapshotRect,
screenshotPageRect,
snapshotViewportRect,
} from "../geometry/coordinate-types";
import { GeometryContext } from "../geometry/frame-context";
import type { CdpRunner } from "../shared";
const target = { tabId: 4 };
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
{ frameId: "main", target },
{ frameId: "child", parentFrameId: "main", ownerBackendNodeId: 10, target },
],
};
function driver() {
const send = vi.fn(async (_tab: number, method: string): Promise<object> => {
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 1000 } };
if (method === "DOM.getBoxModel")
return { model: { content: [52.5, 612.5, 427.5, 612.5, 427.5, 862.5, 52.5, 862.5] } };
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 300, height: 200 } } };
if (method === "Runtime.releaseObject") return {};
throw new Error(method);
});
return { send: send as CdpRunner["send"], getFrameGraph: vi.fn(async () => graph), calls: send };
}
describe("measurement geometry context", () => {
it("shares in-flight frame measurements, but never shares them with the next context", async () => {
const cdp = driver();
const context = new GeometryContext(cdp, 4);
const results = await Promise.all(
Array.from({ length: 20 }, () => context.targetProjection("child")),
);
expect(results.every((value) => value === results[0])).toBe(true);
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1);
expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([
"Page.getLayoutMetrics",
"DOM.getBoxModel",
]);
await new GeometryContext(cdp, 4).targetProjection("child");
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(2);
expect(cdp.calls.mock.calls).toHaveLength(4);
});
it("projects frame-local snapshot CSS bounds through the scaled owner content quad", async () => {
const cdp = driver();
const context = new GeometryContext(cdp, 4);
const source = { target, frameId: "child" };
const projection = await context.snapshotProjection(source, 10, [], {
width: 1000,
height: 1000,
});
if (projection.status !== "available") throw new Error("expected available projection");
const input = snapshotViewportRect([17, 23, 120, 40], source, { x: 0, y: 0 });
expect(projectSnapshotRect(input!, projection.projection)).toEqual({
x: 73.75,
y: 641.25,
width: 150,
height: 50,
});
const scrolled = snapshotViewportRect([17, 23, 120, 40], source, { x: 0, y: 10 });
expect(projectSnapshotRect(scrolled!, projection.projection)?.y).toBe(628.75);
await context.snapshotProjection(source, 10, [], { width: 1000, height: 1000 });
expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.resolveNode")).toHaveLength(
1,
);
expect(
cdp.calls.mock.calls.filter(([, method]) => method === "Runtime.releaseObject"),
).toHaveLength(1);
});
it("rejects owner mismatches and invalid numeric bounds", () => {
const input = snapshotViewportRect(
[30, 50, 120, 40],
{ target, frameId: "main" },
{ x: 10, y: 20 },
)!;
const projection = {
source: { target, frameId: "main" },
geometry: { sourceClips: [], edges: [], topViewport: { width: 800, height: 600 } },
};
expect(projectSnapshotRect(input, projection)).toEqual({
x: 20,
y: 30,
width: 120,
height: 40,
});
expect(
projectSnapshotRect(input, { ...projection, source: { target, frameId: "other" } }),
).toBeNull();
expect(
projectSnapshotRect(input, {
...projection,
source: { target: { tabId: 5 }, frameId: "main" },
}),
).toBeNull();
expect(snapshotViewportRect([0, 0, NaN, 40], { target }, { x: 0, y: 0 })).toBeNull();
expect(snapshotViewportRect([0, 0, 10, 40], { target }, { x: Infinity, y: 0 })).toBeNull();
});
it("does not turn a border quad or a failed owner read into a content projection", async () => {
const cdp: CdpRunner = {
send: vi.fn(async () => ({
model: { border: [0, 0, 100, 0, 100, 100, 0, 100] },
})) as CdpRunner["send"],
};
expect(
await new GeometryContext(cdp, 4).snapshotProjection({ target, frameId: "child" }, 10, [], {
width: 100,
height: 100,
}),
).toEqual({
status: "unavailable",
source: { target, frameId: "child" },
ownerBackendNodeId: 10,
});
});
it("keeps ancestor clips in target coordinates instead of applying nested offsets twice", async () => {
const cdp = driver();
const projection = await new GeometryContext(cdp, 4).snapshotProjection(
{ target, frameId: "nested" },
10,
[rectPolygon({ x: 60, y: 620, w: 100, h: 100 })],
{ width: 1000, height: 1000 },
);
if (projection.status !== "available") throw new Error("expected available projection");
const input = snapshotViewportRect(
[0, 0, 300, 200],
{ target, frameId: "nested" },
{ x: 0, y: 0 },
);
expect(projectSnapshotRect(input!, projection.projection)).toEqual({
x: 60,
y: 620,
width: 100,
height: 100,
});
});
it("rejects malformed frame ancestry without looping", async () => {
const cdp = driver();
const cycle: CdpFrameGraph = {
rootFrameId: "a",
frames: [
{ frameId: "a", parentFrameId: "b", target },
{ frameId: "b", parentFrameId: "a", target },
],
};
expect(await new GeometryContext(cdp, 4, cycle).targetProjection("a")).toBeNull();
expect(cdp.calls).not.toHaveBeenCalled();
const orphan: CdpFrameGraph = {
rootFrameId: "a",
frames: [{ frameId: "a", parentFrameId: "missing", target }],
};
expect(await new GeometryContext(cdp, 4, orphan).targetProjection("a")).toBeNull();
});
it("rejects a live node whose frame belongs to another target before any DOM input", async () => {
const cdp = driver();
expect(
await resolveNodeGeometry(
cdp,
4,
{ target: { tabId: 4, sessionId: "wrong" }, frameId: "child", backendNodeId: 20 },
{ scrollIntoView: true },
),
).toMatchObject({ code: "cdp_failed" });
expect(cdp.calls).not.toHaveBeenCalled();
});
it("bounds concurrent measurements and does not dispatch queued reads after cancellation", async () => {
let active = 0;
let peak = 0;
const release: Array<() => void> = [];
const send = vi.fn(async () => {
active++;
peak = Math.max(peak, active);
await new Promise<void>((resolve) => release.push(resolve));
active--;
return { cssLayoutViewport: { clientWidth: 100, clientHeight: 100 } };
});
const controller = new AbortController();
const context = new GeometryContext(
{ send: send as CdpRunner["send"] },
4,
undefined,
controller.signal,
);
const tasks = Array.from({ length: 8 }, (_, index) =>
context.viewport({ tabId: 4, sessionId: String(index) }),
);
const settled = Promise.allSettled(tasks);
expect(send).toHaveBeenCalledTimes(4);
controller.abort();
for (const done of release) done();
const results = await settled;
expect(peak).toBe(4);
expect(send).toHaveBeenCalledTimes(4);
expect(results.filter((item) => item.status === "rejected")).toHaveLength(4);
});
it("releases an owner object even when cancellation arrives before its viewport read", async () => {
const controller = new AbortController();
const cdp = driver();
const original = cdp.calls.getMockImplementation()!;
cdp.calls.mockImplementation(async (tab, method) => {
const result = await original(tab, method);
if (method === "DOM.resolveNode") controller.abort();
return result;
});
const context = new GeometryContext(cdp, 4, undefined, controller.signal);
await expect(
context.snapshotProjection({ target, frameId: "child" }, 10, [], {
width: 1000,
height: 1000,
}),
).rejects.toMatchObject({ name: "AbortError" });
expect(cdp.calls.mock.calls.map(([, method]) => method)).toEqual([
"DOM.getBoxModel",
"DOM.resolveNode",
"Runtime.releaseObject",
]);
});
it("reads live owner geometry only after scrolling", async () => {
const cdp = driver();
const original = cdp.calls.getMockImplementation()!;
let scrolled = false;
cdp.calls.mockImplementation(async (tab, method) => {
if (method === "DOM.scrollIntoViewIfNeeded") {
scrolled = true;
return {};
}
if (method === "DOM.getContentQuads") {
expect(scrolled).toBe(true);
return { quads: [[60, 620, 100, 620, 100, 640, 60, 640]] };
}
if (method === "DOM.getBoxModel") expect(scrolled).toBe(true);
return original(tab, method);
});
expect(
await resolveNodeGeometry(
cdp,
4,
{ target, frameId: "child", backendNodeId: 20 },
{ scrollIntoView: true },
),
).toMatchObject({ topBounds: { x: 60, y: 620, width: 40, height: 20 } });
expect(cdp.getFrameGraph).toHaveBeenCalledTimes(1);
expect(cdp.calls.mock.calls.filter(([, method]) => method === "DOM.getBoxModel")).toHaveLength(
1,
);
});
it("adapts viewport CSS bounds to page DIP with scroll and browser zoom, never raster DPR", () => {
const viewport = { width: 1600, height: 1000, scrollX: 10, scrollY: 100, cssToDip: 0.9 };
const clip = screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, viewport);
expect(clip).toEqual({ space: "page-dip", rect: { x: 27, y: 117, width: 108, height: 36 } });
expect(
screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: NaN }),
).toBeNull();
expect(
screenshotPageRect({ x: 20, y: 30, width: 120, height: 40 }, { ...viewport, cssToDip: 0 }),
).toBeNull();
});
it("retries a rejected measurement only in a new operation", async () => {
const cdp = driver();
cdp.calls.mockRejectedValueOnce(new Error("unavailable"));
const context = new GeometryContext(cdp, 4);
await expect(context.viewport(target)).rejects.toThrow("unavailable");
await expect(context.viewport(target)).rejects.toThrow("unavailable");
expect(cdp.calls).toHaveBeenCalledTimes(1);
expect(await new GeometryContext(cdp, 4).viewport(target)).toEqual({
width: 1000,
height: 1000,
});
});
});
File diff suppressed because it is too large. Load diff
+46 -151
View File
@@ -1,24 +1,21 @@
import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph"; import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
import type { RpcError } from "@/transport/types"; import type { RpcError } from "@/transport/types";
import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry"; import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry";
import { import {
clipPolygon, clipPolygon,
type GeometryProjection,
type Point, type Point,
type Polygon, type Polygon,
type ProjectiveEdge,
parseCdpQuad,
polygonArea, polygonArea,
polygonCentroid, polygonCentroid,
projectRegionToViewport, projectRegionToViewport,
type Quad,
type Region, type Region,
rectPolygon, rectPolygon,
regionBounds, regionBounds,
type Size,
type ViewportRect, type ViewportRect,
} from "./geometry"; } from "./geometry";
import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared"; import type { CssViewport } from "./geometry/coordinate-types";
import { cssViewport, GeometryContext } from "./geometry/frame-context";
import { type CdpRunner, cdpRunnerForTarget, isRpcError } from "./shared";
export interface NodeAddress { export interface NodeAddress {
target: CdpTarget; target: CdpTarget;
@@ -27,6 +24,8 @@ export interface NodeAddress {
} }
export interface ResolvedNodeGeometry { export interface ResolvedNodeGeometry {
/** The same top-level measurement used by projection, for screenshot coordinate adaptation. */
topViewport: CssViewport;
topVisibleRegions: Region; topVisibleRegions: Region;
topBounds: ViewportRect; topBounds: ViewportRect;
/** Point in the top-level tab viewport, used by root-target input events. */ /** Point in the top-level tab viewport, used by root-target input events. */
@@ -39,145 +38,16 @@ function geometryError(message: string): RpcError {
return { code: "cdp_failed", message }; return { code: "cdp_failed", message };
} }
function frameMap(graph: CdpFrameGraph): Map<string, CdpFrame> {
return new Map(graph.frames.map((frame) => [frame.frameId, frame]));
}
function targetRootFrame(byId: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
const seen = new Set<string>();
let current = frame;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
const parent = byId.get(current.parentFrameId);
if (!parent || parent.target.sessionId !== current.target.sessionId) break;
current = parent;
}
return current;
}
function frameAncestry(graph: CdpFrameGraph, frameId: string): CdpFrame[] | null {
const byId = frameMap(graph);
const path: CdpFrame[] = [];
const seen = new Set<string>();
let current = byId.get(frameId);
if (!current) return null;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
path.push(current);
const parent = byId.get(current.parentFrameId);
if (!parent) return null;
current = parent;
}
return path;
}
async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise<Size | null> {
const metrics = await sendToCdpTarget<{
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
layoutViewport?: { clientWidth?: number; clientHeight?: number };
}>(cdp, target, "Page.getLayoutMetrics", {});
const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
const width = viewport.clientWidth ?? 0;
const height = viewport.clientHeight ?? 0;
return width > 0 && height > 0 ? { width, height } : null;
}
async function ownerContentQuad(
cdp: CdpRunner,
parent: CdpFrame,
ownerBackendNodeId: number,
): Promise<Quad | null> {
const result = await sendToCdpTarget<{ model?: { content?: number[] } }>(
cdp,
parent.target,
"DOM.getBoxModel",
{ backendNodeId: ownerBackendNodeId },
);
return parseCdpQuad(result.model?.content);
}
async function sameTargetFrameClips(
cdp: CdpRunner,
byId: Map<string, CdpFrame>,
frame: CdpFrame,
targetRoot: CdpFrame,
): Promise<Polygon[] | null> {
const clips: Polygon[] = [];
let current = frame;
const seen = new Set<string>();
while (current.frameId !== targetRoot.frameId) {
if (seen.has(current.frameId) || !current.parentFrameId) return null;
seen.add(current.frameId);
const parent = byId.get(current.parentFrameId);
if (!parent || current.ownerBackendNodeId === undefined) return null;
const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId);
if (!clip) return null;
clips.push(clip);
current = parent;
}
return clips;
}
export async function resolveFrameProjection(
cdp: CdpRunner,
graph: CdpFrameGraph,
frameId: string,
): Promise<GeometryProjection | null> {
const byId = frameMap(graph);
const frame = byId.get(frameId);
if (!frame) return null;
let targetRoot = targetRootFrame(byId, frame);
if (!targetRoot) return null;
const sourceViewport = await targetViewport(cdp, targetRoot.target);
if (!sourceViewport) return null;
const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot);
if (!sourceClips) return null;
const edges: ProjectiveEdge[] = [];
while (targetRoot.parentFrameId) {
const parent = byId.get(targetRoot.parentFrameId);
if (!parent || targetRoot.ownerBackendNodeId === undefined) return null;
const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId);
if (!destinationQuad) return null;
const source =
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
if (!source) return null;
const parentTargetRoot = targetRootFrame(byId, parent);
if (!parentTargetRoot) return null;
const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot);
if (!destinationClips) return null;
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
targetRoot = parentTargetRoot;
}
const topViewport =
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
return topViewport ? { sourceClips, edges, topViewport } : null;
}
async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
if (!cdp.getFrameGraph) return null;
try {
return await cdp.getFrameGraph(tabId);
} catch (error) {
console.debug("[bsk frame-geometry] frame graph resolution failed", error);
return null;
}
}
async function scrollFrameOwners( async function scrollFrameOwners(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
graph: CdpFrameGraph, context: GeometryContext,
frameId: string, frameId: string,
): Promise<RpcError | null> { ): Promise<RpcError | null> {
const byId = frameMap(graph); const ancestry = await context.ancestry(frameId);
const ancestry = frameAncestry(graph, frameId);
if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`); if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`);
for (const child of [...ancestry].reverse()) { for (const child of [...ancestry].reverse()) {
const parent = child.parentFrameId ? byId.get(child.parentFrameId) : undefined; const parent = child.parentFrameId ? await context.frame(child.parentFrameId) : undefined;
if (!parent || child.ownerBackendNodeId === undefined) { if (!parent || child.ownerBackendNodeId === undefined) {
return geometryError(`could not resolve frame owner for ${child.frameId}`); return geometryError(`could not resolve frame owner for ${child.frameId}`);
} }
@@ -191,17 +61,20 @@ async function scrollFrameOwners(
return null; return null;
} }
async function scrollElementWithFrameGraph( async function scrollElementWithContext(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
target: CdpTarget, target: CdpTarget,
backendNodeId: number, backendNodeId: number,
frameId: string | undefined, frameId: string | undefined,
graph: CdpFrameGraph | null, context: GeometryContext,
): Promise<RpcError | null> { ): Promise<RpcError | null> {
if (frameId) { if (frameId) {
if (!graph) return geometryError(`could not resolve frame graph for ${frameId}`); const frame = await context.frame(frameId);
const error = await scrollFrameOwners(cdp, tabId, graph, frameId); if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(target)) {
return geometryError("node frame does not belong to its target");
}
const error = await scrollFrameOwners(cdp, tabId, context, frameId);
if (error) return error; if (error) return error;
} else if (target.sessionId) { } else if (target.sessionId) {
return geometryError("an OOPIF node address requires frameId"); return geometryError("an OOPIF node address requires frameId");
@@ -216,8 +89,15 @@ export async function scrollElementAndFramesIntoView(
backendNodeId: number, backendNodeId: number,
frameId?: string, frameId?: string,
): Promise<RpcError | null> { ): Promise<RpcError | null> {
const graph = frameId ? await loadFrameGraph(cdp, tabId) : null; if (target.tabId !== tabId) return geometryError("node target belongs to another tab");
return scrollElementWithFrameGraph(cdp, tabId, target, backendNodeId, frameId, graph); return scrollElementWithContext(
cdp,
tabId,
target,
backendNodeId,
frameId,
new GeometryContext(cdp, tabId),
);
} }
function largestRegion(regions: Region): Polygon | null { function largestRegion(regions: Region): Polygon | null {
@@ -240,23 +120,32 @@ export async function resolveNodeGeometry(
if (address.target.sessionId && !address.frameId) { if (address.target.sessionId && !address.frameId) {
return geometryError("an OOPIF node address requires frameId"); return geometryError("an OOPIF node address requires frameId");
} }
const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null; if (address.target.tabId !== tabId) return geometryError("node target belongs to another tab");
const context = new GeometryContext(cdp, tabId);
const graph = address.frameId ? await context.graph() : null;
if (address.frameId && graph) {
const frame = await context.frame(address.frameId);
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(address.target)) {
return geometryError("node frame does not belong to its target");
}
}
if (address.frameId && !graph) { if (address.frameId && !graph) {
return geometryError(`could not resolve frame graph for ${address.frameId}`); return geometryError(`could not resolve frame graph for ${address.frameId}`);
} }
if (options.scrollIntoView) { if (options.scrollIntoView) {
const scrollError = await scrollElementWithFrameGraph( const scrollError = await scrollElementWithContext(
cdp, cdp,
tabId, tabId,
address.target, address.target,
address.backendNodeId, address.backendNodeId,
address.frameId, address.frameId,
graph, context,
); );
if (scrollError) return scrollError; if (scrollError) return scrollError;
} }
// Only topology was read above. Begin live measurements after scrolling.
const localRegion = await nodeContentRegion( const localRegion = await nodeContentRegion(
cdpRunnerForTarget(cdp, address.target), cdpRunnerForTarget(cdp, address.target),
tabId, tabId,
@@ -267,7 +156,7 @@ export async function resolveNodeGeometry(
let topVisibleRegions: Region; let topVisibleRegions: Region;
let targetActionPoint: Point | null = null; let targetActionPoint: Point | null = null;
if (address.frameId && graph) { if (address.frameId && graph) {
const projection = await resolveFrameProjection(cdp, graph, address.frameId); const projection = await context.targetProjection(address.frameId);
if (!projection) if (!projection)
return geometryError(`could not resolve frame geometry for ${address.frameId}`); return geometryError(`could not resolve frame geometry for ${address.frameId}`);
topVisibleRegions = projectRegionToViewport(localRegion, projection); topVisibleRegions = projectRegionToViewport(localRegion, projection);
@@ -290,7 +179,7 @@ export async function resolveNodeGeometry(
targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null; targetActionPoint = localActionRegion ? polygonCentroid(localActionRegion) : null;
} }
} else { } else {
const viewport = await targetViewport(cdp, address.target); const viewport = await context.viewport(address.target);
if (!viewport) return geometryError("could not resolve top viewport geometry"); if (!viewport) return geometryError("could not resolve top viewport geometry");
topVisibleRegions = localRegion topVisibleRegions = localRegion
.map((polygon) => .map((polygon) =>
@@ -309,7 +198,13 @@ export async function resolveNodeGeometry(
return { code: "permission_denied", message: "element not visible in its target" }; return { code: "permission_denied", message: "element not visible in its target" };
} }
targetActionPoint ??= actionPoint; targetActionPoint ??= actionPoint;
return { topVisibleRegions, topBounds, actionPoint, targetActionPoint }; return {
topVisibleRegions,
topBounds,
actionPoint,
targetActionPoint,
topViewport: cssViewport(await context.layoutMetrics({ tabId })),
};
} catch (error) { } catch (error) {
return geometryError(error instanceof Error ? error.message : String(error)); return geometryError(error instanceof Error ? error.message : String(error));
} }
-19
View File
@@ -245,22 +245,3 @@ export function projectRectToViewport(
if (!rect) return null; if (!rect) return null;
return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection)); return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection));
} }
export function childFrameProjection(
parent: GeometryProjection,
ownerRectInParent: { x: number; y: number; w: number; h: number },
): GeometryProjection {
const destinationQuad = rectPolygon(ownerRectInParent);
return {
sourceClips: [],
edges: [
{
sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
destinationQuad,
destinationClips: parent.sourceClips,
},
...parent.edges,
],
topViewport: parent.topViewport,
};
}
@@ -0,0 +1,102 @@
import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
import type { GeometryProjection, ViewportRect } from "../geometry";
import { projectRectToViewport } from "../geometry";
export interface CoordinateOwner {
target: CdpTarget;
frameId?: string;
}
export interface FrameViewportRect {
space: "frame-viewport-css";
owner: CoordinateOwner;
rect: ViewportRect;
}
export interface SnapshotProjection {
source: CoordinateOwner;
geometry: GeometryProjection;
}
/** A failed owner read is different from a successfully projected, clipped-out rect. */
export interface UnavailableFrameProjection {
status: "unavailable";
source: CoordinateOwner;
ownerBackendNodeId: number;
}
export type SnapshotProjectionResult =
| { status: "available"; projection: SnapshotProjection }
| UnavailableFrameProjection;
/** Descendants inherit the failed boundary without attempting another owner read. */
export type FrameProjectionIssue =
| UnavailableFrameProjection
| { status: "blocked"; source: CoordinateOwner; cause: UnavailableFrameProjection };
export type FrameProjectionState = SnapshotProjectionResult | FrameProjectionIssue;
/** DOMSnapshot bounds are document-relative CSS pixels, independent of raster DPR. */
export function snapshotViewportRect(
bounds: number[],
owner: CoordinateOwner,
scroll: { x: number; y: number },
): FrameViewportRect | null {
if (bounds.length < 4 || !bounds.slice(0, 4).every(Number.isFinite)) return null;
const [x, y, width, height] = bounds;
if (width <= 0 || height <= 0 || !Number.isFinite(scroll.x) || !Number.isFinite(scroll.y)) {
return null;
}
return {
space: "frame-viewport-css",
owner,
rect: { x: x - scroll.x, y: y - scroll.y, width, height },
};
}
export function projectSnapshotRect(
input: FrameViewportRect,
projection: SnapshotProjection,
): ViewportRect | null {
if (
input.owner.frameId !== projection.source.frameId ||
cdpTargetKey(input.owner.target) !== cdpTargetKey(projection.source.target)
) {
return null;
}
const { x, y, width: w, height: h } = input.rect;
return projectRectToViewport({ x, y, w, h }, projection.geometry);
}
/** CSS viewport metadata; browser zoom is distinct from raster devicePixelRatio. */
export interface CssViewport {
width: number;
height: number;
scrollX: number;
scrollY: number;
cssToDip: number;
}
/** Page.captureScreenshot takes page-relative DIP, not viewport-relative CSS pixels. */
export function screenshotPageRect(
rect: ViewportRect,
viewport: CssViewport,
): { space: "page-dip"; rect: ViewportRect } | null {
const { cssToDip, scrollX, scrollY } = viewport;
if (
![rect.x, rect.y, rect.width, rect.height, cssToDip, scrollX, scrollY].every(Number.isFinite) ||
cssToDip <= 0 ||
rect.width <= 0 ||
rect.height <= 0
)
return null;
return {
space: "page-dip",
rect: {
x: (rect.x + scrollX) * cssToDip,
y: (rect.y + scrollY) * cssToDip,
width: rect.width * cssToDip,
height: rect.height * cssToDip,
},
};
}
@@ -0,0 +1,308 @@
import {
type CdpFrame,
type CdpFrameGraph,
type CdpTarget,
cdpTargetKey,
} from "@/browser-driver/frame-graph";
import {
type GeometryProjection,
type Polygon,
type ProjectiveEdge,
parseCdpQuad,
type Quad,
type Size,
} from "../geometry";
import { type CdpRunner, sendToCdpTarget } from "../shared";
import type { CoordinateOwner, CssViewport, SnapshotProjectionResult } from "./coordinate-types";
export interface LayoutMetrics {
cssVisualViewport?: { zoom?: number };
visualViewport?: { zoom?: number };
cssLayoutViewport?: {
clientWidth?: number;
clientHeight?: number;
pageX?: number;
pageY?: number;
};
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
}
/** The legacy field is a compatibility fallback, never a source of raster DPR. */
export function cssViewport(metrics: LayoutMetrics): CssViewport {
const value = metrics.cssLayoutViewport ?? metrics.layoutViewport;
return {
width: value?.clientWidth ?? 0,
height: value?.clientHeight ?? 0,
scrollX: value?.pageX ?? 0,
scrollY: value?.pageY ?? 0,
cssToDip: metrics.cssVisualViewport?.zoom ?? metrics.visualViewport?.zoom ?? 1,
};
}
/** One read-only measurement phase. Discard after scrolling or any later operation. */
export class GeometryContext {
private readonly metrics = new Map<string, Promise<LayoutMetrics>>();
private readonly owners = new Map<string, Promise<Quad | null>>();
private readonly projections = new Map<string, Promise<GeometryProjection | null>>();
private readonly snapshotOwners = new Map<string, Promise<{ quad: Quad; size: Size } | null>>();
private active = 0;
private readonly waiting: Array<() => void> = [];
private graphPromise?: Promise<CdpFrameGraph | null>;
private framesPromise?: Promise<Map<string, CdpFrame>>;
constructor(
private readonly cdp: CdpRunner,
private readonly tabId: number,
graph?: CdpFrameGraph,
private readonly signal?: AbortSignal,
) {
if (graph) this.graphPromise = Promise.resolve(graph);
}
private async request<T>(
target: CdpTarget,
method: string,
params: object,
cleanup = false,
): Promise<T> {
if (!cleanup && this.signal?.aborted) throw new DOMException("geometry aborted", "AbortError");
if (this.active >= 4) await new Promise<void>((resolve) => this.waiting.push(resolve));
else this.active++;
try {
if (!cleanup && this.signal?.aborted)
throw new DOMException("geometry aborted", "AbortError");
return await sendToCdpTarget<T>(this.cdp, target, method, params);
} finally {
const next = this.waiting.shift();
if (next) next();
else this.active--;
}
}
graph(): Promise<CdpFrameGraph | null> {
this.graphPromise ??=
this.cdp.getFrameGraph?.(this.tabId).catch(() => null) ?? Promise.resolve(null);
return this.graphPromise;
}
private frames(): Promise<Map<string, CdpFrame>> {
this.framesPromise ??= this.graph().then(
(graph) => new Map(graph?.frames.map((frame) => [frame.frameId, frame]) ?? []),
);
return this.framesPromise;
}
async frame(frameId: string): Promise<CdpFrame | undefined> {
return (await this.frames()).get(frameId);
}
/** Topology only: safe to resolve before the caller scrolls frame owners. */
async ancestry(frameId: string): Promise<CdpFrame[] | null> {
const frames = await this.frames();
const path: CdpFrame[] = [];
const seen = new Set<string>();
let current = frames.get(frameId);
if (!current) return null;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
path.push(current);
const parent = frames.get(current.parentFrameId);
if (!parent) return null;
current = parent;
}
return path;
}
layoutMetrics(target: CdpTarget): Promise<LayoutMetrics> {
const key = cdpTargetKey(target);
let promise = this.metrics.get(key);
if (!promise) {
promise = this.request<LayoutMetrics>(target, "Page.getLayoutMetrics", {});
this.metrics.set(key, promise);
}
return promise;
}
async viewport(target: CdpTarget): Promise<Size | null> {
const { width, height } = cssViewport(await this.layoutMetrics(target));
return Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0
? { width, height }
: null;
}
ownerContent(target: CdpTarget, backendNodeId: number): Promise<Quad | null> {
const key = `${cdpTargetKey(target)}:${backendNodeId}`;
let promise = this.owners.get(key);
if (!promise) {
promise = this.request<{ model?: { content?: number[] } }>(target, "DOM.getBoxModel", {
backendNodeId,
}).then((reply) => parseCdpQuad(reply.model?.content));
this.owners.set(key, promise);
}
return promise;
}
/** Same-target owner quads are already target-relative, including nested owners. */
async snapshotProjection(
source: CoordinateOwner,
ownerBackendNodeId: number,
ancestorClips: Polygon[],
viewport: Size,
): Promise<SnapshotProjectionResult> {
const key = `${cdpTargetKey(source.target)}:${ownerBackendNodeId}`;
let promise = this.snapshotOwners.get(key);
if (!promise) {
promise = this.snapshotOwner(source.target, ownerBackendNodeId);
this.snapshotOwners.set(key, promise);
}
let owner: { quad: Quad; size: Size } | null;
try {
owner = await promise;
} catch (error) {
if (error && typeof error === "object" && "name" in error && error.name === "AbortError")
throw error;
console.debug("[bsk geometry] frame owner unavailable", error);
owner = null;
}
return owner
? {
status: "available",
projection: {
source,
geometry: {
sourceClips: [],
edges: [
{
sourceViewport: owner.size,
destinationQuad: owner.quad,
destinationClips: ancestorClips,
},
],
topViewport: viewport,
},
},
}
: { status: "unavailable", source, ownerBackendNodeId };
}
private async snapshotOwner(
target: CdpTarget,
backendNodeId: number,
): Promise<{ quad: Quad; size: Size } | null> {
const quad = await this.ownerContent(target, backendNodeId);
if (!quad) return null;
const resolved = await this.request<{ object?: { objectId?: string } }>(
target,
"DOM.resolveNode",
{ backendNodeId },
);
const objectId = resolved.object?.objectId;
if (!objectId) return null;
try {
const reply = await this.request<{ result?: { value?: Size }; exceptionDetails?: unknown }>(
target,
"Runtime.callFunctionOn",
{
objectId,
functionDeclaration: `function() {
if (!this.isConnected) return null;
const style = getComputedStyle(this);
return {
width: this.clientWidth - parseFloat(style.paddingLeft) - parseFloat(style.paddingRight),
height: this.clientHeight - parseFloat(style.paddingTop) - parseFloat(style.paddingBottom)
};
}`,
returnByValue: true,
},
);
const size = reply.result?.value;
return !reply.exceptionDetails &&
size &&
Number.isFinite(size.width) &&
Number.isFinite(size.height) &&
size.width > 0 &&
size.height > 0
? { quad, size }
: null;
} finally {
// Cleanup must still run if cancellation arrived after resolving the object.
await this.request(target, "Runtime.releaseObject", { objectId }, true).catch(() => {});
}
}
private targetRoot(frames: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
const seen = new Set<string>();
let current = frame;
while (current.parentFrameId) {
if (seen.has(current.frameId)) return null;
seen.add(current.frameId);
const parent = frames.get(current.parentFrameId);
if (!parent) return null;
if (cdpTargetKey(parent.target) !== cdpTargetKey(current.target)) break;
current = parent;
}
return current;
}
private async clips(
frames: Map<string, CdpFrame>,
frame: CdpFrame,
root: CdpFrame,
): Promise<Polygon[] | null> {
const clips: Polygon[] = [];
let current = frame;
const seen = new Set<string>();
while (current.frameId !== root.frameId) {
if (seen.has(current.frameId) || !current.parentFrameId) return null;
seen.add(current.frameId);
const parent = frames.get(current.parentFrameId);
if (!parent || current.ownerBackendNodeId === undefined) return null;
const quad = await this.ownerContent(parent.target, current.ownerBackendNodeId);
if (!quad) return null;
clips.push(quad);
current = parent;
}
return clips;
}
/** Input is already in the frame's CDP target viewport, not its local viewport. */
targetProjection(frameId: string): Promise<GeometryProjection | null> {
let promise = this.projections.get(frameId);
if (!promise) {
promise = this.buildTargetProjection(frameId);
this.projections.set(frameId, promise);
}
return promise;
}
private async buildTargetProjection(frameId: string): Promise<GeometryProjection | null> {
const frames = await this.frames();
const frame = frames.get(frameId);
if (!frame) return null;
let root = this.targetRoot(frames, frame);
if (!root) return null;
const sourceViewport = await this.viewport(root.target);
if (!sourceViewport) return null;
const sourceClips = await this.clips(frames, frame, root);
if (!sourceClips) return null;
const edges: ProjectiveEdge[] = [];
const seen = new Set<string>();
while (root.parentFrameId) {
if (seen.has(root.frameId)) return null;
seen.add(root.frameId);
const parent = frames.get(root.parentFrameId);
if (!parent || root.ownerBackendNodeId === undefined) return null;
const destinationQuad = await this.ownerContent(parent.target, root.ownerBackendNodeId);
const source = await this.viewport(root.target);
const parentRoot = this.targetRoot(frames, parent);
if (!destinationQuad || !source || !parentRoot) return null;
const destinationClips = await this.clips(frames, parent, parentRoot);
if (!destinationClips) return null;
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
root = parentRoot;
}
const topViewport = await this.viewport(root.target);
return topViewport ? { sourceClips, edges, topViewport } : null;
}
}
+24 -222
View File
@@ -1,3 +1,4 @@
import type { CapturedSceneInput } from "./vom/facts";
// Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`, // Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`,
// and semantic `tool.observe` (design §7). Each handler resolves the target // and semantic `tool.observe` (design §7). Each handler resolves the target
// tab (defaulting to the Agent Window's active tab when omitted) and // tab (defaulting to the Agent Window's active tab when omitted) and
@@ -34,6 +35,7 @@ import type {
import { attachDialogs, markDialogCursor } from "./dialogs"; import { attachDialogs, markDialogCursor } from "./dialogs";
import { rpcError } from "./errors"; import { rpcError } from "./errors";
import { resolveNodeGeometry } from "./frame-geometry"; import { resolveNodeGeometry } from "./frame-geometry";
import { screenshotPageRect } from "./geometry/coordinate-types";
import { import {
type ChromeTabsApi, type ChromeTabsApi,
enforceToolTargetScope, enforceToolTargetScope,
@@ -47,15 +49,9 @@ import {
type ToolEffect, type ToolEffect,
} from "./shared"; } from "./shared";
import { resolveSnapshotRef } from "./snapshot-ref"; import { resolveSnapshotRef } from "./snapshot-ref";
import { import { type CapturedNode, type CapturedSurfaceProbe, probeHoverSurfaces } from "./vom/capture";
type CapturedNode, import { captureObservationFacts, semanticCapture } from "./vom/capture-coordinator";
type CapturedSurfaceProbe, import type { FrameDocument as CapturedFrameDocument } from "./vom/frame-document";
type CapturedViewModel,
captureViewModel,
collectOverlayExcludedBackendIds,
probeHoverSurfaces,
} from "./vom/capture";
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
import { withOverlayBypass } from "./vom/hover-perception"; import { withOverlayBypass } from "./vom/hover-perception";
import { probeTooltipNames } from "./vom/name-enrichment"; import { probeTooltipNames } from "./vom/name-enrichment";
import { import {
@@ -197,15 +193,14 @@ async function captureElementScreenshot(
if (isRpcError(geometry)) return geometry; if (isRpcError(geometry)) return geometry;
if (signal?.aborted) return cancelled("screenshot"); if (signal?.aborted) return cancelled("screenshot");
const rect = geometry.topBounds; const rect = geometry.topBounds;
const clip = screenshotPageRect(rect, geometry.topViewport);
if (!clip) return { code: "cdp_failed", message: "invalid screenshot coordinate space" };
try { try {
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", { const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
format: "png", format: "png",
clip: { clip: {
x: rect.x, ...clip.rect,
y: rect.y,
width: rect.width,
height: rect.height,
scale: 1, scale: 1,
}, },
}); });
@@ -530,7 +525,7 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A
function buildConditionalSurfaces( function buildConditionalSurfaces(
nodes: VomNode[], nodes: VomNode[],
captured: CapturedViewModel, captured: CapturedSceneInput,
probes: CapturedSurfaceProbe[], probes: CapturedSurfaceProbe[],
): CondSurface[] { ): CondSurface[] {
if (probes.length === 0) return []; if (probes.length === 0) return [];
@@ -660,161 +655,9 @@ export interface BuildVomSceneOptions {
export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>; export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>;
function legacyFrameDocuments(
axNodes: CdpAxNode[],
captured: CapturedViewModel,
pageUrl?: string,
rootTarget: VomFrameDocument["target"] = { tabId: 0 },
): VomFrameDocument[] {
const rootFrameId = captured.rootFrameId ?? "root";
const documents: VomFrameDocument[] = [
{
frameId: rootFrameId,
contextScopeId: rootFrameId,
target: rootTarget,
...(pageUrl ? { url: pageUrl } : {}),
axNodes: [],
domNodes: captured.nodes.map((node) => ({ ...node, frameId: node.frameId ?? rootFrameId })),
},
];
const pending = [...captured.iframeNodes.entries()];
let progress = true;
let nextSyntheticFrame = 1;
while (pending.length > 0 && progress) {
progress = false;
for (let index = pending.length - 1; index >= 0; index -= 1) {
const [ownerBackendNodeId, domNodes] = pending[index];
const parent = documents.find((document) =>
document.domNodes.some((node) => node.backendNodeId === ownerBackendNodeId),
);
if (!parent) continue;
const frameId =
domNodes.find((node) => node.frameId)?.frameId ?? `legacy-frame-${nextSyntheticFrame++}`;
documents.push({
frameId,
parentFrameId: parent.frameId,
ownerBackendNodeId,
contextScopeId: frameId,
target: parent.target,
axNodes: [],
domNodes: domNodes.map((node) => ({ ...node, frameId: node.frameId ?? frameId })),
});
pending.splice(index, 1);
progress = true;
}
}
const documentByFrameId = new Map(documents.map((document) => [document.frameId, document]));
const backendFrame = new Map<number, string>();
const childFrameByOwner = new Map<number, string>();
for (const document of documents) {
if (document.ownerBackendNodeId !== undefined) {
childFrameByOwner.set(document.ownerBackendNodeId, document.frameId);
}
for (const node of document.domNodes) backendFrame.set(node.backendNodeId, document.frameId);
}
const axById = new Map(axNodes.map((node) => [node.nodeId, node]));
const ownership = new Map<string, string>();
const resolving = new Set<string>();
const frameForAx = (node: CdpAxNode): string => {
const cached = ownership.get(node.nodeId);
if (cached) return cached;
let frameId: string | undefined;
if (node.frameId && documentByFrameId.has(node.frameId)) frameId = node.frameId;
if (!frameId && typeof node.backendDOMNodeId === "number") {
frameId = backendFrame.get(node.backendDOMNodeId);
}
if (!frameId && node.parentId && !resolving.has(node.nodeId)) {
const parent = axById.get(node.parentId);
if (parent) {
frameId =
typeof parent.backendDOMNodeId === "number"
? childFrameByOwner.get(parent.backendDOMNodeId)
: undefined;
if (!frameId) {
resolving.add(node.nodeId);
frameId = frameForAx(parent);
resolving.delete(node.nodeId);
}
}
}
frameId ??= rootFrameId;
ownership.set(node.nodeId, frameId);
return frameId;
};
for (const node of axNodes) frameForAx(node);
for (const node of axNodes) {
const frameId = ownership.get(node.nodeId) ?? rootFrameId;
const document = documentByFrameId.get(frameId) ?? documents[0];
document.axNodes.push({
...node,
frameId,
...(node.parentId && ownership.get(node.parentId) === frameId
? { parentId: node.parentId }
: { parentId: undefined }),
...(node.childIds
? { childIds: node.childIds.filter((childId) => ownership.get(childId) === frameId) }
: {}),
});
}
return documents;
}
function withLegacyBackendIds(scene: VomScene): VomScene {
const used = new Set<number>();
const idMap = new Map<number, number>();
let nextVirtualId = -1;
for (const node of scene.nodes) {
const preferred = node.backendNodeId;
const id = preferred !== undefined && !used.has(preferred) ? preferred : nextVirtualId--;
used.add(id);
idMap.set(node.id, id);
}
return {
...scene,
nodes: scene.nodes.map((node) => ({
...node,
id: idMap.get(node.id) as number,
parentId: node.parentId === null ? null : (idMap.get(node.parentId) ?? null),
...(node.domParentId !== undefined
? { domParentId: node.domParentId === null ? null : (idMap.get(node.domParentId) ?? null) }
: {}),
...(node.domAncestorIds
? { domAncestorIds: node.domAncestorIds.flatMap((id) => idMap.get(id) ?? []) }
: {}),
})),
...(scene.surfaces
? {
surfaces: scene.surfaces.map((surface) => ({
...surface,
triggerId: idMap.get(surface.triggerId) ?? surface.triggerId,
})),
}
: {}),
...(scene.activeScopeBlocks
? {
activeScopeBlocks: scene.activeScopeBlocks.map((block) => ({
...block,
triggerId: idMap.get(block.triggerId) ?? block.triggerId,
})),
}
: {}),
};
}
export function buildVomScene(
axNodes: CdpAxNode[],
captured: CapturedViewModel,
options: BuildVomSceneOptions = {},
): VomScene {
return withLegacyBackendIds(
buildFrameVomScene(legacyFrameDocuments(axNodes, captured, options.pageUrl), captured, options),
);
}
export function buildFrameVomScene( export function buildFrameVomScene(
documents: VomFrameDocument[], documents: VomFrameDocument[],
captured: CapturedViewModel, captured: CapturedSceneInput,
options: BuildVomSceneOptions = {}, options: BuildVomSceneOptions = {},
): VomScene { ): VomScene {
const scene = buildSemanticVomScene({ const scene = buildSemanticVomScene({
@@ -830,7 +673,7 @@ export function buildFrameVomScene(
function attachCapturedSceneAnnotations( function attachCapturedSceneAnnotations(
scene: VomScene, scene: VomScene,
documents: VomFrameDocument[], documents: VomFrameDocument[],
captured: CapturedViewModel, captured: CapturedSceneInput,
surfaceProbes: CapturedSurfaceProbe[], surfaceProbes: CapturedSurfaceProbe[],
): VomScene { ): VomScene {
const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId); const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId);
@@ -980,51 +823,6 @@ export async function handleGetHtml(
} }
} }
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
}
interface LayoutMetricsViewportReply {
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
layoutViewport?: { clientWidth?: number; clientHeight?: number };
}
async function fallbackCapturedViewModel(
cdp: CdpRunner,
tabId: number,
signal?: AbortSignal,
): Promise<CapturedViewModel> {
throwIfAborted(signal, "observation");
let viewport = { width: 0, height: 0 };
try {
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
throwIfAborted(signal, "observation");
const source = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
viewport = {
width: source.clientWidth ?? 0,
height: source.clientHeight ?? 0,
};
} catch (error) {
if (isAbortError(error)) throw error;
}
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal);
throwIfAborted(signal, "observation");
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
}
async function captureForVom(
cdp: CdpRunner,
tabId: number,
options: CaptureVomObservationOptions,
): Promise<CapturedViewModel> {
try {
return await captureViewModel(cdp, tabId, { signal: options.signal });
} catch (error) {
if (isAbortError(error)) throw error;
return fallbackCapturedViewModel(cdp, tabId, options.signal);
}
}
export interface HoverProbeOutcome { export interface HoverProbeOutcome {
/** Whether any active hover was dispatched during this observation. */ /** Whether any active hover was dispatched during this observation. */
performed: boolean; performed: boolean;
@@ -1057,7 +855,7 @@ const NO_HOVER_PROBES: HoverProbeOutcome = {
async function runHoverProbes( async function runHoverProbes(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
captured: CapturedViewModel, captured: CapturedSceneInput,
documents: VomFrameDocument[], documents: VomFrameDocument[],
staticSemantics: ReturnType<typeof resolveSemanticGraph>, staticSemantics: ReturnType<typeof resolveSemanticGraph>,
options: CaptureVomObservationOptions, options: CaptureVomObservationOptions,
@@ -1096,14 +894,9 @@ export async function captureVomObservation(
throwIfAborted(options.signal, "observation"); throwIfAborted(options.signal, "observation");
await cdp.ensureAttachedToUrl?.(tabId, url); await cdp.ensureAttachedToUrl?.(tabId, url);
throwIfAborted(options.signal, "observation"); throwIfAborted(options.signal, "observation");
const captured = await captureForVom(cdp, tabId, options); const facts = await captureObservationFacts<CdpAxNode>(cdp, tabId, options.signal, url);
const { captured, documents: normalizedDocuments } = semanticCapture(facts);
throwIfAborted(options.signal, "observation"); throwIfAborted(options.signal, "observation");
const documents = await captureFrameData<CdpAxNode>(cdp, tabId, captured, options.signal);
throwIfAborted(options.signal, "observation");
const normalizedDocuments =
documents.length === 1 && captured.iframeNodes.size > 0
? legacyFrameDocuments(documents[0].axNodes, captured, url, documents[0].target)
: documents;
const semanticGraph = buildSemanticGraph({ const semanticGraph = buildSemanticGraph({
documents: normalizedDocuments, documents: normalizedDocuments,
viewport: captured.viewport, viewport: captured.viewport,
@@ -1131,12 +924,21 @@ export async function captureVomObservation(
captured, captured,
hoverProbes.surfaceProbes, hoverProbes.surfaceProbes,
); );
// Reserve space using the renderer's character-based token estimate. Like VOM
// headers, this integrity notice remains visible even under a tiny token budget.
const geometryNotice = facts.issues.some((issue) => issue.stage === "geometry")
? "@warning iframe geometry incomplete: some frame content has no top-level coordinates.\n"
: "";
const rendered = renderVom(decoratedScene, { const rendered = renderVom(decoratedScene, {
maxDepth: options.maxDepth, maxDepth: options.maxDepth,
maxTokens: options.maxTokens, maxTokens:
!geometryNotice || options.maxTokens === undefined
? options.maxTokens
: Math.max(0, options.maxTokens - Math.ceil(geometryNotice.length / 4)),
redactValues: options.redactValues, redactValues: options.redactValues,
activeRegionPolicy: options.activeRegionPolicy, activeRegionPolicy: options.activeRegionPolicy,
}); });
if (geometryNotice) rendered.text += `\n${geometryNotice.trimEnd()}`;
throwIfAborted(options.signal, "observation"); throwIfAborted(options.signal, "observation");
return projectRecordSafeObservation({ return projectRecordSafeObservation({
rootFrameId: captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root", rootFrameId: captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root",
@@ -0,0 +1,914 @@
import { describe, expect, it, vi } from "vitest";
import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph";
import { cdpTargetKey } from "@/browser-driver/frame-graph";
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import type { CdpRunner } from "../../shared";
import { captureObservationFacts, semanticCapture } from "../capture-coordinator";
import { buildSemanticGraph } from "../semantic-graph/build";
import { REQUESTED_STYLES } from "../snapshot";
function fixture(
options: { frames?: CdpFrame[]; fail?: string; omitDocument?: string; overlay?: string } = {},
) {
const frames = options.frames ?? [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "same",
parentFrameId: "main",
ownerBackendNodeId: 3,
target: { tabId: 4 },
},
{
frameId: "nested",
parentFrameId: "same",
ownerBackendNodeId: 13,
target: { tabId: 4 },
},
{
frameId: "remote",
parentFrameId: "main",
ownerBackendNodeId: 4,
target: { tabId: 4, sessionId: "remote" },
},
];
const elements = new Map(
frames.map((frame, i) => [frame.frameId, frame.target.sessionId ? 1 : i * 10 + 1]),
);
const logs: Array<{ target: CdpTarget; method: string; params: Record<string, unknown> }> = [];
const send = vi.fn(
async <T>(target: CdpTarget, method: string, params: object = {}): Promise<T> => {
logs.push({ target, method, params: params as Record<string, unknown> });
const args = params as Record<string, string | number>;
if (options.fail === `${target.sessionId ?? "main"}:${method}`)
throw new Error("fixture failure");
let result: unknown = {};
if (method === "Page.getLayoutMetrics")
result = { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
if (method === "DOMSnapshot.captureSnapshot") {
expect((params as { computedStyles: unknown }).computedStyles).toEqual(REQUESTED_STYLES);
result = {
strings: [
"#document",
"html",
"button",
OVERLAY_HOST_MARKER_ATTR,
"",
"visible",
"1",
"static",
"auto",
],
documents: frames
.filter(
(frame) =>
cdpTargetKey(frame.target) === cdpTargetKey(target) &&
frame.frameId !== options.omitDocument,
)
.map((frame) => {
const element = elements.get(frame.frameId)!;
return {
frameId: frame.frameId,
nodes: {
backendNodeId: [element - 1, element + 10000, element, element + 1],
nodeName: [0, 1, 1, 2],
nodeType: [9, 10, 1, 1],
parentIndex: [-1, 0, 0, 2],
attributes: [[], [], [], frame.frameId === options.overlay ? [3, 4] : []],
},
layout: {
nodeIndex: [2, 3],
bounds: [
[0, 0, 1000, 800],
[10, 20, 100, 40],
],
styles: [
[7, 8, 8, 5, 6],
[7, 8, 8, 5, 6],
],
},
};
}),
};
}
if (method === "Accessibility.getFullAXTree") {
const frameId = String(args.frameId);
result = {
nodes: [
{
nodeId: `${frameId}-button`,
frameId,
backendDOMNodeId: elements.get(frameId)! + 1,
role: { value: "button" },
name: { value: frameId },
},
],
};
}
return result as T;
},
);
const cdp: CdpRunner = {
getFrameGraph: async () => ({ rootFrameId: frames[0].frameId, frames }),
send: (tabId, method, params) =>
(send as NonNullable<CdpRunner["sendToTarget"]>)({ tabId }, method, params),
sendToTarget: send as NonNullable<CdpRunner["sendToTarget"]>,
};
return { cdp, logs, elements };
}
describe("captureObservationFacts", () => {
it("collects each target once and scopes equal backend IDs", async () => {
const { cdp, logs } = fixture();
const facts = await captureObservationFacts(cdp, 4);
expect(logs.filter((call) => call.method === "DOMSnapshot.captureSnapshot")).toHaveLength(2);
expect(logs.filter((call) => call.method === "Accessibility.enable")).toHaveLength(2);
expect(logs.filter((call) => call.method === "Accessibility.getFullAXTree")).toHaveLength(4);
const main = facts.documents.find((doc) => doc.frame.frameId === "main")!;
const remote = facts.documents.find((doc) => doc.frame.frameId === "remote")!;
expect(main.index.nodes.get(2)).not.toBe(remote.index.nodes.get(2));
expect(main.index.nodes.get(2)?.parentBackendNodeId).toBe(1);
expect(main.index.nodes.get(10001)?.nodeType).toBe(10);
expect(remote.axNodes[0].frameId).toBe("remote");
expect(main.index.nodes.get(2)).toBe(main.domNodes.find((node) => node.backendNodeId === 2));
expect(facts.finishedAt).toBeGreaterThanOrEqual(facts.startedAt);
});
it("does not retry failed or missing documents and retains valid AX-only semantics", async () => {
const { cdp, logs } = fixture({
fail: "remote:DOMSnapshot.captureSnapshot",
omitDocument: "same",
});
const facts = await captureObservationFacts(cdp, 4);
const remote = facts.documents.find((doc) => doc.frame.frameId === "remote")!;
expect(remote.domNodes).toHaveLength(0);
expect(remote.axNodes).toHaveLength(1);
expect(logs.filter((call) => call.method === "DOMSnapshot.captureSnapshot")).toHaveLength(2);
expect(facts.issues).toContainEqual(
expect.objectContaining({ frameId: "same", stage: "dom", reason: "capture-unavailable" }),
);
expect(facts.issues).toContainEqual(
expect.objectContaining({ target: { tabId: 4, sessionId: "remote" }, stage: "dom" }),
);
});
it("excludes overlay AX in its own document without excluding equal IDs elsewhere", async () => {
const facts = await captureObservationFacts(fixture({ overlay: "remote" }).cdp, 4);
const { documents } = semanticCapture(facts);
const graph = buildSemanticGraph({
documents,
viewport: facts.viewport,
rootFrameId: facts.rootFrameId,
});
expect(
[...graph.nodes.values()].find(
(node) => node.frameId === "remote" && node.backendNodeId === 2,
)?.excluded,
).toBe(true);
expect(
[...graph.nodes.values()].find((node) => node.frameId === "main" && node.backendNodeId === 2)
?.excluded,
).toBe(false);
});
it.each([
"cancel",
"worker-abort",
] as const)("joins target cleanup and stops claiming targets after %s", async (mode) => {
const frames = Array.from({ length: 5 }, (_, i) => ({
frameId: `f${i}`,
target: { tabId: 4, ...(i ? { sessionId: `s${i}` } : {}) },
}));
const { cdp } = fixture({ frames });
const original = cdp.sendToTarget!;
const pending = new Map<string, { resolve: () => void; reject: (error: Error) => void }>();
const calls: Array<{ target: string; method: string }> = [];
let release: (() => void) | undefined;
let released = false;
const send: NonNullable<CdpRunner["sendToTarget"]> = async <T>(
target: CdpTarget,
method: string,
params?: object,
) => {
calls.push({ target: target.sessionId ?? "main", method });
if (method === "DOMSnapshot.captureSnapshot" && target.sessionId) {
await new Promise<void>((resolve, reject) =>
pending.set(target.sessionId!, { resolve, reject }),
);
}
if (method === "Runtime.releaseObjectGroup") {
await new Promise<void>((resolve) => {
release = resolve;
});
released = true;
}
const result = await original<T>(target, method, params);
if (method === "DOMSnapshot.captureSnapshot" && !target.sessionId)
(result as { strings: string[] }).strings[2] = "input";
return result;
};
cdp.sendToTarget = send;
cdp.send = (tabId, method, params) => send({ tabId }, method, params);
const controller = new AbortController();
let settled = false;
const capture = captureObservationFacts(cdp, 4, controller.signal);
const rejected = expect(capture).rejects.toMatchObject({ name: "AbortError" });
void capture.then(
() => {
settled = true;
},
() => {
settled = true;
},
);
await vi.waitFor(() => {
expect(pending.size).toBe(3);
expect(release).toBeDefined();
});
if (mode === "cancel") controller.abort();
// Cancellation may race with an ordinary transport rejection. Without an
// external signal, an escaping AbortError still terminates the worker pool.
pending
.get("s1")!
.reject(
mode === "cancel"
? new Error("transport closed")
: new DOMException("target aborted", "AbortError"),
);
pending.get("s2")!.resolve();
pending.get("s3")!.resolve();
await vi.waitFor(() => {
if (mode === "worker-abort")
expect(
calls.some(
(call) => call.target === "s3" && call.method === "Accessibility.getFullAXTree",
),
).toBe(true);
else
expect(
calls.some(
(call) => call.target === "s3" && call.method === "DOMSnapshot.captureSnapshot",
),
).toBe(true);
});
// Allow worker rejection and the outer promise chain to run while cleanup
// is deliberately held. The old fail-fast pool settles here.
await new Promise((resolve) => setTimeout(resolve, 0));
expect(settled).toBe(false);
expect(released).toBe(false);
expect(calls.some((call) => call.target === "s4")).toBe(false);
if (mode === "cancel")
expect(
calls.some(
(call) => call.method === "Accessibility.enable" || call.method === "DOM.getDocument",
),
).toBe(false);
release!();
await rejected;
expect(released).toBe(true);
expect(calls.some((call) => call.target === "s4")).toBe(false);
});
it("bounds collection across many targets and stops scheduling after cancellation", async () => {
const frames = Array.from({ length: 12 }, (_, i) => ({
frameId: `f${i}`,
target: { tabId: 4, ...(i ? { sessionId: `s${i}` } : {}) },
}));
const { cdp } = fixture({ frames });
let active = 0,
peak = 0;
const original = cdp.sendToTarget!;
const send: NonNullable<CdpRunner["sendToTarget"]> = async (target, method, params) => {
active++;
peak = Math.max(peak, active);
try {
await new Promise((resolve) => setTimeout(resolve, 1));
return await original(target, method, params);
} finally {
active--;
}
};
cdp.sendToTarget = send;
cdp.send = (tabId, method, params) => send({ tabId }, method, params);
await captureObservationFacts(cdp, 4);
// This fixture has no frame edges, so no concurrent owner requests.
expect(peak).toBeLessThanOrEqual(4);
expect(peak).toBeGreaterThan(1);
const controller = new AbortController();
controller.abort();
await expect(captureObservationFacts(cdp, 4, controller.signal)).rejects.toMatchObject({
name: "AbortError",
});
expect(active).toBe(0);
});
});
function childSnapshot(frameId: string, backendNodeId: number) {
const strings = [frameId, "body", "button", "static", "auto", "pointer"];
return {
strings,
documents: [
{
frameId,
nodes: {
parentIndex: [-1, 0],
nodeName: [1, 2],
backendNodeId: [backendNodeId - 1, backendNodeId],
attributes: [[], []],
},
layout: {
nodeIndex: [0, 1],
styles: [
[3, 4, 4],
[3, 4, 5],
],
bounds: [
[0, 0, 300, 200],
[10, 20, 100, 40],
],
paintOrders: [0, 1],
},
},
],
};
}
describe("OOPIF capture", () => {
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
const sendToTarget = vi.fn(async (target, method) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } };
}
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
return target.sessionId === "left-session"
? childSnapshot("left", 101)
: childSnapshot("right", 201);
}
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
throw new Error(`unexpected ${method}`);
});
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method, params) => {
if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("main", 1);
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
if (method === "DOM.getBoxModel") {
const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId;
const x = backendNodeId === 10 ? 50 : 500;
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "left",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "left-session" },
},
{
frameId: "right",
parentFrameId: "main",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "right-session" },
},
],
})),
};
const { documents: trees } = semanticCapture(await captureObservationFacts(cdp, 4));
expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]);
expect(
trees
.find((doc) => doc.frameId === "left")
?.domNodes?.find((node) => node.backendNodeId === 101)?.rect,
).toEqual({ x: 60, y: 120, w: 100, h: 40 });
expect(
trees
.find((doc) => doc.frameId === "right")
?.domNodes?.find((node) => node.backendNodeId === 201)?.rect,
).toEqual({ x: 510, y: 120, w: 100, h: 40 });
expect(
new Map(
trees
.filter((doc) => doc.ownerBackendNodeId !== undefined)
.map((doc) => [doc.frameId, doc.ownerBackendNodeId]),
),
).toEqual(
new Map([
["left", 10],
["right", 20],
]),
);
expect(
new Map(
trees.filter((doc) => doc.parentFrameId).map((doc) => [doc.frameId, doc.parentFrameId]),
),
).toEqual(
new Map([
["left", "main"],
["right", "main"],
]),
);
});
it("retains nested owner failures when merging a captured OOPIF", async () => {
const child = childSnapshot("child", 101);
const nested = childSnapshot("nested", 201);
const document = child.documents[0];
const snapshot = {
strings: child.strings,
documents: [
{
...document,
nodes: { ...document.nodes, contentDocumentIndex: { index: [1], value: [1] } },
},
nested.documents[0],
],
};
const reply = async (_target: unknown, method: string) => {
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200 } };
if (method === "DOMSnapshot.captureSnapshot")
return typeof _target === "number" ? childSnapshot("main", 11) : snapshot;
if (method === "DOM.getBoxModel")
return { model: { content: [50, 100, 350, 100, 350, 300, 50, 300] } };
if (method === "DOM.resolveNode") throw new Error("nested owner replaced");
return {};
};
const cdp: CdpRunner = {
send: vi.fn(reply) as CdpRunner["send"],
sendToTarget: vi.fn(reply) as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "nested",
parentFrameId: "child",
ownerBackendNodeId: 101,
target: { tabId: 4, sessionId: "child-session" },
},
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
})),
};
const captured = await captureObservationFacts(cdp, 4);
expect(
captured.issues
.filter((issue) => issue.projectionIssue)
.map((issue) => issue.projectionIssue),
).toEqual([
{
status: "unavailable",
source: { target: { tabId: 4, sessionId: "child-session" }, frameId: "nested" },
ownerBackendNodeId: 101,
},
]);
expect(
captured.documents.find((doc) => doc.frame.frameId === "nested")?.domNodes[1],
).toMatchObject({
rect: null,
localRect: { x: 10, y: 20, w: 100, h: 40 },
});
});
it("keeps OOPIF semantics when viewport projection is unavailable", async () => {
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method) => {
if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("main", 1);
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable");
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Page.getLayoutMetrics") throw new Error("viewport unavailable");
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("child", 101);
if (method === "Accessibility.getFullAXTree") {
return {
nodes: [
{
nodeId: "button",
backendDOMNodeId: 101,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Continue" },
},
],
};
}
throw new Error(`unexpected ${method}`);
}) as unknown as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
})),
};
const { documents } = semanticCapture(await captureObservationFacts(cdp, 4));
const childDocument = documents.find((document) => document.frameId === "child");
expect(childDocument?.axNodes).toEqual([
expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }),
]);
expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual(
expect.objectContaining({ rect: null, localRect: { x: 10, y: 20, w: 100, h: 40 } }),
);
});
});
// Six sibling frames and one nested frame exercise scheduling through the real capture entry.
function siblingCaptureFixture(
beforeReply: (method: string, params: Record<string, unknown>) => Promise<void> = async () => {},
) {
const document = (id: number, owners: number[], childIndexes: number[]) => ({
frameId: `frame-${id}`,
nodes: {
parentIndex: [-1, ...owners.map(() => 0)],
nodeName: [0, ...owners.map(() => 1)],
backendNodeId: [1000 + id, ...owners],
attributes: [[], ...owners.map(() => [])],
contentDocumentIndex: { index: owners.map((_, i) => i + 1), value: childIndexes },
},
layout: {
nodeIndex: [0, ...owners.map((_, i) => i + 1)],
bounds: [[0, 0, 200, 100], ...owners.map(() => [0, 0, 200, 100])],
},
});
const snapshot = {
strings: ["body", "iframe"],
documents: [
document(0, [100, 101, 102, 103, 104, 105], [1, 2, 3, 4, 5, 6]),
document(1, [200], [7]),
...Array.from({ length: 6 }, (_, i) => document(i + 2, [], [])),
],
};
let active = 0;
let peak = 0;
const send = vi.fn(async (_tabId: number, method: string, params: object = {}) => {
const args = params as Record<string, unknown>;
active++;
peak = Math.max(peak, active);
try {
await beforeReply(method, args);
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
if (method === "DOM.getBoxModel")
return { model: { content: [0, 0, 200, 0, 200, 100, 0, 100] } };
if (method === "DOM.resolveNode") return { object: { objectId: String(args.backendNodeId) } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 200, height: 100 } } };
return {};
} finally {
active--;
}
});
return {
snapshot,
cdp: { send: send as CdpRunner["send"] },
send,
peak: () => peak,
active: () => active,
measured: () =>
send.mock.calls
.filter(([, method]) => method === "DOM.getBoxModel")
.map(([, , params]) => (params as { backendNodeId: number }).backendNodeId),
};
}
describe("snapshot document provenance", () => {
it.each([
"new-child",
"new-root",
"stale-owner",
])("retains current snapshot evidence with %s graph data", async (mode) => {
const f = siblingCaptureFixture();
const cdp: CdpRunner = {
...f.cdp,
getFrameGraph: async () => ({
rootFrameId: mode === "new-root" ? "old-root" : "frame-0",
frames: [
{ frameId: mode === "new-root" ? "old-root" : "frame-0", target: { tabId: 4 } },
...(mode === "stale-owner"
? [
{
frameId: "frame-1",
parentFrameId: "frame-0",
ownerBackendNodeId: 999,
target: { tabId: 4 },
},
]
: []),
],
}),
};
const facts = await captureObservationFacts(cdp, 4);
expect(facts.rootFrameId).toBe("frame-0");
expect(facts.documents.find((doc) => doc.frame.frameId === "frame-1")).toMatchObject({
frame: { ownerBackendNodeId: 100 },
domNodes: [
expect.objectContaining({ rect: { x: 0, y: 0, w: 200, h: 100 } }),
expect.anything(),
],
});
expect(
facts.documents.find((doc) => doc.frame.frameId === "frame-7")?.domNodes[0].rect,
).not.toBeNull();
expect(f.measured()).not.toContain(999);
expect(
f.send.mock.calls.filter(([, method]) => method === "DOMSnapshot.captureSnapshot"),
).toHaveLength(1);
expect(
f.send.mock.calls.some(
([, method, params]) =>
method === "Accessibility.getFullAXTree" &&
(params as { frameId?: string }).frameId === "frame-7",
),
).toBe(true);
});
it("uses snapshot URLs before graph URLs and limits the caller URL to the root", async () => {
const f = siblingCaptureFixture();
f.snapshot.strings.push("https://current.test/page", "https://child.test/page");
Object.assign(f.snapshot.documents[0], { documentURL: 2 });
Object.assign(f.snapshot.documents[1], { documentURL: 3 });
const facts = await captureObservationFacts(
{
...f.cdp,
getFrameGraph: async () => ({
rootFrameId: "frame-0",
frames: [{ frameId: "frame-0", target: { tabId: 4 }, url: "https://old.test/" }],
}),
},
4,
undefined,
"https://caller.test/",
);
expect(facts.documents.find((doc) => doc.frame.frameId === "frame-0")?.frame.url).toBe(
"https://current.test/page",
);
expect(facts.documents.find((doc) => doc.frame.frameId === "frame-1")?.frame.url).toBe(
"https://child.test/page",
);
expect(
facts.documents.find((doc) => doc.frame.frameId === "frame-2")?.frame.url,
).toBeUndefined();
});
it("accepts a migrated snapshot claim without merging AX from the old target", async () => {
const f = siblingCaptureFixture();
const cdp: CdpRunner = {
...f.cdp,
getFrameGraph: async () => ({
rootFrameId: "frame-0",
frames: [
{ frameId: "frame-0", target: { tabId: 4 } },
{
frameId: "frame-1",
parentFrameId: "frame-0",
ownerBackendNodeId: 100,
target: { tabId: 4, sessionId: "old" },
},
],
}),
sendToTarget: async <T>(_target: CdpTarget, method: string): Promise<T> =>
(method === "Accessibility.getFullAXTree"
? { nodes: [{ nodeId: "stale", role: { value: "button" }, name: { value: "stale" } }] }
: {}) as T,
};
const facts = await captureObservationFacts(cdp, 4);
const current = facts.documents.find((doc) => doc.frame.frameId === "frame-1");
expect(current?.frame.target).toEqual({ tabId: 4 });
expect(current?.domNodes.length).toBeGreaterThan(0);
expect(current?.axNodes).toEqual([]);
});
it.each([
0, 1,
])("rejects duplicate snapshot frame identity at document %i without overwriting", async (index) => {
const f = siblingCaptureFixture();
f.snapshot.documents.push(f.snapshot.documents[index]);
if (index === 0) {
await expect(captureObservationFacts(f.cdp, 4)).rejects.toThrow(
"root document ownership is ambiguous",
);
} else {
const facts = await captureObservationFacts(f.cdp, 4);
expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-1");
expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-7");
expect(
facts.documents.find((doc) => doc.frame.frameId === "frame-2")?.domNodes.length,
).toBeGreaterThan(0);
}
});
it("retains the caller URL in the root AX-only fallback", async () => {
const cdp: CdpRunner = {
getFrameGraph: async () => {
throw new Error("no graph");
},
send: async <T>(_tabId: number, method: string): Promise<T> => {
if (method === "DOMSnapshot.captureSnapshot") throw new Error("no snapshot");
return (
method === "Accessibility.getFullAXTree"
? { nodes: [{ nodeId: "root", role: { value: "RootWebArea" } }] }
: {}
) as T;
},
};
const facts = await captureObservationFacts(cdp, 4, undefined, "https://page.test/");
expect(facts.documents[0].frame.url).toBe("https://page.test/");
expect(facts.documents[0].axNodes).toHaveLength(1);
});
it("retains disconnected document data without inventing top-level geometry", async () => {
const f = siblingCaptureFixture();
f.snapshot.documents[0].nodes.contentDocumentIndex = { index: [], value: [] };
const facts = await captureObservationFacts(f.cdp, 4);
expect(
facts.documents.find((doc) => doc.frame.frameId === "frame-1")?.domNodes[0],
).toMatchObject({ rect: null, localRect: { x: 0, y: 0, w: 200, h: 100 } });
expect(f.measured()).toEqual([]);
expect(facts.issues).toContainEqual(
expect.objectContaining({ frameId: "frame-1", stage: "ownership" }),
);
});
it.each([
false,
true,
])("isolates competing target claims independent of batch order %s", async (reverse) => {
const f = siblingCaptureFixture();
const remote = {
frameId: "frame-1",
parentFrameId: "frame-0",
ownerBackendNodeId: 100,
target: { tabId: 4, sessionId: "remote" },
};
const main = { frameId: "frame-0", target: { tabId: 4 } };
const cdp: CdpRunner = {
...f.cdp,
getFrameGraph: async () => ({
rootFrameId: "frame-0",
frames: reverse ? [remote, main] : [main, remote],
}),
sendToTarget: async <T>(_target: CdpTarget, method: string): Promise<T> =>
(method === "DOMSnapshot.captureSnapshot"
? { ...f.snapshot, documents: [f.snapshot.documents[1]] }
: method === "Page.getLayoutMetrics"
? { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }
: {}) as T,
};
const facts = await captureObservationFacts(cdp, 4);
expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-1");
expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-7");
expect(
facts.documents.find((doc) => doc.frame.frameId === "frame-2")?.domNodes[0].rect,
).not.toBeNull();
expect(facts.issues).toContainEqual(
expect.objectContaining({ frameId: "frame-1", stage: "ownership" }),
);
});
});
describe("sibling frame measurement scheduling", () => {
it("bounds concurrent reads, fills free slots and preserves breadth-first output despite reordered replies", async () => {
const pending = new Map<number, () => void>();
const fixture = siblingCaptureFixture(async (method, params) => {
if (method === "DOM.getBoxModel" && Number(params.backendNodeId) < 200)
await new Promise<void>((resolve) => pending.set(Number(params.backendNodeId), resolve));
});
const capture = captureObservationFacts(fixture.cdp, 4);
await vi.waitFor(() => expect(pending.size).toBe(4));
expect(fixture.measured()).toEqual([100, 101, 102, 103]);
pending.get(103)!();
await vi.waitFor(() => expect(pending.has(104)).toBe(true));
pending.get(104)!();
await vi.waitFor(() => expect(pending.has(105)).toBe(true));
expect(fixture.measured()).not.toContain(200);
for (const id of [105, 102, 101, 100]) pending.get(id)!();
const captured = await capture;
expect(
captured.issues
.filter((issue) => issue.stage === "geometry")
.map((issue) => issue.projectionIssue),
).toEqual([]);
expect(fixture.peak()).toBe(4);
expect(fixture.active()).toBe(0);
expect(fixture.measured()).toEqual([100, 101, 102, 103, 104, 105, 200]);
expect(captured.documents.map((doc) => doc.frame.frameId)).toEqual(
Array.from({ length: 8 }, (_, i) => `frame-${i}`),
);
expect(
captured.documents.find((doc) => doc.frame.frameId === "frame-7")?.domNodes[0].rect,
).toEqual({ x: 0, y: 0, w: 200, h: 100 });
for (const method of [
"DOM.getBoxModel",
"DOM.resolveNode",
"Runtime.callFunctionOn",
"Runtime.releaseObject",
])
expect(fixture.send.mock.calls.filter(([, name]) => name === method)).toHaveLength(7);
});
it("does not measure descendants of a failed owner and retains sibling geometry and local nodes", async () => {
const fixture = siblingCaptureFixture(async (method, params) => {
if (method === "DOM.resolveNode" && params.backendNodeId === 100)
throw new Error("owner replaced");
});
const captured = await captureObservationFacts(fixture.cdp, 4);
expect(
captured.issues
.filter((issue) => issue.stage === "geometry")
.map((issue) => issue.projectionIssue),
).toEqual([
{
status: "unavailable",
source: { target: { tabId: 4 }, frameId: "frame-1" },
ownerBackendNodeId: 100,
},
{
status: "blocked",
source: { target: { tabId: 4 }, frameId: "frame-7" },
cause: captured.issues.find((issue) => issue.frameId === "frame-1")?.projectionIssue,
},
]);
expect(fixture.measured()).not.toContain(200);
expect(
captured.documents.find((doc) => doc.frame.frameId === "frame-7")?.domNodes[0],
).toMatchObject({
tag: "body",
rect: null,
localRect: { x: 0, y: 0, w: 200, h: 100 },
});
expect(
captured.documents.find((doc) => doc.frame.frameId === "frame-2")?.domNodes[0].rect,
).toEqual({ x: 0, y: 0, w: 200, h: 100 });
expect(captured.documents.map((doc) => doc.frame.frameId)).toEqual(
Array.from({ length: 8 }, (_, i) => `frame-${i}`),
);
});
it("stops scheduling on cancellation and waits for resolved objects to be released", async () => {
const controller = new AbortController();
const resolutions = new Map<string, () => void>();
const releases = new Map<string, () => void>();
const fixture = siblingCaptureFixture(async (method, params) => {
if (method === "DOM.resolveNode")
await new Promise<void>((resolve) =>
resolutions.set(String(params.backendNodeId), resolve),
);
if (method === "Runtime.releaseObject")
await new Promise<void>((resolve) => releases.set(String(params.objectId), resolve));
});
let settled = false;
const capture = captureObservationFacts(fixture.cdp, 4, controller.signal);
const rejected = expect(capture).rejects.toMatchObject({ name: "AbortError" });
void capture.then(
() => {
settled = true;
},
() => {
settled = true;
},
);
await vi.waitFor(() => expect(resolutions.size).toBe(4));
controller.abort();
for (const resolve of resolutions.values()) resolve();
await vi.waitFor(() => expect(releases.size).toBe(4));
expect(settled).toBe(false);
expect(fixture.measured()).toEqual([100, 101, 102, 103]);
for (const release of releases.values()) release();
await rejected;
expect(fixture.active()).toBe(0);
expect(fixture.send.mock.calls.some(([, method]) => method === "Runtime.callFunctionOn")).toBe(
false,
);
});
});
@@ -1,6 +1,7 @@
import { describe, expect, it, vi } from "vitest"; import { describe, expect, it, vi } from "vitest";
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge"; import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture"; import { collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
import { captureObservationFacts, semanticCapture } from "../capture-coordinator";
// Minimal but format-accurate captureSnapshot reply: a body with one // Minimal but format-accurate captureSnapshot reply: a body with one
// fixed full-screen overlay div carrying a password input. // fixed full-screen overlay div carrying a password input.
@@ -26,6 +27,7 @@ function fakeSnapshotReply() {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1, 2], parentIndex: [-1, 0, 1, 2],
nodeType: [1, 1, 1, 1], nodeType: [1, 1, 1, 1],
@@ -73,6 +75,7 @@ function hoverTriggerSnapshotReply() {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeType: [1, 1, 1], nodeType: [1, 1, 1],
@@ -117,6 +120,7 @@ function twoHoverTriggerSnapshotReply() {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1, 1], parentIndex: [-1, 0, 1, 1],
nodeType: [1, 1, 1, 1], nodeType: [1, 1, 1, 1],
@@ -170,6 +174,7 @@ function nestedHoverTriggerSnapshotReply() {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1, 2, 3], parentIndex: [-1, 0, 1, 2, 3],
nodeType: [1, 1, 1, 1, 1], nodeType: [1, 1, 1, 1, 1],
@@ -243,13 +248,11 @@ function makeCdp(snapshot: unknown) {
}; };
} }
describe("captureViewModel", () => { describe("snapshot capture and hover", () => {
it("parses nodes, attrs, rects, paint order and styles", async () => { it("parses nodes, attrs, rects, paint order and styles", async () => {
const { nodes, viewport, iframeNodes } = await captureViewModel( const { nodes, viewport } = semanticCapture(
makeCdp(fakeSnapshotReply()), await captureObservationFacts(makeCdp(fakeSnapshotReply()), 4),
4, ).captured;
);
expect(iframeNodes.size).toBe(0);
expect(viewport).toEqual({ width: 1000, height: 800 }); expect(viewport).toEqual({ width: 1000, height: 800 });
const div = nodes.find((n) => n.backendNodeId === 12); const div = nodes.find((n) => n.backendNodeId === 12);
expect(div).toMatchObject({ tag: "div", position: "fixed", parentBackendNodeId: 11 }); expect(div).toMatchObject({ tag: "div", position: "fixed", parentBackendNodeId: 11 });
@@ -261,35 +264,11 @@ describe("captureViewModel", () => {
it("never hovers the page while capturing", async () => { it("never hovers the page while capturing", async () => {
const cdp = makeCdp(fakeSnapshotReply()); const cdp = makeCdp(fakeSnapshotReply());
await captureViewModel(cdp, 4); semanticCapture(await captureObservationFacts(cdp, 4)).captured;
expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything()); expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything());
}); });
it("batch-enriches form controls without per-node object resolution", async () => {
const cdp = makeCdp(fakeSnapshotReply());
const result = await captureViewModel(cdp, 4);
const input = result.nodes.find((node) => node.backendNodeId === 13);
expect(input).toMatchObject({
formState: "filled",
formPlaceholder: "secret",
attrs: { type: "password" },
});
expect(input?.formValue).toBeUndefined();
expect(input?.formDefaultValue).toBeUndefined();
expect(cdp.send).toHaveBeenCalledWith(
4,
"Runtime.evaluate",
expect.objectContaining({
serializationOptions: expect.objectContaining({ serialization: "deep" }),
}),
);
expect(cdp.send).toHaveBeenCalledWith(4, "Runtime.releaseObjectGroup", expect.anything());
expect(cdp.send).not.toHaveBeenCalledWith(4, "DOM.resolveNode", expect.anything());
expect(cdp.send).not.toHaveBeenCalledWith(4, "Runtime.callFunctionOn", expect.anything());
});
it("runs hover surface probes when explicitly enabled", async () => { it("runs hover surface probes when explicitly enabled", async () => {
let hoverStateCalls = 0; let hoverStateCalls = 0;
const cdp = { const cdp = {
@@ -329,7 +308,7 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const captured = await captureViewModel(cdp, 4); const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes); const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
expect(surfaceProbes).toEqual([ expect(surfaceProbes).toEqual([
@@ -389,7 +368,7 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const captured = await captureViewModel(cdp, 4); const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes); const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
expect(surfaceProbes).toEqual([ expect(surfaceProbes).toEqual([
@@ -434,7 +413,7 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const captured = await captureViewModel(cdp, 4); const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
await probeHoverSurfaces(cdp, 4, captured.nodes); await probeHoverSurfaces(cdp, 4, captured.nodes);
expect(hoverMoves).toBe(1); expect(hoverMoves).toBe(1);
@@ -466,6 +445,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1, 0, 3, 4], parentIndex: [-1, 0, 1, 0, 3, 4],
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")], nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
@@ -496,7 +476,7 @@ describe("captureViewModel", () => {
], ],
}; };
const { nodes } = await captureViewModel(makeCdp(snapshot), 4); const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
const ids = nodes.map((n) => n.backendNodeId); const ids = nodes.map((n) => n.backendNodeId);
// Real page nodes survive. // Real page nodes survive.
expect(ids).toContain(12); expect(ids).toContain(12);
@@ -527,6 +507,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1, 0, 3, 4], parentIndex: [-1, 0, 1, 0, 3, 4],
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")], nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
@@ -557,7 +538,9 @@ describe("captureViewModel", () => {
], ],
}; };
const { excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4); const { excludedBackendNodeIds } = semanticCapture(
await captureObservationFacts(makeCdp(snapshot), 4),
).captured;
expect(excludedBackendNodeIds).toEqual(new Set([13, 14, 15])); expect(excludedBackendNodeIds).toEqual(new Set([13, 14, 15]));
}); });
@@ -578,6 +561,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i(OVERLAY_HOST_NAME)], nodeName: [i("html"), i("body"), i(OVERLAY_HOST_NAME)],
@@ -600,7 +584,9 @@ describe("captureViewModel", () => {
], ],
}; };
const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4); const { nodes, excludedBackendNodeIds } = semanticCapture(
await captureObservationFacts(makeCdp(snapshot), 4),
).captured;
expect(nodes.map((n) => n.backendNodeId)).not.toContain(13); expect(nodes.map((n) => n.backendNodeId)).not.toContain(13);
expect(excludedBackendNodeIds).toEqual(new Set([13])); expect(excludedBackendNodeIds).toEqual(new Set([13]));
}); });
@@ -623,6 +609,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("div")], nodeName: [i("html"), i("body"), i("div")],
@@ -645,7 +632,9 @@ describe("captureViewModel", () => {
], ],
}; };
const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4); const { nodes, excludedBackendNodeIds } = semanticCapture(
await captureObservationFacts(makeCdp(snapshot), 4),
).captured;
expect(nodes.map((n) => n.backendNodeId)).not.toContain(13); expect(nodes.map((n) => n.backendNodeId)).not.toContain(13);
expect(excludedBackendNodeIds).toEqual(new Set([13])); expect(excludedBackendNodeIds).toEqual(new Set([13]));
}); });
@@ -668,6 +657,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("div")], nodeName: [i("html"), i("body"), i("div")],
@@ -690,7 +680,7 @@ describe("captureViewModel", () => {
}, },
], ],
}; };
const { nodes } = await captureViewModel(makeCdp(snapshot), 4); const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
expect(nodes.find((n) => n.backendNodeId === 12)?.cursor).toBe("pointer"); expect(nodes.find((n) => n.backendNodeId === 12)?.cursor).toBe("pointer");
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto"); expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
}); });
@@ -702,6 +692,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1, 1, 1], parentIndex: [-1, 0, 1, 1, 1],
nodeName: [i("html"), i("body"), i("button"), i("button"), i("button")], nodeName: [i("html"), i("body"), i("button"), i("button"), i("button")],
@@ -728,7 +719,7 @@ describe("captureViewModel", () => {
], ],
}; };
const { nodes } = await captureViewModel(makeCdp(snapshot), 4); const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
expect(nodes.find((node) => node.backendNodeId === 12)?.rendered).toBe(true); expect(nodes.find((node) => node.backendNodeId === 12)?.rendered).toBe(true);
expect(nodes.find((node) => node.backendNodeId === 13)?.rendered).toBe(false); expect(nodes.find((node) => node.backendNodeId === 13)?.rendered).toBe(false);
@@ -748,19 +739,20 @@ describe("captureViewModel", () => {
throw new Error(method); throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { nodes } = await captureViewModel(cdp, 4); const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
const div = nodes.find((n) => n.backendNodeId === 12); const div = nodes.find((n) => n.backendNodeId === 12);
expect(div?.localRect?.y).toBe(-200); expect(div?.localRect?.y).toBe(-200);
expect(div?.rect).toMatchObject({ y: 0, h: 600 }); expect(div?.rect).toMatchObject({ y: 0, h: 600 });
}); });
it("normalizes device-pixel bounds by devicePixelRatio", async () => { it("keeps CSS snapshot bounds independent of legacy metrics ratio", async () => {
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"]; const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
const i = (s: string) => S.indexOf(s); const i = (s: string) => S.indexOf(s);
const snapshot = { const snapshot = {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("div")], nodeName: [i("html"), i("body"), i("div")],
@@ -795,13 +787,19 @@ describe("captureViewModel", () => {
throw new Error(method); throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { nodes } = await captureViewModel(cdp, 4); const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
expect(nodes.find((n) => n.backendNodeId === 12)?.rect).toEqual({ expect(nodes.find((n) => n.backendNodeId === 12)?.rect).toEqual({
x: 0, x: 0,
y: 0, y: 0,
w: 1000, w: 1000,
h: 800, h: 800,
}); });
expect(nodes.find((n) => n.backendNodeId === 12)?.localRect).toEqual({
x: 0,
y: 0,
w: 2000,
h: 1600,
});
}); });
it("extracts textContent from #text child nodes via nodeValue", async () => { it("extracts textContent from #text child nodes via nodeValue", async () => {
@@ -825,6 +823,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
scrollOffsetX: 0, scrollOffsetX: 0,
scrollOffsetY: 0, scrollOffsetY: 0,
nodes: { nodes: {
@@ -863,7 +862,7 @@ describe("captureViewModel", () => {
throw new Error(method); throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { nodes } = await captureViewModel(cdp, 4); const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
const btn = nodes.find((n) => n.backendNodeId === 12); const btn = nodes.find((n) => n.backendNodeId === 12);
expect(btn?.textContent).toBe("登录"); expect(btn?.textContent).toBe("登录");
const link = nodes.find((n) => n.backendNodeId === 14); const link = nodes.find((n) => n.backendNodeId === 14);
@@ -873,7 +872,7 @@ describe("captureViewModel", () => {
expect(body?.textContent).toBeUndefined(); expect(body?.textContent).toBeUndefined();
}); });
it("parses iframe sub-documents and returns iframeNodes keyed by iframe backendNodeId", async () => { it("retains explicit iframe document ownership and projected nodes", async () => {
const S = [ const S = [
"html", "html",
"body", "body",
@@ -950,36 +949,33 @@ describe("captureViewModel", () => {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
}; };
} }
if (method === "DOM.getBoxModel")
return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } };
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 800, height: 400 } } };
if (method === "Runtime.releaseObject") return {};
throw new Error(method); throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { const facts = await captureObservationFacts(cdp, 4);
nodes, const root = facts.documents.find((doc) => doc.frame.frameId === "main-frame")!;
iframeNodes, const child = facts.documents.find((doc) => doc.frame.frameId === "child-frame")!;
frameNodes, expect(facts.documents).toHaveLength(2);
frameOwnerBackendNodeIds, expect(root.domNodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
frameParentIds, const subNodes = child.domNodes;
rootFrameId,
} = await captureViewModel(cdp, 4);
// Main frame has 4 nodes; iframe is included
expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
// iframeNodes keyed by the <iframe> element's backendNodeId=13
expect(iframeNodes.size).toBe(1);
expect(iframeNodes.has(13)).toBe(true);
const subNodes = iframeNodes.get(13)!;
const input = subNodes.find((n) => n.backendNodeId === 101); const input = subNodes.find((n) => n.backendNodeId === 101);
expect(input?.tag).toBe("input"); expect(input?.tag).toBe("input");
expect(input?.attrs.type).toBe("text"); expect(input?.attrs.type).toBe("text");
expect(input?.ownerFrameBackendNodeId).toBe(13); expect(input?.ownerFrameBackendNodeId).toBe(13);
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 }); expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 }); expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
expect(rootFrameId).toBe("main-frame"); expect(facts.rootFrameId).toBe("main-frame");
expect(frameNodes?.get("child-frame")).toBe(subNodes); expect(child.frame.ownerBackendNodeId).toBe(13);
expect(frameOwnerBackendNodeIds?.get("child-frame")).toBe(13); expect(child.frame.parentFrameId).toBe("main-frame");
expect(frameParentIds?.get("child-frame")).toBe("main-frame");
}); });
it("captures iframe documents without requiring owner-frame geometry", async () => { it("projects iframe documents independently of owner snapshot geometry", async () => {
const S = [ const S = [
"html", "html",
"body", "body",
@@ -997,6 +993,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
scrollOffsetX: 0, scrollOffsetX: 0,
scrollOffsetY: 0, scrollOffsetY: 0,
nodes: { nodes: {
@@ -1014,6 +1011,7 @@ describe("captureViewModel", () => {
}, },
}, },
{ {
frameId: "child",
scrollOffsetX: 0, scrollOffsetX: 0,
scrollOffsetY: 0, scrollOffsetY: 0,
nodes: { nodes: {
@@ -1040,15 +1038,23 @@ describe("captureViewModel", () => {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
}; };
} }
if (method === "DOM.getBoxModel")
return { model: { content: [100, 300, 900, 300, 900, 700, 100, 700] } };
if (method === "DOM.resolveNode") return { object: { objectId: "owner" } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 800, height: 400 } } };
if (method === "Runtime.releaseObject") return {};
throw new Error(method); throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { iframeNodes } = await captureViewModel(cdp, 4); const { documents } = await captureObservationFacts(cdp, 4);
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101); const input = documents
.find((doc) => doc.frame.frameId === "child")
?.domNodes?.find((n) => n.backendNodeId === 101);
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 }); expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
expect(input?.rect).toBeNull(); expect(input?.rect).toEqual({ x: 120, y: 320, w: 120, h: 60 });
}); });
it("recursively normalizes nested iframe sub-documents", async () => { it("recursively normalizes nested iframe sub-documents", async () => {
@@ -1069,6 +1075,7 @@ describe("captureViewModel", () => {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("iframe")], nodeName: [i("html"), i("body"), i("iframe")],
@@ -1090,6 +1097,7 @@ describe("captureViewModel", () => {
}, },
}, },
{ {
frameId: "child",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("iframe")], nodeName: [i("html"), i("body"), i("iframe")],
@@ -1111,6 +1119,7 @@ describe("captureViewModel", () => {
}, },
}, },
{ {
frameId: "nested",
nodes: { nodes: {
parentIndex: [-1, 0], parentIndex: [-1, 0],
nodeName: [i("body"), i("input")], nodeName: [i("body"), i("input")],
@@ -1127,21 +1136,52 @@ describe("captureViewModel", () => {
], ],
}; };
const cdp = { const cdp = {
send: vi.fn(async (_t: number, method: string) => { send: vi.fn(
if (method === "DOMSnapshot.enable") return {}; async (
if (method === "DOMSnapshot.captureSnapshot") return snapshot; _t: number,
if (method === "Page.getLayoutMetrics") { method: string,
return { params?: { backendNodeId?: number; objectId?: string },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, ) => {
}; if (method === "DOMSnapshot.enable") return {};
} if (method === "DOMSnapshot.captureSnapshot") return snapshot;
throw new Error(method); if (method === "Page.getLayoutMetrics") {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, return {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
if (method === "DOM.getBoxModel")
return {
model: {
content:
params?.backendNodeId === 13
? [10, 20, 310, 20, 310, 220, 10, 220]
: [15, 26, 115, 26, 115, 106, 15, 106],
},
};
if (method === "DOM.resolveNode")
return { object: { objectId: String(params?.backendNodeId) } };
if (method === "Runtime.callFunctionOn")
return {
result: {
value:
params?.objectId === "13"
? { width: 300, height: 200 }
: { width: 100, height: 80 },
},
};
if (method === "Runtime.releaseObject") return {};
throw new Error(method);
},
) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { iframeNodes } = await captureViewModel(cdp, 4); const { documents } = await captureObservationFacts(cdp, 4);
const nestedIframe = iframeNodes.get(13)?.find((n) => n.backendNodeId === 23); const nestedIframe = documents
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31); .find((doc) => doc.frame.frameId === "child")
?.domNodes?.find((n) => n.backendNodeId === 23);
const input = documents
.find((doc) => doc.frame.frameId === "nested")
?.domNodes?.find((n) => n.backendNodeId === 31);
expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 }); expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 });
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 }); expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
@@ -1150,13 +1190,14 @@ describe("captureViewModel", () => {
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 }); expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
}); });
it("normalizes bounds then subtracts CSS scroll at dpr>1", async () => { it("subtracts CSS scroll once without dividing snapshot bounds by a metrics ratio", async () => {
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"]; const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
const i = (s: string) => S.indexOf(s); const i = (s: string) => S.indexOf(s);
const snapshot = { const snapshot = {
strings: S, strings: S,
documents: [ documents: [
{ {
frameId: "root",
nodes: { nodes: {
parentIndex: [-1, 0, 1], parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("div")], nodeName: [i("html"), i("body"), i("div")],
@@ -1191,8 +1232,8 @@ describe("captureViewModel", () => {
throw new Error(method); throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>, }) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
}; };
const { nodes } = await captureViewModel(cdp, 4); const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 / 2 - 100); expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 - 100);
}); });
it("collectOverlayExcludedBackendIds walks the pierced overlay host subtree", async () => { it("collectOverlayExcludedBackendIds walks the pierced overlay host subtree", async () => {
@@ -0,0 +1,110 @@
import { describe, expect, it } from "vitest";
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import { buildDocumentIndex, type DecodedNode } from "../facts";
import { decodeDocument, REQUESTED_STYLES } from "../snapshot";
function node(id: number, parent: number | null): DecodedNode {
return {
backendNodeId: id,
parentBackendNodeId: parent,
tag: "div",
attrs: {},
paintOrder: 0,
position: "static",
pointerEvents: "auto",
};
}
describe("document facts", () => {
it.each([
1_000, 10_000, 100_000,
])("indexes deep and wide inputs of %i without copying ancestors", async (count) => {
for (const wide of [false, true]) {
let reads = 0;
const input = Array.from({ length: count }, (_, i) => {
const item = node(i, i ? (wide ? 0 : i - 1) : null);
if (i === 0) item.attrs[OVERLAY_HOST_MARKER_ATTR] = "";
Object.defineProperty(item, "parentBackendNodeId", {
get: () => {
reads++;
return i ? (wide ? 0 : i - 1) : null;
},
});
return item;
}).reverse(); // parent need not precede child
const index = await buildDocumentIndex(input);
expect(index.nodes.size).toBe(count);
expect(index.excludedBackendNodeIds.size).toBe(count);
expect(reads).toBeLessThanOrEqual(count * 8);
expect(index.nodes.get(count - 1)).toBe(input[0]);
}
});
it("handles cycles and orphans and propagates overlay through shadow roots", async () => {
const host = { ...node(1, null), attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } };
const input = [
node(3, 2),
{ ...node(2, 1), tag: "#document-fragment" },
host,
node(4, 99),
node(5, 6),
node(6, 5),
node(7, 6),
];
const index = await buildDocumentIndex(input);
expect([...index.excludedBackendNodeIds].sort()).toEqual([1, 2, 3]);
expect(index.nodes.size).toBe(input.length);
});
it("does not propagate a descendant overlay backwards into a cycle", async () => {
const overlay = { ...node(3, 1), attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } };
const index = await buildDocumentIndex([overlay, node(1, 2), node(2, 1)]);
expect([...index.excludedBackendNodeIds]).toEqual([3]);
});
it("decodes current observation bounds, styles and attributes without projecting coordinates", async () => {
const strings = ["div", "aria-hidden", "true", "inert", "", "visible", "0", "static", "auto"];
const styles = REQUESTED_STYLES.map((style) =>
style === "visibility" ? 5 : style === "opacity" ? 6 : style === "position" ? 7 : 8,
);
const { nodes } = await decodeDocument(
{
nodes: { backendNodeId: [1], parentIndex: [-1], nodeName: [0], attributes: [[1, 2, 3, 4]] },
layout: { nodeIndex: [0], bounds: [[10, 20, 120, 40]], styles: [styles] },
},
strings,
);
expect(nodes[0].attrs).toEqual({ "aria-hidden": "true", inert: "" });
expect(nodes[0].layout).toEqual({
boundsSpace: "snapshot-document-css",
bounds: [10, 20, 120, 40],
styles: {
position: "static",
"pointer-events": "auto",
cursor: "auto",
visibility: "visible",
opacity: "0",
},
});
expect(nodes[0]).not.toHaveProperty("rect");
expect(nodes[0]).not.toHaveProperty("excluded");
});
it("yields during large inputs so cancellation interrupts indexing", async () => {
const controller = new AbortController();
const input = Array.from({ length: 100_000 }, (_, i) => node(i, i ? i - 1 : null));
let reads = 0;
for (const item of input)
Object.defineProperty(item, "attrs", {
get: () => {
reads++;
if (reads === 100) controller.abort();
return {};
},
});
await expect(buildDocumentIndex(input, controller.signal)).rejects.toMatchObject({
name: "AbortError",
});
expect(reads).toBeLessThanOrEqual(356);
});
});
@@ -1,6 +1,6 @@
import { describe, expect, it, vi } from "vitest"; import { describe, expect, it, vi } from "vitest";
import { type CdpRunner, cdpRunnerForTarget } from "../../shared"; import { type CdpRunner, cdpRunnerForTarget } from "../../shared";
import { captureViewModel } from "../capture"; import { captureObservationFacts } from "../capture-coordinator";
interface Control { interface Control {
id: number; id: number;
@@ -106,17 +106,19 @@ describe("form state identity", () => {
[11, state("alice@example.com")], [11, state("alice@example.com")],
]), ]),
); );
const captured = await captureViewModel(cdp, 7); const captured = await captureObservationFacts(cdp, 7);
expect(captured.nodes.find((n) => n.backendNodeId === 10)).toMatchObject({ expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 10)).toMatchObject({
formValue: "", formValue: "",
formState: "empty", formState: "empty",
formPlaceholder: "snapshot placeholder", formPlaceholder: "snapshot placeholder",
}); });
expect(captured.nodes.find((n) => n.backendNodeId === 11)).toMatchObject({ expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 11)).toMatchObject({
formValue: "alice@example.com", formValue: "alice@example.com",
}); });
expect(captured.nodes.find((n) => n.backendNodeId === 12)).toMatchObject({ formValue: "上海" }); expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 12)).toMatchObject({
expect(captured.nodes.some((n) => n.backendNodeId === 99)).toBe(false); formValue: "上海",
});
expect(captured.documents[0].domNodes.some((n) => n.backendNodeId === 99)).toBe(false);
}); });
it("does not shift state between frames when one frame is absent from the runtime batch", async () => { it("does not shift state between frames when one frame is absent from the runtime batch", async () => {
@@ -133,8 +135,10 @@ describe("form state identity", () => {
[21, state("second frame")], [21, state("second frame")],
]), ]),
); );
const captured = await captureViewModel(cdp, 7); const captured = await captureObservationFacts(cdp, 7);
const controls = [...captured.iframeNodes.values()].flat(); const controls = captured.documents
.filter((doc) => doc.frame.parentFrameId)
.flatMap((doc) => doc.domNodes);
expect(controls.find((n) => n.backendNodeId === 11)?.formValue).toBe("inaccessible frame"); expect(controls.find((n) => n.backendNodeId === 11)?.formValue).toBe("inaccessible frame");
expect(controls.find((n) => n.backendNodeId === 21)?.formValue).toBe("second frame"); expect(controls.find((n) => n.backendNodeId === 21)?.formValue).toBe("second frame");
expect(controls.find((n) => n.backendNodeId === 31)?.formValue).toBe("third frame"); expect(controls.find((n) => n.backendNodeId === 31)?.formValue).toBe("third frame");
@@ -151,8 +155,13 @@ describe("form state identity", () => {
}); });
const cdp = { send, sendToTarget } as unknown as CdpRunner; const cdp = { send, sendToTarget } as unknown as CdpRunner;
for (const sessionId of ["left-frame", "right-frame"]) { for (const sessionId of ["left-frame", "right-frame"]) {
const captured = await captureViewModel(cdpRunnerForTarget(cdp, { tabId: 7, sessionId }), 7); const captured = await captureObservationFacts(
expect(captured.nodes.find((n) => n.backendNodeId === 11)?.formValue).toBe(sessionId); cdpRunnerForTarget(cdp, { tabId: 7, sessionId }),
7,
);
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 11)?.formValue).toBe(
sessionId,
);
expect(sendToTarget).toHaveBeenCalledWith( expect(sendToTarget).toHaveBeenCalledWith(
{ tabId: 7, sessionId }, { tabId: 7, sessionId },
"Runtime.releaseObjectGroup", "Runtime.releaseObjectGroup",
@@ -178,8 +187,8 @@ describe("form state identity", () => {
result.result.deepSerializedValue.value[0].value[1].value = "not json"; result.result.deepSerializedValue.value[0].value[1].value = "not json";
return result; return result;
}); });
const captured = await captureViewModel(cdp, 7); const captured = await captureObservationFacts(cdp, 7);
expect(captured.nodes.find((n) => n.backendNodeId === 11)).toMatchObject({ expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 11)).toMatchObject({
formValue: "snapshot value", formValue: "snapshot value",
formPlaceholder: "snapshot placeholder", formPlaceholder: "snapshot placeholder",
}); });
@@ -192,7 +201,7 @@ describe("form state identity", () => {
controller.abort(); controller.abort();
return reply([[11, state("new")]]); return reply([[11, state("new")]]);
}); });
await expect(captureViewModel(cdp, 7, { signal: controller.signal })).rejects.toMatchObject({ await expect(captureObservationFacts(cdp, 7, controller.signal)).rejects.toMatchObject({
name: "AbortError", name: "AbortError",
}); });
expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything()); expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
@@ -216,9 +225,13 @@ describe("form state identity", () => {
return result; return result;
}, },
); );
const captured = await captureViewModel(cdp, 7); const captured = await captureObservationFacts(cdp, 7);
expect(captured.nodes.find((n) => n.backendNodeId === 11)?.formValue).toBe("snapshot value"); expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 11)?.formValue).toBe(
expect(captured.nodes.find((n) => n.backendNodeId === 12)?.formValue).toBe("live value"); "snapshot value",
);
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 12)?.formValue).toBe(
"live value",
);
}); });
it.each([ it.each([
@@ -228,8 +241,8 @@ describe("form state identity", () => {
const { cdp } = fakeCdp([[{ id: 11, value: "secret", type }]], () => const { cdp } = fakeCdp([[{ id: 11, value: "secret", type }]], () =>
reply([[11, { ...state("secret"), sensitive: true }]]), reply([[11, { ...state("secret"), sensitive: true }]]),
); );
const captured = await captureViewModel(cdp, 7); const captured = await captureObservationFacts(cdp, 7);
const control = captured.nodes.find((n) => n.backendNodeId === 11); const control = captured.documents[0].domNodes.find((n) => n.backendNodeId === 11);
expect(control?.formState).toBe("filled"); expect(control?.formState).toBe("filled");
expect(control?.formValue).toBeUndefined(); expect(control?.formValue).toBeUndefined();
expect(control?.formDefaultValue).toBeUndefined(); expect(control?.formDefaultValue).toBeUndefined();
@@ -257,9 +270,13 @@ describe("form state identity", () => {
expect(entries).toHaveLength(250); expect(entries).toHaveLength(250);
return reply(entries.map(([element, json]) => [element.backendNodeId, JSON.parse(json)])); return reply(entries.map(([element, json]) => [element.backendNodeId, JSON.parse(json)]));
}); });
const captured = await captureViewModel(cdp, 7); const captured = await captureObservationFacts(cdp, 7);
expect(captured.nodes.find((n) => n.backendNodeId === 250)?.formValue).toBe("live 250"); expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 250)?.formValue).toBe(
expect(captured.nodes.find((n) => n.backendNodeId === 251)?.formValue).toBe("old"); "live 250",
);
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 251)?.formValue).toBe(
"old",
);
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1); expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1);
expect( expect(
send.mock.calls.filter(([, method]) => method === "Runtime.releaseObjectGroup"), send.mock.calls.filter(([, method]) => method === "Runtime.releaseObjectGroup"),
@@ -1,198 +0,0 @@
import { describe, expect, it, vi } from "vitest";
import type { CdpRunner } from "../../shared";
import type { CapturedNode, CapturedViewModel } from "../capture";
import { captureFrameData } from "../frame-capture";
function ownerNode(backendNodeId: number, x: number): CapturedNode {
return {
backendNodeId,
parentBackendNodeId: 1,
frameId: "main",
ownerFrameBackendNodeId: null,
tag: "iframe",
attrs: {},
rect: { x, y: 100, w: 300, h: 200 },
localRect: { x, y: 100, w: 300, h: 200 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
};
}
function childSnapshot(frameId: string, backendNodeId: number) {
const strings = [frameId, "body", "button", "static", "auto", "pointer"];
return {
strings,
documents: [
{
frameId,
nodes: {
parentIndex: [-1, 0],
nodeName: [1, 2],
backendNodeId: [backendNodeId - 1, backendNodeId],
attributes: [[], []],
},
layout: {
nodeIndex: [0, 1],
styles: [
[3, 4, 4],
[3, 4, 5],
],
bounds: [
[0, 0, 300, 200],
[10, 20, 100, 40],
],
paintOrders: [0, 1],
},
},
],
};
}
describe("captureFrameData", () => {
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
const leftOwner = ownerNode(10, 50);
const rightOwner = ownerNode(20, 500);
const captured: CapturedViewModel = {
nodes: [leftOwner, rightOwner],
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
frameNodes: new Map([["main", [leftOwner, rightOwner]]]),
frameOwnerBackendNodeIds: new Map(),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const sendToTarget = vi.fn(async (target, method) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } };
}
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
return target.sessionId === "left-session"
? childSnapshot("left", 101)
: childSnapshot("right", 201);
}
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
throw new Error(`unexpected ${method}`);
});
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method, params) => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
if (method === "DOM.getBoxModel") {
const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId;
const x = backendNodeId === 10 ? 50 : 500;
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "left",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "left-session" },
},
{
frameId: "right",
parentFrameId: "main",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "right-session" },
},
],
})),
};
const trees = await captureFrameData(cdp, 4, captured);
expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]);
expect(
captured.frameNodes?.get("left")?.find((node) => node.backendNodeId === 101)?.rect,
).toEqual({ x: 60, y: 120, w: 100, h: 40 });
expect(
captured.frameNodes?.get("right")?.find((node) => node.backendNodeId === 201)?.rect,
).toEqual({ x: 510, y: 120, w: 100, h: 40 });
expect(captured.frameOwnerBackendNodeIds).toEqual(
new Map([
["left", 10],
["right", 20],
]),
);
expect(captured.frameParentIds).toEqual(
new Map([
["left", "main"],
["right", "main"],
]),
);
});
it("keeps OOPIF semantics when viewport projection is unavailable", async () => {
const owner = { ...ownerNode(10, 50), rect: null, localRect: null };
const captured: CapturedViewModel = {
nodes: [owner],
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
frameNodes: new Map([["main", [owner]]]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method) => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable");
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Page.getLayoutMetrics") throw new Error("viewport unavailable");
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("child", 101);
if (method === "Accessibility.getFullAXTree") {
return {
nodes: [
{
nodeId: "button",
backendDOMNodeId: 101,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Continue" },
},
],
};
}
throw new Error(`unexpected ${method}`);
}) as unknown as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
})),
};
const documents = await captureFrameData(cdp, 4, captured);
const childDocument = documents.find((document) => document.frameId === "child");
expect(childDocument?.axNodes).toEqual([
expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }),
]);
expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual(
expect.objectContaining({ rect: null, localRect: { x: 10, y: 20, w: 100, h: 40 } }),
);
});
});
@@ -1,7 +1,12 @@
import { describe, expect, it } from "vitest"; import { describe, expect, it, vi } from "vitest";
import type { CdpFrameGraph } from "@/browser-driver/frame-graph"; import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
import type { CapturedNode, CapturedViewModel } from "../capture"; import type { CapturedNode } from "../capture";
import { buildFrameDocuments, type FrameAxBatch, type FrameOwnedAxNode } from "../frame-document"; import {
buildFrameDocuments,
type FrameAxBatch,
type FrameDomInput,
type FrameOwnedAxNode,
} from "../frame-document";
function domNode(frameId: string, backendNodeId: number): CapturedNode { function domNode(frameId: string, backendNodeId: number): CapturedNode {
return { return {
@@ -17,19 +22,16 @@ function domNode(frameId: string, backendNodeId: number): CapturedNode {
}; };
} }
function captured(frameNodes: Map<string, CapturedNode[]>): CapturedViewModel { function captured(frameNodes: Map<string, CapturedNode[]>): FrameDomInput {
return { return {
nodes: frameNodes.get("main") ?? [], nodes: frameNodes.get("main") ?? [],
viewport: { width: 800, height: 600 },
iframeNodes: new Map(),
frameNodes, frameNodes,
rootFrameId: "main", rootFrameId: "main",
excludedBackendNodeIds: new Set(),
}; };
} }
describe("buildFrameDocuments", () => { describe("buildFrameDocuments", () => {
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", () => { it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", async () => {
const graph: CdpFrameGraph = { const graph: CdpFrameGraph = {
rootFrameId: "main", rootFrameId: "main",
frames: [ frames: [
@@ -54,7 +56,7 @@ describe("buildFrameDocuments", () => {
}, },
]; ];
const documents = buildFrameDocuments( const documents = await buildFrameDocuments(
graph, graph,
batches, batches,
captured( captured(
@@ -72,7 +74,7 @@ describe("buildFrameDocuments", () => {
expect(childNodes.find((node) => node.nodeId === "virtual")?.frameId).toBe("child"); expect(childNodes.find((node) => node.nodeId === "virtual")?.frameId).toBe("child");
}); });
it("deduplicates overlapping same-target AX results using the strongest ownership", () => { it("deduplicates overlapping same-target AX results using the strongest ownership", async () => {
const graph: CdpFrameGraph = { const graph: CdpFrameGraph = {
rootFrameId: "main", rootFrameId: "main",
frames: [ frames: [
@@ -92,7 +94,7 @@ describe("buildFrameDocuments", () => {
}, },
]; ];
const documents = buildFrameDocuments( const documents = await buildFrameDocuments(
graph, graph,
batches, batches,
captured( captured(
@@ -107,7 +109,7 @@ describe("buildFrameDocuments", () => {
expect(documents.find((document) => document.frameId === "child")?.axNodes).toHaveLength(1); expect(documents.find((document) => document.frameId === "child")?.axNodes).toHaveLength(1);
}); });
it("uses target-scoped backend ownership when OOPIF backend ids collide", () => { it("uses target-scoped backend ownership when OOPIF backend ids collide", async () => {
const graph: CdpFrameGraph = { const graph: CdpFrameGraph = {
rootFrameId: "main", rootFrameId: "main",
frames: [ frames: [
@@ -129,7 +131,7 @@ describe("buildFrameDocuments", () => {
nodes: [{ nodeId: `${frame.frameId}-node`, backendDOMNodeId: 7 }], nodes: [{ nodeId: `${frame.frameId}-node`, backendDOMNodeId: 7 }],
})); }));
const documents = buildFrameDocuments( const documents = await buildFrameDocuments(
graph, graph,
batches, batches,
captured( captured(
@@ -149,7 +151,7 @@ describe("buildFrameDocuments", () => {
); );
}); });
it("uses captured frame parent identity instead of guessing from backend ids", () => { it("uses captured frame parent identity instead of guessing from backend ids", async () => {
const frameNodes = new Map([ const frameNodes = new Map([
["main", [domNode("main", 1)]], ["main", [domNode("main", 1)]],
["left", [domNode("left", 7)]], ["left", [domNode("left", 7)]],
@@ -157,18 +159,51 @@ describe("buildFrameDocuments", () => {
["child", [domNode("child", 9)]], ["child", [domNode("child", 9)]],
]); ]);
const capture = captured(frameNodes); const capture = captured(frameNodes);
capture.frameOwnerBackendNodeIds = new Map([["child", 7]]);
capture.frameParentIds = new Map([["child", "left"]]);
const frames = ["main", "left", "right", "child"].map((frameId) => ({ const frames = ["main", "left", "right", "child"].map((frameId) => ({
frame: { frameId, target: { tabId: 4 } }, frame: {
frameId,
target: { tabId: 4 },
...(frameId === "child" ? { parentFrameId: "left", ownerBackendNodeId: 7 } : {}),
},
nodes: [], nodes: [],
})); }));
const documents = buildFrameDocuments(null, frames, capture); const documents = await buildFrameDocuments(null, frames, capture);
expect(documents.find((document) => document.frameId === "child")).toMatchObject({ expect(documents.find((document) => document.frameId === "child")).toMatchObject({
parentFrameId: "left", parentFrameId: "left",
ownerBackendNodeId: 7, ownerBackendNodeId: 7,
}); });
}); });
it("rejects explicit AX frame ownership in a different target", async () => {
const frames = [
{ frameId: "main", target: { tabId: 4 } },
{ frameId: "remote", target: { tabId: 4, sessionId: "remote" } },
];
const unresolved = vi.fn();
const documents = await buildFrameDocuments(
{ rootFrameId: "main", frames },
[{ frame: frames[0], nodes: [{ nodeId: "wrong", frameId: "remote" }] }],
captured(new Map()),
undefined,
unresolved,
);
expect(documents.every((doc) => doc.axNodes.length === 0)).toBe(true);
expect(unresolved).toHaveBeenCalledOnce();
});
it("resolves a reverse-ordered deep AX parent chain without recursion", async () => {
const frame = { frameId: "main", target: { tabId: 4 } };
const nodes: FrameOwnedAxNode[] = Array.from({ length: 10000 }, (_, i) => ({
nodeId: String(i),
...(i ? { parentId: String(i - 1) } : { frameId: "main" }),
})).reverse();
const documents = await buildFrameDocuments(
{ rootFrameId: "main", frames: [frame] },
[{ frame, nodes }],
captured(new Map()),
);
expect(documents[0].axNodes).toHaveLength(10000);
expect(documents[0].axNodes[0].frameId).toBe("main");
});
}); });
@@ -0,0 +1,334 @@
import {
type CdpFrame,
type CdpFrameGraph,
type CdpTarget,
cdpTargetKey,
} from "@/browser-driver/frame-graph";
import { cssViewport, GeometryContext } from "../geometry/frame-context";
import { type CdpRunner, cdpRunnerForTarget } from "../shared";
import { collectOverlayExcludedBackendIds } from "./capture";
import type { CapturedSceneInput } from "./facts";
import {
buildDocumentIndex,
type CaptureIssue,
isCaptureAbort,
type ObservationFacts,
throwCaptureAborted,
} from "./facts";
import { enrichFormControlStates } from "./form-capture";
import {
buildFrameDocuments,
type FrameAxBatch,
type FrameDocument,
type FrameOwnedAxNode,
} from "./frame-document";
import { type NormalizedDocument, normalizeSnapshot } from "./normalize";
import { describeSnapshotFrames, REQUESTED_STYLES, type SnapshotReply } from "./snapshot";
interface TargetBatch<T extends FrameOwnedAxNode> {
target: CdpTarget;
frames: CdpFrame[];
documents: NormalizedDocument[];
ax: FrameAxBatch<T>[];
fallbackExcluded: Set<number>;
snapshotFrameIds?: Set<string>;
rootFrameId?: string;
}
/** A fixed worker pool scoped to one capture; each worker issues at most one
* collection request at a time. Geometry has its own measurement-phase bound. */
async function collectTargets<T>(
items: readonly T[],
collect: (item: T) => Promise<void>,
signal?: AbortSignal,
): Promise<void> {
let cursor = 0;
let failed = false;
let failure: unknown;
await Promise.all(
Array.from({ length: Math.min(4, items.length) }, async () => {
try {
while (cursor < items.length && !failed && !signal?.aborted) {
const item = items[cursor++];
await collect(item);
}
} catch (error) {
if (!failed) failure = error;
failed = true;
}
}),
);
// A worker's rejection must not let the capture outlive its own task scope.
// In-flight reads and their finally blocks settle before cancellation/failure.
throwCaptureAborted(signal);
if (failed) throw failure;
}
export async function captureObservationFacts<T extends FrameOwnedAxNode>(
cdp: CdpRunner,
tabId: number,
signal?: AbortSignal,
pageUrl?: string,
): Promise<ObservationFacts<T>> {
const startedAt = Date.now();
throwCaptureAborted(signal);
let graph: CdpFrameGraph | undefined;
try {
graph = await cdp.getFrameGraph?.(tabId);
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
throwCaptureAborted(signal);
const geometry = new GeometryContext(cdp, tabId, graph, signal);
const issues: CaptureIssue[] = [];
const graphFrames = new Map(graph?.frames.map((frame) => [frame.frameId, frame]) ?? []);
const groups = new Map<string, TargetBatch<T>>();
for (const frame of graph?.frames ?? []) {
const key = cdpTargetKey(frame.target);
let group = groups.get(key);
if (!group) {
group = {
target: frame.target,
frames: [],
documents: [],
ax: [],
fallbackExcluded: new Set(),
};
groups.set(key, group);
}
group.frames.push(
!frame.target.sessionId && frame.frameId === graph?.rootFrameId && !frame.url && pageUrl
? { ...frame, url: pageUrl }
: frame,
);
}
if (!groups.has(cdpTargetKey({ tabId })))
groups.set(cdpTargetKey({ tabId }), {
target: { tabId },
frames: [],
documents: [],
ax: [],
fallbackExcluded: new Set(),
});
const batches = [...groups.values()];
const unavailable = (batch: TargetBatch<T>, stage: CaptureIssue["stage"], frameId?: string) =>
issues.push({ target: batch.target, frameId, stage, reason: "capture-unavailable" });
let firstFailure: unknown;
await collectTargets(
batches,
async (batch) => {
const scoped = cdpRunnerForTarget(cdp, batch.target);
try {
throwCaptureAborted(signal);
await scoped.send(tabId, "DOMSnapshot.enable", {});
throwCaptureAborted(signal);
const snapshot = await scoped.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
computedStyles: REQUESTED_STYLES,
includePaintOrder: true,
includeDOMRects: true,
});
throwCaptureAborted(signal);
const observed = describeSnapshotFrames(snapshot, batch.target, graphFrames, pageUrl);
batch.snapshotFrameIds = observed.ids;
batch.rootFrameId = observed.rootFrameId;
// Keep graph-only frames for AX fallback, never in preference to observed membership.
batch.frames = [
...observed.frames,
...batch.frames.filter((frame) => !observed.ids.has(frame.frameId)),
];
batch.documents = await normalizeSnapshot(
snapshot,
batch.target,
batch.frames.filter((frame) => observed.ids.has(frame.frameId)),
geometry,
issues,
signal,
observed.rootFrameId,
);
const formsAvailable = await enrichFormControlStates(
scoped,
tabId,
batch.documents.map((doc) => doc.domNodes),
signal,
);
throwCaptureAborted(signal);
if (!formsAvailable) unavailable(batch, "forms");
const present = new Set(batch.documents.map((doc) => doc.frame.frameId));
for (const frame of batch.frames)
if (!present.has(frame.frameId)) unavailable(batch, "dom", frame.frameId);
} catch (error) {
throwCaptureAborted(signal);
if (isCaptureAbort(error)) throw error;
firstFailure ??= error;
unavailable(batch, "dom");
batch.fallbackExcluded = await collectOverlayExcludedBackendIds(scoped, tabId, signal);
}
if (!batch.frames.length && !graph)
batch.frames = [
{
frameId: "root",
target: batch.target,
...(!batch.target.sessionId && pageUrl ? { url: pageUrl } : {}),
},
];
try {
throwCaptureAborted(signal);
await scoped.send(tabId, "Accessibility.enable", {});
} catch (error) {
throwCaptureAborted(signal);
if (isCaptureAbort(error)) throw error;
firstFailure ??= error;
unavailable(batch, "ax");
return;
}
for (const frame of batch.frames) {
try {
throwCaptureAborted(signal);
const result = await scoped.send<{ nodes?: T[] }>(
tabId,
"Accessibility.getFullAXTree",
frame.frameId === "root" ? {} : { frameId: frame.frameId },
);
throwCaptureAborted(signal);
batch.ax.push({ frame, nodes: result.nodes ?? [] });
} catch (error) {
throwCaptureAborted(signal);
if (isCaptureAbort(error)) throw error;
firstFailure ??= error;
unavailable(batch, "ax", frame.frameId);
}
}
},
signal,
);
// A snapshot claim outranks an old graph hint. Conflicting actual claims
// cannot be resolved by completion order or by overwriting a frameId map.
const claims = new Map<string, string>();
const invalid = new Set<string>();
for (const batch of batches) {
const targetKey = cdpTargetKey(batch.target);
for (const id of batch.snapshotFrameIds ?? []) {
const previous = claims.get(id);
if (previous !== undefined && previous !== targetKey) invalid.add(id);
claims.set(id, targetKey);
}
const accepted = new Set(batch.frames.map((frame) => frame.frameId));
for (const id of batch.snapshotFrameIds ?? []) if (!accepted.has(id)) invalid.add(id);
}
const currentFrames = batches.flatMap((batch) =>
batch.frames.filter(
(frame) =>
!claims.has(frame.frameId) || claims.get(frame.frameId) === cdpTargetKey(batch.target),
),
);
const children = new Map<string, string[]>();
for (const frame of currentFrames) {
if (!frame.parentFrameId) continue;
const siblings = children.get(frame.parentFrameId);
if (siblings) siblings.push(frame.frameId);
else children.set(frame.parentFrameId, [frame.frameId]);
}
const blocked = [...invalid];
for (let i = 0; i < blocked.length; i++) {
for (const child of children.get(blocked[i]) ?? []) {
if (!invalid.has(child)) {
invalid.add(child);
blocked.push(child);
}
}
}
for (const frame of currentFrames) {
if (invalid.has(frame.frameId))
issues.push({
target: frame.target,
frameId: frame.frameId,
stage: "ownership",
reason: "frame-ownership-unresolved",
});
}
const rootBatch = batches.find((batch) => !batch.target.sessionId);
const rootFrameId =
rootBatch?.rootFrameId ?? graph?.rootFrameId ?? rootBatch?.frames[0]?.frameId ?? "root";
if (invalid.has(rootFrameId)) throw new Error("observation root document ownership is ambiguous");
const frames = currentFrames.filter((frame) => !invalid.has(frame.frameId));
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
const belongs = (frame: CdpFrame) => {
const current = frameById.get(frame.frameId);
return current && cdpTargetKey(current.target) === cdpTargetKey(frame.target);
};
const decoded = new Map(
batches
.flatMap((batch) => batch.documents)
.filter((doc) => belongs(doc.frame))
.map((doc) => [doc.frame.frameId, doc]),
);
const ax = batches.flatMap((batch) => batch.ax).filter((batch) => belongs(batch.frame));
const documents = await buildFrameDocuments(
{ rootFrameId, frames },
ax,
{
nodes: [],
rootFrameId,
frameNodes: new Map([...decoded].map(([id, doc]) => [id, doc.domNodes])),
},
signal,
(frame) =>
issues.push({
target: frame.target,
frameId: frame.frameId,
stage: "ownership",
reason: "frame-ownership-unresolved",
}),
);
let viewport = { width: 0, height: 0 };
try {
const measured = cssViewport(await geometry.layoutMetrics({ tabId }));
viewport = { width: measured.width, height: measured.height };
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
const facts: ObservationFacts<T>["documents"][number][] = [];
for (const document of documents) {
const doc = decoded.get(document.frameId);
const index = doc?.index ?? (await buildDocumentIndex([], signal));
const fallback = groups.get(cdpTargetKey(document.target))?.fallbackExcluded;
const { domNodes, axNodes, contextScopeId: _scope, ...frame } = document;
facts.push({
frame,
index: fallback?.size ? { ...index, excludedBackendNodeIds: fallback } : index,
domNodes,
axNodes,
});
}
throwCaptureAborted(signal);
if (firstFailure && facts.every((doc) => !doc.domNodes.length && !doc.axNodes.length))
throw firstFailure;
return { rootFrameId, viewport, documents: facts, issues, startedAt, finishedAt: Date.now() };
}
/** Existing semantic consumers get a narrow view; no raw snapshot
* data can leak through recording's allowlisted output DTO. */
export function semanticCapture<T extends FrameOwnedAxNode>(
facts: ObservationFacts<T>,
): { captured: CapturedSceneInput; documents: FrameDocument<T>[] } {
const documents = facts.documents.map((doc) => ({
...doc.frame,
contextScopeId: doc.frame.frameId,
domNodes: doc.domNodes.filter(
(node) => !doc.index.excludedBackendNodeIds.has(node.backendNodeId),
),
axNodes: doc.axNodes,
excludedBackendNodeIds: doc.index.excludedBackendNodeIds,
}));
const root = documents.find((doc) => doc.frameId === facts.rootFrameId);
return {
documents,
captured: {
nodes: root?.domNodes ?? [],
viewport: facts.viewport,
rootFrameId: facts.rootFrameId,
excludedBackendNodeIds: root?.excludedBackendNodeIds ?? new Set(),
},
};
}
+11 -715
View File
@@ -1,332 +1,21 @@
// CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics // Browser-side hover probes and overlay exclusion fallback. Static collection
// → CapturedNode[] + Viewport. This is the ONLY VOM module that touches // lives in capture-coordinator; snapshot decoding has no browser dependency.
// raw CDP. The captureSnapshot reply is columnar (parallel arrays + a
// shared string table); requested computed styles are decoded through
// STYLE_COL so capture and visibility policy share one explicit contract.
import type { Rect, Viewport } from "@browser-skill/vom";
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy"; import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge"; import { OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
import type { CdpRunner } from "../shared"; import type { CdpRunner } from "../shared";
import type { CapturedNode, CapturedSurfaceProbe } from "./facts";
export type {
CapturedNode,
CapturedSurfaceProbe,
} from "./facts";
import { isCaptureAbort as isAbortError, throwCaptureAborted as throwIfAborted } from "./facts";
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception"; import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
const STYLE_COL = Object.fromEntries(
REQUESTED_STYLES.map((name, index) => [name, index]),
) as Record<(typeof REQUESTED_STYLES)[number], number>;
export interface CapturedNode {
backendNodeId: number;
parentBackendNodeId: number | null;
frameId?: string;
/** Owning iframe backend node id; `null` for the top-level document. */
ownerFrameBackendNodeId?: number | null;
tag: string;
attrs: Record<string, string>;
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
rect: Rect | null;
/** Frame-local viewport-relative CSS px before top-level projection. */
localRect?: Rect | null;
paintOrder: number;
position: string;
pointerEvents: string;
/**
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
* is actually an interactive control — used by the adapter to surface
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
* `textContent`: the live parser always sets it, hand-built fixtures may not.
*/
cursor?: string;
/**
* Whether the live DOM snapshot provides a painted, non-hidden box for this
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
* nodes remain authoritative even when they are outside the viewport.
*/
rendered?: boolean;
textContent?: string;
formState?: "empty" | "filled" | "default";
formValue?: string;
formDefaultValue?: string;
formPlaceholder?: string;
}
export type CapturedIframeNodes = Map<number, CapturedNode[]>;
export interface CapturedSurfaceProbe {
triggerBackendNodeId: number;
triggerPoint?: { x: number; y: number };
triggerAction: "hover" | "focus" | string;
subItems: string[];
confidence?: "high" | "medium" | "low";
}
export interface CapturedViewModel {
nodes: CapturedNode[];
viewport: Viewport;
iframeNodes: CapturedIframeNodes;
frameNodes?: Map<string, CapturedNode[]>;
frameOwnerBackendNodeIds?: Map<string, number>;
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
frameParentIds?: Map<string, string>;
rootFrameId?: string;
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
excludedBackendNodeIds: Set<number>;
}
export interface CaptureViewModelOptions {
signal?: AbortSignal;
}
function captureAbortError(): Error {
const error = new Error("observation aborted");
error.name = "AbortError";
return error;
}
function isAbortError(error: unknown): boolean {
return (
typeof error === "object" &&
error !== null &&
(error as { name?: string }).name === "AbortError"
);
}
function throwIfAborted(signal: AbortSignal | undefined): void {
if (signal?.aborted) throw captureAbortError();
}
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
interface SparseArray {
index: number[];
value: number[];
}
interface RareBooleanData {
index: number[];
}
interface SnapshotDocument {
frameId?: string | number;
scrollOffsetX?: number;
scrollOffsetY?: number;
nodes?: {
parentIndex?: number[];
nodeName?: number[];
backendNodeId?: number[];
attributes?: number[][];
/**
* Per-node text value (index into strings), set for `#text` / CDATA nodes.
* Element nodes carry -1. Same length as `backendNodeId`.
*/
nodeValue?: number[];
/** Maps node array index → index into `documents[]` for frame content. */
contentDocumentIndex?: SparseArray;
inputValue?: SparseArray;
textValue?: SparseArray;
inputChecked?: RareBooleanData;
optionSelected?: RareBooleanData;
};
layout?: {
nodeIndex?: number[];
styles?: number[][];
bounds?: number[][];
paintOrders?: number[];
};
}
function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
if (typeof document.frameId === "string") return document.frameId || undefined;
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
return undefined;
}
interface SnapshotReply {
strings?: string[];
documents?: SnapshotDocument[];
}
interface LayoutMetricsReply {
cssLayoutViewport?: {
clientWidth?: number;
clientHeight?: number;
pageX?: number;
pageY?: number;
};
layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number };
}
function isFormControlTag(tag: string): boolean {
return tag === "input" || tag === "textarea" || tag === "select";
}
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
}
function snapshotFormState(
value: string | undefined,
defaultValue: string,
hasDefaultValue: boolean,
sensitive: boolean,
): CapturedNode["formState"] {
if (sensitive) {
if (value === undefined && !hasDefaultValue) return undefined;
return (value ?? defaultValue) === "" ? "empty" : "filled";
}
if (value === undefined) return undefined;
if (value === "") return "empty";
return value === defaultValue ? "default" : "filled";
}
const MAX_FORM_ENRICH_CONTROLS = 250;
interface CapturedFormState {
value?: string;
defaultValue?: string;
placeholder?: string;
state?: "empty" | "filled" | "default";
sensitive?: boolean;
}
interface DeepSerializedValue {
type: string;
value?: unknown;
}
function formStateBatchExpression(maxControls: number): string {
return `(() => {
const maxControls = ${JSON.stringify(maxControls)};
let remaining = maxControls;
const controlSelector = "input,textarea,select";
const controlState = (el) => {
const tag = el.tagName.toLowerCase();
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
const sensitive = type === "password" || type === "credit-card";
const rawValue = typeof el.value === "string" ? el.value : "";
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
return {
state,
sensitive,
placeholder,
...(sensitive ? {} : { value: rawValue, defaultValue }),
};
};
const controls = [];
const collect = (doc) => {
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
if (remaining <= 0) break;
// Deep serialization supplies the node's backend id. Keep the
// state as JSON so decoding needs no general-purpose V8 deserializer.
controls.push([el, JSON.stringify(controlState(el))]);
remaining -= 1;
}
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
if (remaining <= 0) break;
let childDoc = null;
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
if (childDoc) collect(childDoc);
}
};
collect(document);
return controls;
})()`;
}
function formStatesByBackendId(result: RuntimeEvaluateReply): Map<number, CapturedFormState> {
const states = new Map<number, CapturedFormState>();
const serialized = result.result?.deepSerializedValue;
if (serialized?.type !== "array" || !Array.isArray(serialized.value)) return states;
for (const entry of serialized.value as DeepSerializedValue[]) {
if (entry?.type !== "array" || !Array.isArray(entry.value)) continue;
const [element, json] = entry.value as DeepSerializedValue[];
if (element?.type !== "node" || json?.type !== "string" || typeof json.value !== "string") {
continue;
}
const backendNodeId = (element.value as { backendNodeId?: number } | undefined)?.backendNodeId;
if (typeof backendNodeId !== "number" || !Number.isSafeInteger(backendNodeId)) continue;
try {
const state = JSON.parse(json.value) as CapturedFormState | null;
if (
!state ||
!["empty", "filled", "default"].includes(state.state ?? "") ||
typeof state.sensitive !== "boolean" ||
typeof state.placeholder !== "string" ||
(!state.sensitive &&
(typeof state.value !== "string" || typeof state.defaultValue !== "string"))
) {
continue;
}
states.set(backendNodeId, state);
} catch {
// A malformed entry must not overwrite the snapshot's own state.
}
}
return states;
}
function applyFormStates(nodes: CapturedNode[], states: Map<number, CapturedFormState>): void {
for (const node of nodes) {
if (!isFormControlTag(node.tag)) continue;
const state = states.get(node.backendNodeId);
if (!state) continue;
node.formState = state.state;
node.formPlaceholder = state.placeholder ?? "";
if (state.sensitive || isSensitiveFormControl(node)) {
delete node.formValue;
delete node.formDefaultValue;
delete node.attrs.value;
} else {
node.formDefaultValue = state.defaultValue ?? "";
if (state.value !== undefined) node.formValue = state.value;
}
}
}
async function enrichFormControlStates(
cdp: CdpRunner,
tabId: number,
frameNodeGroups: CapturedNode[][],
signal?: AbortSignal,
): Promise<void> {
const hasControls = frameNodeGroups.some((nodes) =>
nodes.some((node) => isFormControlTag(node.tag)),
);
if (!hasControls) return;
const objectGroup = `bsk-vom-forms-${crypto.randomUUID()}`;
try {
throwIfAborted(signal);
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
objectGroup,
serializationOptions: {
serialization: "deep",
maxDepth: 3,
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
},
});
throwIfAborted(signal);
// Backend ids are scoped to this capture's CDP target. OOPIF targets
// use their own captureViewModel call and never share this lookup.
const states = formStatesByBackendId(result);
for (const nodes of frameNodeGroups) {
applyFormStates(nodes, states);
}
} catch (error) {
if (isAbortError(error)) throw error;
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
} finally {
await cdp.send(tabId, "Runtime.releaseObjectGroup", { objectGroup }).catch(() => undefined);
}
}
interface RuntimeEvaluateReply { interface RuntimeEvaluateReply {
result?: { result?: {
value?: unknown; value?: unknown;
deepSerializedValue?: DeepSerializedValue;
}; };
} }
@@ -345,44 +34,6 @@ interface CdpDomNode {
shadowRoots?: CdpDomNode[]; shadowRoots?: CdpDomNode[];
} }
interface ParseDocumentResult {
nodes: CapturedNode[];
excludedBackendNodeIds: Set<number>;
}
interface FrameContext {
frameId?: string;
ownerFrameBackendNodeId: number | null;
projection: GeometryProjection | null;
scrollX: number;
scrollY: number;
}
function str(strings: string[], idx: number | undefined): string {
if (idx === undefined || idx < 0) return "";
return strings[idx] ?? "";
}
function devicePixelRatio(metrics: LayoutMetricsReply): number {
const layoutW = metrics.layoutViewport?.clientWidth ?? 0;
const cssW = metrics.cssLayoutViewport?.clientWidth ?? 0;
if (!layoutW || !cssW) return 1;
const dpr = layoutW / cssW;
if (!Number.isFinite(dpr) || dpr <= 0) return 1;
return dpr >= 1 ? dpr : 1;
}
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
const out = new Map<number, number>();
if (!sparse?.index || !sparse.value) return out;
for (let i = 0; i < sparse.index.length; i++) {
const nodeIndex = sparse.index[i];
const docIndex = sparse.value[i];
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
}
return out;
}
function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<number>): void { function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<number>): void {
if (!node) return; if (!node) return;
if (typeof node.backendNodeId === "number") { if (typeof node.backendNodeId === "number") {
@@ -791,358 +442,3 @@ export async function collectOverlayExcludedBackendIds(
} }
return excluded; return excluded;
} }
/**
* Parse one DOMSnapshot document into CapturedNode[].
*
* @param doc - the raw DOMSnapshot document object
* @param strings - the shared string table for the whole snapshot
* @param dpr - device-pixel-ratio from Page.getLayoutMetrics
* @param context - frame coordinate context; output rects are top viewport-relative
*/
function parseDocumentNodes(
doc: SnapshotDocument,
strings: string[],
dpr: number,
context: FrameContext,
): ParseDocumentResult {
const dn = doc.nodes;
const dl = doc.layout;
if (!dn?.backendNodeId) {
return { nodes: [], excludedBackendNodeIds: new Set() };
}
const count = dn.backendNodeId.length;
const layoutByNode = new Map<number, number>();
(dl?.nodeIndex ?? []).forEach((nodeIdx, layoutIdx) => layoutByNode.set(nodeIdx, layoutIdx));
// The agent's own overlay (WXT shadow host, marked with OVERLAY_HOST_MARKER_ATTR)
// is injected into the page and its open shadow root
// is inlined by DOMSnapshot as descendants of the host. Its fixed
// full-viewport click-blocker would otherwise dominate occlusion detection
// and hide the real page, so mark the host + its whole subtree excluded.
// parentIndex always references an earlier node, so one forward pass suffices.
const excluded = new Array<boolean>(count).fill(false);
const excludedBackendNodeIds = new Set<number>();
for (let n = 0; n < count; n++) {
const parentIdx = dn.parentIndex?.[n] ?? -1;
const inherited = parentIdx >= 0 && excluded[parentIdx];
let isOverlayHost = inherited;
if (!isOverlayHost) {
const pairs = dn.attributes?.[n] ?? [];
const attrNames: string[] = [];
for (let a = 0; a + 1 < pairs.length; a += 2) {
attrNames.push(str(strings, pairs[a]));
}
isOverlayHost = isOverlayHostNode(str(strings, dn.nodeName?.[n]), attrNames);
}
if (!isOverlayHost) continue;
excluded[n] = true;
const backendNodeId = dn.backendNodeId[n];
if (backendNodeId !== undefined) {
excludedBackendNodeIds.add(backendNodeId);
}
}
const posCol = STYLE_COL.position;
const peCol = STYLE_COL["pointer-events"];
const cursorCol = STYLE_COL.cursor;
const visibilityCol = STYLE_COL.visibility;
const opacityCol = STYLE_COL.opacity;
const inputValues = sparseIndexMap(dn.inputValue);
const textValues = sparseIndexMap(dn.textValue);
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
// Collect visible text from #text child nodes so element CapturedNodes
// carry a textContent value usable as a button/link label fallback.
// nodeValue is a parallel array: string index for text nodes, -1 otherwise.
const nodeTextContent = new Map<number, string>();
if (dn.nodeValue) {
for (let n = 0; n < count; n++) {
const nvIdx = dn.nodeValue[n] ?? -1;
if (nvIdx < 0) continue;
const text = str(strings, nvIdx).trim();
if (!text) continue;
const parentIdx = dn.parentIndex?.[n] ?? -1;
if (parentIdx >= 0) {
const existing = nodeTextContent.get(parentIdx);
nodeTextContent.set(parentIdx, existing ? `${existing} ${text}` : text);
}
}
}
// Build CapturedNodes.
const nodes: CapturedNode[] = [];
for (let n = 0; n < count; n++) {
if (excluded[n]) continue;
const backendNodeId = dn.backendNodeId[n];
const parentIdx = dn.parentIndex?.[n] ?? -1;
const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null;
const tag = str(strings, dn.nodeName?.[n]).toLowerCase();
const attrs: Record<string, string> = {};
const pairs = dn.attributes?.[n] ?? [];
for (let a = 0; a + 1 < pairs.length; a += 2) {
attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]);
}
let rect: Rect | null = null;
let localRect: Rect | null = null;
let paintOrder = 0;
let position = "static";
let pointerEvents = "auto";
let cursor = "auto";
let rendered = false;
const li = layoutByNode.get(n);
if (li !== undefined) {
const b = dl?.bounds?.[li];
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
localRect = {
x: b[0] / dpr - context.scrollX,
y: b[1] / dpr - context.scrollY,
w: b[2] / dpr,
h: b[3] / dpr,
};
const bounds = context.projection
? projectRectToViewport(localRect, context.projection)
: null;
rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
}
paintOrder = dl?.paintOrders?.[li] ?? 0;
const styleRow = dl?.styles?.[li] ?? [];
position = str(strings, styleRow[posCol]) || "static";
pointerEvents = str(strings, styleRow[peCol]) || "auto";
cursor = str(strings, styleRow[cursorCol]) || "auto";
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
const opacity = str(strings, styleRow[opacityCol]) || "1";
rendered =
localRect !== null &&
visibility !== "hidden" &&
visibility !== "collapse" &&
(Number.parseFloat(opacity) || 0) > 0;
}
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
// geometry and are never queried by callers.
if (tag.startsWith("#")) continue;
const textContent = nodeTextContent.get(n);
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
const rawFormValue =
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
const sensitive = isSensitiveFormControl({ tag, attrs });
const formDefaultValue = attrs.value ?? "";
const formValue = sensitive ? undefined : rawFormValue;
const formState = snapshotFormState(
rawFormValue,
formDefaultValue,
Object.prototype.hasOwnProperty.call(attrs, "value"),
sensitive,
);
if (sensitive) delete attrs.value;
nodes.push({
backendNodeId,
parentBackendNodeId,
...(context.frameId ? { frameId: context.frameId } : {}),
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
tag,
attrs,
rect,
localRect,
paintOrder,
position,
pointerEvents,
cursor,
rendered,
textContent,
...(formValue !== undefined ? { formValue } : {}),
...(tag === "input" || tag === "textarea"
? {
formPlaceholder: attrs.placeholder ?? "",
...(!sensitive ? { formDefaultValue } : {}),
...(formState ? { formState } : {}),
}
: {}),
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
});
}
return { nodes, excludedBackendNodeIds };
}
interface ParseFrameDocumentsResult {
iframeNodes: CapturedIframeNodes;
frameOwnerBackendNodeIds: Map<string, number>;
frameParentIds: Map<string, string>;
excludedBackendNodeIds: Set<number>;
}
function parseChildFrameDocuments(
documents: SnapshotDocument[],
strings: string[],
dpr: number,
parentDocIndex: number,
parentNodes: CapturedNode[],
parentContext: FrameContext,
visited = new Set<number>(),
): ParseFrameDocumentsResult {
const iframeNodes: CapturedIframeNodes = new Map();
const frameOwnerBackendNodeIds = new Map<string, number>();
const frameParentIds = new Map<string, string>();
const excludedBackendNodeIds = new Set<number>();
const parentDoc = documents[parentDocIndex];
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
if (cdi.size === 0) {
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
}
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
for (const [nodeArrayIdx, childDocIndex] of cdi) {
if (visited.has(childDocIndex)) continue;
const childDoc = documents[childDocIndex];
if (!childDoc) continue;
const iframeBackendId = parentBackendIds[nodeArrayIdx];
if (iframeBackendId === undefined) continue;
const iframeNode = parentNodeByBackendId.get(iframeBackendId);
const projection =
parentContext.projection && iframeNode?.localRect
? childFrameProjection(parentContext.projection, iframeNode.localRect)
: null;
const childContext: FrameContext = {
frameId: snapshotFrameId(childDoc, strings),
ownerFrameBackendNodeId: iframeBackendId,
projection,
scrollX: childDoc.scrollOffsetX ?? 0,
scrollY: childDoc.scrollOffsetY ?? 0,
};
const nextVisited = new Set(visited);
nextVisited.add(childDocIndex);
const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext);
iframeNodes.set(iframeBackendId, parsed.nodes);
if (childContext.frameId) {
frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId);
if (parentContext.frameId) frameParentIds.set(childContext.frameId, parentContext.frameId);
}
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
const nested = parseChildFrameDocuments(
documents,
strings,
dpr,
childDocIndex,
parsed.nodes,
childContext,
nextVisited,
);
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
iframeNodes.set(nestedFrameId, nestedNodes);
}
for (const [frameId, ownerBackendNodeId] of nested.frameOwnerBackendNodeIds) {
frameOwnerBackendNodeIds.set(frameId, ownerBackendNodeId);
}
for (const [frameId, parentFrameId] of nested.frameParentIds) {
frameParentIds.set(frameId, parentFrameId);
}
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
}
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
}
export async function captureViewModel(
cdp: CdpRunner,
tabId: number,
options: CaptureViewModelOptions = {},
): Promise<CapturedViewModel> {
throwIfAborted(options.signal);
let metrics: LayoutMetricsReply = {};
try {
metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
} catch (error) {
if (isAbortError(error)) throw error;
console.debug("[bsk capture] layout metrics unavailable", error);
}
throwIfAborted(options.signal);
const dpr = devicePixelRatio(metrics);
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
const viewport: Viewport = {
width: vpSrc.clientWidth ?? 0,
height: vpSrc.clientHeight ?? 0,
};
const scrollX = vpSrc.pageX ?? 0;
const scrollY = vpSrc.pageY ?? 0;
await cdp.send(tabId, "DOMSnapshot.enable", {});
throwIfAborted(options.signal);
const snap = await cdp.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
computedStyles: REQUESTED_STYLES,
includePaintOrder: true,
includeDOMRects: true,
});
throwIfAborted(options.signal);
const strings = snap.strings ?? [];
const documents = snap.documents ?? [];
const doc0 = documents[0];
if (!doc0?.nodes?.backendNodeId) {
return {
nodes: [],
viewport,
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
};
}
const topContext: FrameContext = {
frameId: snapshotFrameId(doc0, strings),
ownerFrameBackendNodeId: null,
projection: {
sourceClips: [],
edges: [],
topViewport: viewport,
},
scrollX,
scrollY,
};
const mainParsed = parseDocumentNodes(doc0, strings, dpr, topContext);
const nodes = mainParsed.nodes;
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
const frameParsed = parseChildFrameDocuments(documents, strings, dpr, 0, nodes, topContext);
const iframeNodes = frameParsed.iframeNodes;
for (const id of frameParsed.excludedBackendNodeIds) {
excludedBackendNodeIds.add(id);
}
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()], options.signal);
throwIfAborted(options.signal);
const frameNodes = new Map<string, CapturedNode[]>();
if (topContext.frameId) frameNodes.set(topContext.frameId, nodes);
for (const iframeFrameNodes of iframeNodes.values()) {
const frameId = iframeFrameNodes.find((node) => node.frameId)?.frameId;
if (frameId) frameNodes.set(frameId, iframeFrameNodes);
}
for (const [frameId, ownerBackendNodeId] of frameParsed.frameOwnerBackendNodeIds) {
if (!frameNodes.has(frameId)) {
frameNodes.set(frameId, iframeNodes.get(ownerBackendNodeId) ?? []);
}
}
return {
nodes,
viewport,
iframeNodes,
frameNodes,
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
frameParentIds: frameParsed.frameParentIds,
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
excludedBackendNodeIds,
};
}
+183
View File
@@ -0,0 +1,183 @@
import type { Rect, Viewport } from "@browser-skill/vom";
import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph";
import { isOverlayHostNode } from "@/lib/overlay-bridge";
import type { FrameProjectionIssue } from "../geometry/coordinate-types";
import type { FrameOwnedAxNode } from "./frame-document";
export interface CapturedNode {
backendNodeId: number;
parentBackendNodeId: number | null;
frameId?: string;
/** Owning iframe backend node id; `null` for the top-level document. */
ownerFrameBackendNodeId?: number | null;
tag: string;
attrs: Record<string, string>;
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
rect: Rect | null;
/** Frame-local viewport-relative CSS px before top-level projection. */
localRect?: Rect | null;
paintOrder: number;
position: string;
pointerEvents: string;
/**
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
* is actually an interactive control — used by the adapter to surface
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
* `textContent`: the live parser always sets it, hand-built fixtures may not.
*/
cursor?: string;
/**
* Whether the live DOM snapshot provides a painted, non-hidden box for this
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
* nodes remain authoritative even when they are outside the viewport.
*/
rendered?: boolean;
textContent?: string;
formState?: "empty" | "filled" | "default";
formValue?: string;
formDefaultValue?: string;
formPlaceholder?: string;
}
export interface CapturedSurfaceProbe {
triggerBackendNodeId: number;
triggerPoint?: { x: number; y: number };
triggerAction: "hover" | "focus" | string;
subItems: string[];
confidence?: "high" | "medium" | "low";
}
/** Narrow input shared by the existing semantic scene/hover consumers. */
export interface CapturedSceneInput {
nodes: CapturedNode[];
viewport: Viewport;
rootFrameId?: string;
excludedBackendNodeIds: ReadonlySet<number>;
}
/** Bounds retain their snapshot document CSS units until normalization. */
export interface SnapshotLayout {
readonly boundsSpace: "snapshot-document-css";
bounds?: number[];
styles: Readonly<Record<string, string>>;
}
export interface DecodedNode extends Omit<CapturedNode, "rect" | "localRect" | "rendered"> {
nodeType?: number;
layout?: SnapshotLayout;
}
export interface NodeFacts extends CapturedNode {
nodeType?: number;
layout?: SnapshotLayout;
}
export interface DocumentIndex<T extends DecodedNode = NodeFacts> {
readonly nodes: ReadonlyMap<number, T>;
readonly excludedBackendNodeIds: ReadonlySet<number>;
}
export interface DecodedDocument {
nodes: DecodedNode[];
}
export interface CaptureIssue {
/** Owner failure or inherited blockage; absent for other geometry failures. */
projectionIssue?: FrameProjectionIssue;
target: CdpTarget;
frameId?: string;
stage: "dom" | "ax" | "ownership" | "geometry" | "forms";
reason: "capture-unavailable" | "frame-ownership-unresolved" | "geometry-unavailable";
}
export interface DocumentFacts<T extends FrameOwnedAxNode> {
readonly frame: CdpFrame;
readonly index: DocumentIndex;
readonly domNodes: CapturedNode[];
readonly axNodes: T[];
}
export interface ObservationFacts<T extends FrameOwnedAxNode> {
readonly rootFrameId: string;
readonly viewport: Viewport;
readonly documents: readonly DocumentFacts<T>[];
readonly issues: readonly CaptureIssue[];
readonly startedAt: number;
readonly finishedAt: number;
}
export function throwCaptureAborted(signal?: AbortSignal): void {
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
}
export function isCaptureAbort(error: unknown): boolean {
return error instanceof Error && error.name === "AbortError";
}
/** Called only at bounded block boundaries, not once per node. */
export async function captureCheckpoint(signal?: AbortSignal): Promise<void> {
throwCaptureAborted(signal);
await new Promise<void>((resolve) => {
const channel = new MessageChannel();
channel.port1.onmessage = () => {
channel.port1.close();
channel.port2.close();
resolve();
};
channel.port2.postMessage(null);
});
throwCaptureAborted(signal);
}
/** All snapshot nodes participate, including document and shadow roots. This
* preserves overlay propagation when the semantic adapter omits those nodes. */
export async function buildDocumentIndex<T extends DecodedNode>(
input: readonly T[],
signal?: AbortSignal,
): Promise<DocumentIndex<T>> {
const nodes = new Map<number, T>();
const overlayByNode = new Map<number, boolean>();
const excludedBackendNodeIds = new Set<number>();
for (let i = 0; i < input.length; i++) {
if (i % 256 === 0) await captureCheckpoint(signal);
const node = input[i];
nodes.set(node.backendNodeId, node);
}
let work = 0;
for (const node of input) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
if (overlayByNode.has(node.backendNodeId)) continue;
const path: DecodedNode[] = [];
const visiting = new Set<number>();
let current: DecodedNode | undefined = node;
let overlay = false;
while (current && !overlayByNode.has(current.backendNodeId)) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
if (visiting.has(current.backendNodeId)) {
// Only an overlay inside the cycle may mark the cycle and its descendants.
overlay = path
.slice(path.findIndex((item) => item.backendNodeId === current!.backendNodeId))
.some((item) => isOverlayHostNode(item.tag, Object.keys(item.attrs)));
break;
}
visiting.add(current.backendNodeId);
path.push(current);
if (current.parentBackendNodeId === null) {
current = undefined;
break;
}
current = nodes.get(current.parentBackendNodeId);
}
if (current && overlayByNode.has(current.backendNodeId))
overlay = overlayByNode.get(current.backendNodeId)!;
for (let i = path.length - 1; i >= 0; i--) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
const item = path[i];
overlay = overlay || isOverlayHostNode(item.tag, Object.keys(item.attrs));
overlayByNode.set(item.backendNodeId, overlay);
if (overlay) excludedBackendNodeIds.add(item.backendNodeId);
}
}
return { nodes, excludedBackendNodeIds };
}
@@ -0,0 +1,155 @@
import type { CdpRunner } from "../shared";
import type { CapturedNode } from "./facts";
import { isCaptureAbort as isAbortError, throwCaptureAborted as throwIfAborted } from "./facts";
interface RuntimeEvaluateReply {
result?: { deepSerializedValue?: DeepSerializedValue };
}
const MAX_FORM_ENRICH_CONTROLS = 250;
interface CapturedFormState {
value?: string;
defaultValue?: string;
placeholder?: string;
state?: "empty" | "filled" | "default";
sensitive?: boolean;
}
interface DeepSerializedValue {
type: string;
value?: unknown;
}
function formStateBatchExpression(maxControls: number): string {
return `(() => {
const maxControls = ${JSON.stringify(maxControls)};
let remaining = maxControls;
const controlSelector = "input,textarea,select";
const controlState = (el) => {
const tag = el.tagName.toLowerCase();
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
const sensitive = type === "password" || type === "credit-card";
const rawValue = typeof el.value === "string" ? el.value : "";
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
return {
state,
sensitive,
placeholder,
...(sensitive ? {} : { value: rawValue, defaultValue }),
};
};
const controls = [];
const collect = (doc) => {
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
if (remaining <= 0) break;
// Deep serialization supplies the node's backend id. Keep the
// state as JSON so decoding needs no general-purpose V8 deserializer.
controls.push([el, JSON.stringify(controlState(el))]);
remaining -= 1;
}
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
if (remaining <= 0) break;
let childDoc = null;
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
if (childDoc) collect(childDoc);
}
};
collect(document);
return controls;
})()`;
}
function formStatesByBackendId(result: RuntimeEvaluateReply): Map<number, CapturedFormState> {
const states = new Map<number, CapturedFormState>();
const serialized = result.result?.deepSerializedValue;
if (serialized?.type !== "array" || !Array.isArray(serialized.value)) return states;
for (const entry of serialized.value as DeepSerializedValue[]) {
if (entry?.type !== "array" || !Array.isArray(entry.value)) continue;
const [element, json] = entry.value as DeepSerializedValue[];
if (element?.type !== "node" || json?.type !== "string" || typeof json.value !== "string") {
continue;
}
const backendNodeId = (element.value as { backendNodeId?: number } | undefined)?.backendNodeId;
if (typeof backendNodeId !== "number" || !Number.isSafeInteger(backendNodeId)) continue;
try {
const state = JSON.parse(json.value) as CapturedFormState | null;
if (
!state ||
!["empty", "filled", "default"].includes(state.state ?? "") ||
typeof state.sensitive !== "boolean" ||
typeof state.placeholder !== "string" ||
(!state.sensitive &&
(typeof state.value !== "string" || typeof state.defaultValue !== "string"))
) {
continue;
}
states.set(backendNodeId, state);
} catch {
// A malformed entry must not overwrite the snapshot's own state.
}
}
return states;
}
function applyFormStates(nodes: CapturedNode[], states: Map<number, CapturedFormState>): void {
for (const node of nodes) {
if (!["input", "textarea", "select"].includes(node.tag)) continue;
const state = states.get(node.backendNodeId);
if (!state) continue;
node.formState = state.state;
node.formPlaceholder = state.placeholder ?? "";
if (
state.sensitive ||
(node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password")
) {
delete node.formValue;
delete node.formDefaultValue;
delete node.attrs.value;
} else {
node.formDefaultValue = state.defaultValue ?? "";
if (state.value !== undefined) node.formValue = state.value;
}
}
}
export async function enrichFormControlStates(
cdp: CdpRunner,
tabId: number,
frameNodeGroups: CapturedNode[][],
signal?: AbortSignal,
): Promise<boolean> {
const hasControls = frameNodeGroups.some((nodes) =>
nodes.some((node) => ["input", "textarea", "select"].includes(node.tag)),
);
if (!hasControls) return true;
const objectGroup = `bsk-vom-forms-${crypto.randomUUID()}`;
try {
throwIfAborted(signal);
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
objectGroup,
serializationOptions: {
serialization: "deep",
maxDepth: 3,
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
},
});
throwIfAborted(signal);
// Backend ids are scoped to this capture's CDP target. OOPIF targets
// have separate enrichment calls and never share this lookup.
const states = formStatesByBackendId(result);
for (const nodes of frameNodeGroups) {
applyFormStates(nodes, states);
}
return true;
} catch (error) {
if (isAbortError(error)) throw error;
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
return false;
} finally {
await cdp.send(tabId, "Runtime.releaseObjectGroup", { objectGroup }).catch(() => undefined);
throwIfAborted(signal);
}
}
@@ -1,181 +0,0 @@
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
import { resolveFrameProjection } from "../frame-geometry";
import { type GeometryProjection, projectRectToViewport } from "../geometry";
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
import {
buildFrameDocuments,
type FrameAxBatch,
type FrameDocument,
type FrameOwnedAxNode,
} from "./frame-document";
export type FrameAxNode = FrameOwnedAxNode;
export type CapturedFrameDocument<T extends FrameAxNode> = FrameDocument<T>;
function throwIfAborted(signal?: AbortSignal): void {
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
}
function isAbortError(err: unknown): boolean {
return err instanceof Error && err.name === "AbortError";
}
async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
if (!cdp.getFrameGraph) return null;
try {
return await cdp.getFrameGraph(tabId);
} catch (err) {
console.debug("[bsk observation] frame graph capture failed", err);
return null;
}
}
function transformFrameNodes(
nodes: CapturedNode[],
projection: GeometryProjection | null,
): CapturedNode[] {
return nodes.map((node) => ({
...node,
rect: projection
? (() => {
const bounds = projectRectToViewport(node.rect, projection);
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
})()
: null,
}));
}
async function captureMissingFrameDocuments(
cdp: CdpRunner,
tabId: number,
graph: CdpFrameGraph,
captured: CapturedViewModel,
signal?: AbortSignal,
): Promise<void> {
if (!captured.frameNodes) captured.frameNodes = new Map();
if (!captured.frameOwnerBackendNodeIds) captured.frameOwnerBackendNodeIds = new Map();
if (!captured.frameParentIds) captured.frameParentIds = new Map();
for (const frame of graph.frames) {
if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue;
try {
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
signal,
});
let projection: GeometryProjection | null = null;
try {
projection = await resolveFrameProjection(cdp, graph, frame.frameId);
} catch (err) {
if (isAbortError(err)) throw err;
console.debug("[bsk observation] child frame geometry projection failed", {
frameId: frame.frameId,
err,
});
}
const rootFrameId = child.rootFrameId ?? frame.frameId;
const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
if (rootFrameId !== frame.frameId && !childFrames.has(frame.frameId)) {
childFrames.set(
frame.frameId,
child.nodes.map((node) => ({ ...node, frameId: frame.frameId })),
);
}
for (const [childFrameId, nodes] of childFrames) {
captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
}
if (frame.ownerBackendNodeId !== undefined) {
captured.iframeNodes.set(
frame.ownerBackendNodeId,
captured.frameNodes.get(frame.frameId) ?? [],
);
captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
}
if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId);
for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) {
captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId);
}
for (const [childFrameId, parentFrameId] of child.frameParentIds ?? []) {
captured.frameParentIds.set(childFrameId, parentFrameId);
}
} catch (err) {
if (isAbortError(err)) throw err;
console.debug("[bsk observation] child frame DOM capture failed", {
frameId: frame.frameId,
err,
});
}
}
}
async function captureAxTrees<T extends FrameAxNode>(
cdp: CdpRunner,
tabId: number,
discoveredFrames: CdpFrame[],
captured: CapturedViewModel,
signal?: AbortSignal,
): Promise<FrameAxBatch<T>[]> {
let frames = [...discoveredFrames];
const capturedFrameIds = [...(captured.frameNodes?.keys() ?? [])];
if (frames.length === 0) {
const rootFrameId = captured.rootFrameId ?? capturedFrameIds[0] ?? "root";
frames = [{ frameId: rootFrameId, target: { tabId } }];
}
for (const frameId of capturedFrameIds) {
if (!frames.some((frame) => frame.frameId === frameId)) {
frames.push({ frameId, target: { tabId } });
}
}
let firstFailure: unknown;
const trees = await Promise.all(
frames.map(async (frame): Promise<FrameAxBatch<T>> => {
try {
throwIfAborted(signal);
await sendToCdpTarget(cdp, frame.target, "Accessibility.enable", {});
const result = await sendToCdpTarget<{ nodes?: T[] }>(
cdp,
frame.target,
"Accessibility.getFullAXTree",
frame.frameId === "root" ? {} : { frameId: frame.frameId },
);
throwIfAborted(signal);
return {
frame,
nodes: result.nodes ?? [],
};
} catch (err) {
if (isAbortError(err)) throw err;
firstFailure ??= err;
console.debug("[bsk observation] frame AX capture failed", {
frameId: frame.frameId,
err,
});
return { frame, nodes: [] };
}
}),
);
const capturedNodeCount = [...captured.iframeNodes.values()].reduce(
(count, nodes) => count + nodes.length,
captured.nodes.length,
);
if (firstFailure && capturedNodeCount === 0 && trees.every((tree) => tree.nodes.length === 0)) {
throw firstFailure;
}
return trees;
}
export async function captureFrameData<T extends FrameAxNode>(
cdp: CdpRunner,
tabId: number,
captured: CapturedViewModel,
signal?: AbortSignal,
): Promise<CapturedFrameDocument<T>[]> {
const graph = await discoverFrameGraph(cdp, tabId);
const frames = graph?.frames ?? [];
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal);
throwIfAborted(signal);
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
return buildFrameDocuments(graph, batches, captured);
}
+72 -53
View File
@@ -4,7 +4,8 @@ import {
type CdpTarget, type CdpTarget,
cdpTargetKey, cdpTargetKey,
} from "@/browser-driver/frame-graph"; } from "@/browser-driver/frame-graph";
import type { CapturedNode, CapturedViewModel } from "./capture"; import type { CapturedNode } from "./facts";
import { captureCheckpoint } from "./facts";
export interface FrameOwnedAxNode { export interface FrameOwnedAxNode {
nodeId: string; nodeId: string;
@@ -23,6 +24,14 @@ export interface FrameDocument<T extends FrameOwnedAxNode> extends CdpFrame {
contextScopeId: string; contextScopeId: string;
axNodes: T[]; axNodes: T[];
domNodes: CapturedNode[]; domNodes: CapturedNode[];
excludedBackendNodeIds?: ReadonlySet<number>;
}
/** Explicit snapshot document data used to join DOM and AX ownership. */
export interface FrameDomInput {
nodes: CapturedNode[];
rootFrameId?: string;
frameNodes?: ReadonlyMap<string, CapturedNode[]>;
} }
interface Ownership { interface Ownership {
@@ -47,35 +56,24 @@ function targetBackendKey(target: CdpTarget, backendNodeId: number): string {
function frameList<T extends FrameOwnedAxNode>( function frameList<T extends FrameOwnedAxNode>(
graph: CdpFrameGraph | null, graph: CdpFrameGraph | null,
batches: FrameAxBatch<T>[], batches: FrameAxBatch<T>[],
captured: CapturedViewModel,
): CdpFrame[] { ): CdpFrame[] {
const frames = new Map<string, CdpFrame>(); const frames = new Map<string, CdpFrame>();
for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame); for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame);
for (const batch of batches) { for (const batch of batches) {
if (!frames.has(batch.frame.frameId)) frames.set(batch.frame.frameId, batch.frame); if (!frames.has(batch.frame.frameId)) frames.set(batch.frame.frameId, batch.frame);
} }
for (const frameId of captured.frameNodes?.keys() ?? []) { return [...frames.values()];
if (!frames.has(frameId))
frames.set(frameId, { frameId, target: { tabId: batches[0]?.frame.target.tabId ?? 0 } });
}
return [...frames.values()].map((frame) => {
const ownerBackendNodeId =
frame.ownerBackendNodeId ?? captured.frameOwnerBackendNodeIds?.get(frame.frameId);
const parentFrameId = frame.parentFrameId ?? captured.frameParentIds?.get(frame.frameId);
return {
...frame,
...(ownerBackendNodeId !== undefined ? { ownerBackendNodeId } : {}),
...(parentFrameId ? { parentFrameId } : {}),
};
});
} }
export function buildFrameDocuments<T extends FrameOwnedAxNode>( export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
graph: CdpFrameGraph | null, graph: CdpFrameGraph | null,
batches: FrameAxBatch<T>[], batches: FrameAxBatch<T>[],
captured: CapturedViewModel, captured: FrameDomInput,
): FrameDocument<T>[] { signal?: AbortSignal,
const frames = frameList(graph, batches, captured); unresolved?: (frame: CdpFrame) => void,
): Promise<FrameDocument<T>[]> {
let work = 0;
const frames = frameList(graph, batches);
const frameById = new Map(frames.map((frame) => [frame.frameId, frame])); const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId; const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId;
const domNodesForFrame = (frameId: string): CapturedNode[] => const domNodesForFrame = (frameId: string): CapturedNode[] =>
@@ -83,54 +81,74 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
const backendOwner = new Map<string, string>(); const backendOwner = new Map<string, string>();
for (const frame of frames) { for (const frame of frames) {
for (const node of domNodesForFrame(frame.frameId)) { for (const node of domNodesForFrame(frame.frameId)) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId); backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId);
} }
} }
const candidates = new Map<string, OwnedCandidate<T>>(); const candidates = new Map<string, OwnedCandidate<T>>();
for (const batch of batches) { for (const batch of batches) {
const nodeById = new Map(batch.nodes.map((node) => [node.nodeId, node])); const nodeById = new Map<string, T>();
for (const node of batch.nodes) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
nodeById.set(node.nodeId, node);
}
let reported = false;
const ownershipByNodeId = new Map<string, Ownership>(); const ownershipByNodeId = new Map<string, Ownership>();
const resolving = new Set<string>(); const resolveOwnership = async (node: T): Promise<Ownership> => {
const resolveOwnership = (node: T): Ownership => { const path: T[] = [];
const cached = ownershipByNodeId.get(node.nodeId); const visiting = new Set<string>();
if (cached) return cached; let current: T | undefined = node;
if (node.frameId && frameById.has(node.frameId)) { let ownership: Ownership = { frameId: batch.frame.frameId, strength: 1 };
const ownership = { frameId: node.frameId, strength: 4 }; while (current) {
ownershipByNodeId.set(node.nodeId, ownership); if (work++ % 256 === 0) await captureCheckpoint(signal);
return ownership; const cached = ownershipByNodeId.get(current.nodeId);
} if (cached) {
if (typeof node.backendDOMNodeId === "number") { ownership = cached;
const frameId = backendOwner.get( break;
targetBackendKey(batch.frame.target, node.backendDOMNodeId), }
); if (visiting.has(current.nodeId)) break;
visiting.add(current.nodeId);
if (current.frameId) {
const frame = frameById.get(current.frameId);
// An explicit contradiction must not fall through to a guessed owner.
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(batch.frame.target)) {
if (!reported) {
unresolved?.(batch.frame);
reported = true;
}
ownership = { frameId: "", strength: 0 };
break;
}
ownership = { frameId: frame.frameId, strength: 4 };
ownershipByNodeId.set(current.nodeId, ownership);
break;
}
const frameId =
typeof current.backendDOMNodeId === "number"
? backendOwner.get(targetBackendKey(batch.frame.target, current.backendDOMNodeId))
: undefined;
if (frameId) { if (frameId) {
const ownership = { frameId, strength: 3 }; ownership = { frameId, strength: 3 };
ownershipByNodeId.set(node.nodeId, ownership); ownershipByNodeId.set(current.nodeId, ownership);
return ownership; break;
} }
path.push(current);
current = current.parentId ? nodeById.get(current.parentId) : undefined;
} }
if (node.parentId && !resolving.has(node.nodeId)) { for (let i = path.length - 1; i >= 0; i--) {
const parent = nodeById.get(node.parentId); if (work++ % 256 === 0) await captureCheckpoint(signal);
if (parent) { ownership = { frameId: ownership.frameId, strength: Math.min(2, ownership.strength) };
resolving.add(node.nodeId); ownershipByNodeId.set(path[i].nodeId, ownership);
const inherited = resolveOwnership(parent);
resolving.delete(node.nodeId);
const ownership = {
frameId: inherited.frameId,
strength: Math.min(2, inherited.strength),
};
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
}
} }
const ownership = { frameId: batch.frame.frameId, strength: 1 }; if (!ownershipByNodeId.has(node.nodeId)) ownershipByNodeId.set(node.nodeId, ownership);
ownershipByNodeId.set(node.nodeId, ownership);
return ownership; return ownership;
}; };
for (const node of batch.nodes) { for (const node of batch.nodes) {
const ownership = resolveOwnership(node); if (work++ % 256 === 0) await captureCheckpoint(signal);
const ownership = await resolveOwnership(node);
if (!ownership.frameId) continue;
const key = targetNodeKey(batch.frame.target, node.nodeId); const key = targetNodeKey(batch.frame.target, node.nodeId);
const existing = candidates.get(key); const existing = candidates.get(key);
if (!existing || ownership.strength > existing.ownership.strength) { if (!existing || ownership.strength > existing.ownership.strength) {
@@ -144,6 +162,7 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
); );
const axNodesByFrame = new Map<string, T[]>(); const axNodesByFrame = new Map<string, T[]>();
for (const candidate of candidates.values()) { for (const candidate of candidates.values()) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
const { node, target, ownership } = candidate; const { node, target, ownership } = candidate;
const parentFrameId = node.parentId const parentFrameId = node.parentId
? ownershipByTargetNode.get(targetNodeKey(target, node.parentId)) ? ownershipByTargetNode.get(targetNodeKey(target, node.parentId))
+233
View File
@@ -0,0 +1,233 @@
import type { Viewport } from "@browser-skill/vom";
import { type CdpFrame, type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
import { type GeometryProjection, projectRectToViewport } from "../geometry";
import {
type FrameProjectionState,
projectSnapshotRect,
snapshotViewportRect,
} from "../geometry/coordinate-types";
import { cssViewport, GeometryContext } from "../geometry/frame-context";
import type { CapturedNode } from "./facts";
import {
buildDocumentIndex,
type CaptureIssue,
captureCheckpoint,
type DocumentIndex,
isCaptureAbort,
type NodeFacts,
throwCaptureAborted,
} from "./facts";
import {
decodeDocument,
type SnapshotDocument,
type SnapshotReply,
snapshotFrameId,
} from "./snapshot";
export interface NormalizedDocument {
index: DocumentIndex;
frame: CdpFrame;
domNodes: CapturedNode[];
}
/** One target's snapshot. Source ownership is checked before interpreting any
* coordinates. Snapshot and live quad coordinates are deliberately not conflated. */
export async function normalizeSnapshot(
snapshot: SnapshotReply,
target: CdpTarget,
frames: readonly CdpFrame[],
geometry: GeometryContext,
issues: CaptureIssue[],
signal?: AbortSignal,
rootFrameId?: string,
): Promise<NormalizedDocument[]> {
const strings = snapshot.strings ?? [];
const raw = snapshot.documents ?? [];
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
const sources = new Map<string, SnapshotDocument>();
for (const doc of raw) {
const id = snapshotFrameId(doc, strings);
if (!id || !frameById.has(id) || sources.has(id)) {
issues.push({
target,
frameId: id,
stage: "ownership",
reason: "frame-ownership-unresolved",
});
continue;
}
sources.set(id, doc);
}
let viewport: Viewport = { width: 0, height: 0 };
let scrollX = 0,
scrollY = 0;
try {
const measured = cssViewport(await geometry.layoutMetrics(target));
viewport = { width: measured.width, height: measured.height };
scrollX = measured.scrollX;
scrollY = measured.scrollY;
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
const children = new Map<string, CdpFrame[]>();
const pending: CdpFrame[] = [];
for (const frame of frames) {
if (!frame.parentFrameId || !frameById.has(frame.parentFrameId)) pending.push(frame);
else {
const siblings = children.get(frame.parentFrameId);
if (siblings) siblings.push(frame);
else children.set(frame.parentFrameId, [frame]);
}
}
const projections = new Map<string, FrameProjectionState | null>();
const result: NormalizedDocument[] = [];
let targetProjection: GeometryProjection | null = null;
const targetRoot = rootFrameId ? frameById.get(rootFrameId) : undefined;
if (target.sessionId && targetRoot) {
try {
const known = await geometry.frame(targetRoot.frameId);
if (known && cdpTargetKey(known.target) === cdpTargetKey(target))
targetProjection = await geometry.targetProjection(targetRoot.frameId);
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
}
const visited = new Set<string>();
const remaining = frames[Symbol.iterator]();
for (let cursor = 0; visited.size < frames.length; cursor++) {
if (cursor >= pending.length) {
let next = remaining.next();
while (!next.done && visited.has(next.value.frameId)) next = remaining.next();
if (next.done) break;
pending.push(next.value);
}
if (cursor % 256 === 0) await captureCheckpoint(signal);
const frame = pending[cursor];
if (visited.has(frame.frameId)) continue;
visited.add(frame.frameId);
for (const child of children.get(frame.frameId) ?? []) pending.push(child);
const doc = sources.get(frame.frameId);
if (!doc) continue;
const source = { target, frameId: frame.frameId };
const state: FrameProjectionState | null =
frame.frameId === rootFrameId
? {
status: "available",
projection: { source, geometry: { sourceClips: [], edges: [], topViewport: viewport } },
}
: (projections.get(frame.frameId) ?? null);
if (!state && frame.frameId !== rootFrameId)
issues.push({
target,
frameId: frame.frameId,
stage: "ownership",
reason: "frame-ownership-unresolved",
});
projections.set(frame.frameId, state);
const projection = state?.status === "available" ? state.projection : null;
if (!projection || (target.sessionId && !targetProjection))
issues.push({
target,
frameId: frame.frameId,
stage: "geometry",
reason: "geometry-unavailable",
...(state && state.status !== "available" ? { projectionIssue: state } : {}),
});
// Only direct siblings share this pool. Keep normalization in its existing
// breadth-first order, independent of owner read completion order.
const siblings = (children.get(frame.frameId) ?? []).filter((child) =>
sources.has(child.frameId),
);
const edge = projection?.geometry.edges[0];
const clips = edge
? [edge.destinationQuad, ...(edge.destinationClips ?? [])]
: (projection?.geometry.sourceClips ?? []);
let next = 0;
let failure: unknown;
await Promise.all(
Array.from({ length: Math.min(4, siblings.length) }, async () => {
while (next < siblings.length && !signal?.aborted && !failure) {
const child = siblings[next++];
const childSource = { target, frameId: child.frameId };
if (!projection) {
projections.set(
child.frameId,
state && state.status !== "available"
? {
status: "blocked",
source: childSource,
cause: state.status === "blocked" ? state.cause : state,
}
: null,
);
continue;
}
if (child.ownerBackendNodeId === undefined) {
projections.set(child.frameId, null);
continue;
}
try {
projections.set(
child.frameId,
await geometry.snapshotProjection(
childSource,
child.ownerBackendNodeId,
clips,
viewport,
),
);
} catch (error) {
failure ??= error;
}
}
}),
);
// Join active reads and their cleanup before returning cancellation.
if (failure) throw failure;
throwCaptureAborted(signal);
const decoded = await decodeDocument(doc, strings, signal);
const nodes: NodeFacts[] = [];
for (let i = 0; i < decoded.nodes.length; i++) {
if (i % 256 === 0) await captureCheckpoint(signal);
const node = decoded.nodes[i];
const input = snapshotViewportRect(node.layout?.bounds ?? [], source, {
x: doc.scrollOffsetX ?? (frame.frameId === rootFrameId ? scrollX : 0),
y: doc.scrollOffsetY ?? (frame.frameId === rootFrameId ? scrollY : 0),
});
const local = input?.rect;
let rect = input && projection ? projectSnapshotRect(input, projection) : null;
if (rect && target.sessionId)
rect = targetProjection
? projectRectToViewport(
{ x: rect.x, y: rect.y, w: rect.width, h: rect.height },
targetProjection,
)
: null;
const styles = node.layout?.styles;
const visibility = styles?.visibility || "visible";
const opacity = styles?.opacity || "1";
nodes.push({
...node,
frameId: frame.frameId,
ownerFrameBackendNodeId: frame.ownerBackendNodeId ?? null,
localRect: local ? { x: local.x, y: local.y, w: local.width, h: local.height } : null,
rect: rect ? { x: rect.x, y: rect.y, w: rect.width, h: rect.height } : null,
rendered:
!!local &&
visibility !== "hidden" &&
visibility !== "collapse" &&
(Number.parseFloat(opacity) || 0) > 0,
});
}
const index = await buildDocumentIndex(nodes, signal);
const domNodes = nodes.filter((node) => !node.tag.startsWith("#"));
result.push({
index,
frame,
domNodes,
});
}
return result;
}
@@ -2,7 +2,10 @@ import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph"; import type { CdpTarget } from "@/browser-driver/frame-graph";
import { readRecordingDocumentIdentity } from "@/shared/recording-document-identity"; import { readRecordingDocumentIdentity } from "@/shared/recording-document-identity";
import type { CapturedSurfaceProbe } from "./capture"; import type { CapturedSurfaceProbe } from "./capture";
import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture"; import type {
FrameDocument as CapturedFrameDocument,
FrameOwnedAxNode as FrameAxNode,
} from "./frame-document";
/** Frame identity needed to resolve an `@eN` that lives in an iframe. */ /** Frame identity needed to resolve an `@eN` that lives in an iframe. */
export interface CaptureVomFrame { export interface CaptureVomFrame {
@@ -61,8 +61,9 @@ export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGrap
dom, dom,
sourceOrder: index, sourceOrder: index,
excluded: excluded:
document.frameId === rootFrameId && document.excludedBackendNodeIds?.has(dom.backendNodeId) === true ||
input.excludedBackendNodeIds?.has(dom.backendNodeId) === true, (document.frameId === rootFrameId &&
input.excludedBackendNodeIds?.has(dom.backendNodeId) === true),
}; };
nodes.set(id, node); nodes.set(id, node);
frameNodeIds.push(id); frameNodeIds.push(id);
@@ -90,9 +91,10 @@ export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGrap
ax, ax,
sourceOrder: document.domNodes.length + index, sourceOrder: document.domNodes.length + index,
excluded: excluded:
document.frameId === rootFrameId &&
typeof ax.backendDOMNodeId === "number" && typeof ax.backendDOMNodeId === "number" &&
input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true, (document.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true ||
(document.frameId === rootFrameId &&
input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true)),
}); });
frameNodeIds.push(id); frameNodeIds.push(id);
} }
+280
View File
@@ -0,0 +1,280 @@
import { type CdpFrame, type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
import type { CapturedNode } from "./facts";
import { captureCheckpoint, type DecodedDocument, type DecodedNode } from "./facts";
export const REQUESTED_STYLES = [
"position",
"pointer-events",
"cursor",
"visibility",
"opacity",
] as const;
const STYLE_COL = Object.fromEntries(
REQUESTED_STYLES.map((name, index) => [name, index]),
) as Record<(typeof REQUESTED_STYLES)[number], number>;
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
interface SparseArray {
index: number[];
value: number[];
}
interface RareBooleanData {
index: number[];
}
export interface SnapshotDocument {
frameId?: string | number;
documentURL?: number;
scrollOffsetX?: number;
scrollOffsetY?: number;
nodes?: {
parentIndex?: number[];
nodeName?: number[];
nodeType?: number[];
backendNodeId?: number[];
attributes?: number[][];
/**
* Per-node text value (index into strings), set for `#text` / CDATA nodes.
* Element nodes carry -1. Same length as `backendNodeId`.
*/
nodeValue?: number[];
/** Maps node array index → index into `documents[]` for frame content. */
contentDocumentIndex?: SparseArray;
inputValue?: SparseArray;
textValue?: SparseArray;
inputChecked?: RareBooleanData;
optionSelected?: RareBooleanData;
};
layout?: {
nodeIndex?: number[];
styles?: number[][];
bounds?: number[][];
paintOrders?: number[];
};
}
export function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
if (typeof document.frameId === "string") return document.frameId || undefined;
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
return undefined;
}
export interface SnapshotReply {
strings?: string[];
documents?: SnapshotDocument[];
}
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
}
function snapshotFormState(
value: string | undefined,
defaultValue: string,
hasDefaultValue: boolean,
sensitive: boolean,
): CapturedNode["formState"] {
if (sensitive) {
if (value === undefined && !hasDefaultValue) return undefined;
return (value ?? defaultValue) === "" ? "empty" : "filled";
}
if (value === undefined) return undefined;
if (value === "") return "empty";
return value === defaultValue ? "default" : "filled";
}
function str(strings: string[], idx: number | undefined): string {
if (idx === undefined || idx < 0) return "";
return strings[idx] ?? "";
}
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
const out = new Map<number, number>();
if (!sparse?.index || !sparse.value) return out;
for (let i = 0; i < sparse.index.length; i++) {
const nodeIndex = sparse.index[i];
const docIndex = sparse.value[i];
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
}
return out;
}
export async function decodeDocument(
doc: SnapshotDocument,
strings: string[],
signal?: AbortSignal,
): Promise<DecodedDocument> {
const dn = doc.nodes;
const dl = doc.layout;
if (!dn?.backendNodeId) {
return { nodes: [] };
}
const count = dn.backendNodeId.length;
const layoutByNode = new Map<number, number>();
for (let i = 0; i < (dl?.nodeIndex?.length ?? 0); i++) {
if (i % 256 === 0) await captureCheckpoint(signal);
layoutByNode.set(dl!.nodeIndex![i], i);
}
const inputValues = sparseIndexMap(dn.inputValue);
const textValues = sparseIndexMap(dn.textValue);
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
// Collect visible text from #text child nodes so element CapturedNodes
// carry a textContent value usable as a button/link label fallback.
// nodeValue is a parallel array: string index for text nodes, -1 otherwise.
const nodeTextContent = new Map<number, string[]>();
if (dn.nodeValue) {
for (let n = 0; n < count; n++) {
if (n % 256 === 0) await captureCheckpoint(signal);
const nvIdx = dn.nodeValue[n] ?? -1;
if (nvIdx < 0) continue;
const text = str(strings, nvIdx).trim();
if (!text) continue;
const parentIdx = dn.parentIndex?.[n] ?? -1;
if (parentIdx >= 0) {
const existing = nodeTextContent.get(parentIdx);
if (existing) existing.push(text);
else nodeTextContent.set(parentIdx, [text]);
}
}
}
// Decode fields without assigning a coordinate projection or semantic policy.
const nodes: DecodedNode[] = [];
for (let n = 0; n < count; n++) {
if (n % 256 === 0) await captureCheckpoint(signal);
const backendNodeId = dn.backendNodeId[n];
const parentIdx = dn.parentIndex?.[n] ?? -1;
const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null;
const tag = str(strings, dn.nodeName?.[n]).toLowerCase();
const attrs: Record<string, string> = {};
const pairs = dn.attributes?.[n] ?? [];
for (let a = 0; a + 1 < pairs.length; a += 2) {
attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]);
}
const li = layoutByNode.get(n);
const styleRow = li === undefined ? [] : (dl?.styles?.[li] ?? []);
const styles: Record<string, string> = {};
for (const name of REQUESTED_STYLES) styles[name] = str(strings, styleRow[STYLE_COL[name]]);
const layout =
li === undefined
? undefined
: {
boundsSpace: "snapshot-document-css" as const,
bounds: dl?.bounds?.[li],
styles,
};
const textContent = nodeTextContent.get(n)?.join(" ");
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
const rawFormValue =
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
const sensitive = isSensitiveFormControl({ tag, attrs });
const formDefaultValue = attrs.value ?? "";
const formValue = sensitive ? undefined : rawFormValue;
const formState = snapshotFormState(
rawFormValue,
formDefaultValue,
Object.prototype.hasOwnProperty.call(attrs, "value"),
sensitive,
);
if (sensitive) delete attrs.value;
nodes.push({
backendNodeId,
nodeType: dn.nodeType?.[n],
parentBackendNodeId,
tag,
attrs,
layout,
paintOrder: li === undefined ? 0 : (dl?.paintOrders?.[li] ?? 0),
position: styles.position || "static",
pointerEvents: styles["pointer-events"] || "auto",
cursor: styles.cursor || "auto",
textContent,
...(formValue !== undefined ? { formValue } : {}),
...(tag === "input" || tag === "textarea"
? {
formPlaceholder: attrs.placeholder ?? "",
...(!sensitive ? { formDefaultValue } : {}),
...(formState ? { formState } : {}),
}
: {}),
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
});
}
return { nodes };
}
/** Snapshot membership and same-target edges are current response evidence.
* The earlier graph only supplies compatible metadata and cross-target edges. */
export function describeSnapshotFrames(
snapshot: SnapshotReply,
target: CdpTarget,
hintById: ReadonlyMap<string, CdpFrame>,
pageUrl?: string,
): { frames: CdpFrame[]; ids: Set<string>; rootFrameId?: string } {
const raw = snapshot.documents ?? [];
const strings = snapshot.strings ?? [];
const ids = new Set<string>();
const invalid = new Set<string>();
const frames = raw.map((doc) => {
const frameId = snapshotFrameId(doc, strings);
if (!frameId) return undefined;
if (ids.has(frameId)) invalid.add(frameId);
ids.add(frameId);
const hint = hintById.get(frameId);
const compatible =
hint && cdpTargetKey(hint.target) === cdpTargetKey(target) ? hint : undefined;
const url =
(doc.documentURL === undefined ? undefined : strings[doc.documentURL]) || compatible?.url;
return { frameId, target, ...(url ? { url } : {}) } as CdpFrame;
});
const root = frames[0];
if (root) {
const hint = hintById.get(root.frameId);
const parent = hint?.parentFrameId ? hintById.get(hint.parentFrameId) : undefined;
if (
target.sessionId &&
hint &&
cdpTargetKey(hint.target) === cdpTargetKey(target) &&
parent &&
cdpTargetKey(parent.target) !== cdpTargetKey(target)
) {
root.parentFrameId = parent.frameId;
root.ownerBackendNodeId = hint.ownerBackendNodeId;
}
if (!target.sessionId && !root.url && pageUrl) root.url = pageUrl;
}
const owned = new Set<string>();
for (let i = 0; i < raw.length; i++) {
const edges = raw[i].nodes?.contentDocumentIndex;
const parent = frames[i];
if (!parent || !edges) continue;
for (let e = 0; e < edges.index.length; e++) {
const child = frames[edges.value[e]];
const owner = raw[i].nodes?.backendNodeId?.[edges.index[e]];
if (!child || owner === undefined) continue;
if (child === root || child === parent || owned.has(child.frameId)) {
invalid.add(child.frameId);
continue;
}
owned.add(child.frameId);
child.parentFrameId = parent.frameId;
child.ownerBackendNodeId = owner;
}
}
return {
frames: frames.filter((frame): frame is CdpFrame => !!frame && !invalid.has(frame.frameId)),
ids,
rootFrameId: root?.frameId,
};
}
@@ -47,7 +47,6 @@
"client": { "client": {
"platform": "web", "platform": "web",
"inject": [ "inject": [
"@deepseek-ai/dsh-client-runtime",
"@deepseek-ai/dsh-client-ui-tool", "@deepseek-ai/dsh-client-ui-tool",
"@deepseek-ai/dsh-client-ui-layout" "@deepseek-ai/dsh-client-ui-layout"
], ],
@@ -69,7 +68,6 @@
"peerDependencies": { "peerDependencies": {
"@deepseek-ai/cordis": "^4.0.1", "@deepseek-ai/cordis": "^4.0.1",
"@deepseek-ai/dsh-attachment": "^0.1.0-rc.6", "@deepseek-ai/dsh-attachment": "^0.1.0-rc.6",
"@deepseek-ai/dsh-client-runtime": "^0.1.0-rc.6",
"@deepseek-ai/dsh-client-ui-attachment": "^0.1.0-rc.6", "@deepseek-ai/dsh-client-ui-attachment": "^0.1.0-rc.6",
"@deepseek-ai/dsh-client-ui-primitives": "^0.1.0-rc.6", "@deepseek-ai/dsh-client-ui-primitives": "^0.1.0-rc.6",
"@deepseek-ai/dsh-client-ui-tool": "^0.1.0-rc.6", "@deepseek-ai/dsh-client-ui-tool": "^0.1.0-rc.6",
@@ -82,9 +80,6 @@
"@deepseek-ai/dsh-attachment": { "@deepseek-ai/dsh-attachment": {
"optional": true "optional": true
}, },
"@deepseek-ai/dsh-client-runtime": {
"optional": true
},
"@deepseek-ai/dsh-client-ui-attachment": { "@deepseek-ai/dsh-client-ui-attachment": {
"optional": true "optional": true
}, },
@@ -7,6 +7,7 @@
*/ */
import type { ImageAttachmentRef } from "@deepseek-ai/dsh-attachment"; import type { ImageAttachmentRef } from "@deepseek-ai/dsh-attachment";
// Development-only types for the injected services; newer hosts no longer ship this package.
import type { ClientContext, ISessions, SessionId } from "@deepseek-ai/dsh-client-runtime/client"; import type { ClientContext, ISessions, SessionId } from "@deepseek-ai/dsh-client-runtime/client";
// Type-only: pulls the 'shell.overlay' SlotMap merge into scope. // Type-only: pulls the 'shell.overlay' SlotMap merge into scope.
import type {} from "@deepseek-ai/dsh-client-ui-layout/client"; import type {} from "@deepseek-ai/dsh-client-ui-layout/client";
@@ -25,7 +25,6 @@ const CLIENT_EXTERNALS: readonly string[] = [
"@deepseek-ai/dsh-client-ui-primitives", "@deepseek-ai/dsh-client-ui-primitives",
"@deepseek-ai/dsh-client-ui-attachment", "@deepseek-ai/dsh-client-ui-attachment",
"@deepseek-ai/dsh-client-schema-form", "@deepseek-ai/dsh-client-schema-form",
"@deepseek-ai/dsh-client-runtime/client",
]; ];
/** Wire/type layers with no shared runtime identity: safe to inline. */ /** Wire/type layers with no shared runtime identity: safe to inline. */