Compare commits

..
Author SHA1 Message Date
drakezhang 4834d8b5fa test(geometry): identify the regression browser attachment 2026-09-09 16:11:39 +08:00
Ljy-0827 b4006d5b85 perf(observation): validate document identity against snapshot roots 2026-09-09 16:09:09 +08:00
Ljy-0827 2d1e5c6d9d fix(observation): validate document consistency with bounded identity checks 2026-09-09 16:09:09 +08:00
drakezhang 282e6f5623 fix(observation): carry snapshot units through capture coordination 2026-09-09 16:05:42 +08:00
Ljy-0827 bd68c293b6 perf(observation): collect frame AX trees with bounded concurrency 2026-09-09 15:58:56 +08:00
Ljy-0827 35a20582ad refactor(observation): separate capture coordination from document consistency 2026-09-09 15:58:56 +08:00
Ljy-0827 4e3365450a refactor(observation): integrate capture orchestration and document consistency 2026-09-09 15:58:56 +08:00
Ljy-0827 df5ed1a495 fix(observation): validate document consistency during capture 2026-09-09 15:58:40 +08:00
drakezhang eefbe03258 fix(observation): preserve snapshot unit normalization after document extraction 2026-09-09 15:58:28 +08:00
Ljy-0827 1d275f047c perf(observation): reduce document processing scheduling overhead 2026-09-09 15:54:41 +08:00
Ljy-0827 388b3ae2d5 refactor(observation): isolate document processing from capture orchestration 2026-09-09 15:54:41 +08:00
Ljy-0827 36d0d91419 fix(observation): preserve snapshot document provenance and URL metadata 2026-09-09 15:54:41 +08:00
Ljy-0827 bcb9beb4f8 refactor(observation): defer visual facts until visual discovery 2026-09-09 15:54:41 +08:00
Ljy-0827 34ccb96fd6 fix(observation): await capture task cleanup before propagating cancellation 2026-09-09 15:54:41 +08:00
Ljy-0827 8dfa097e55 refactor(observation): organize capture around document facts 2026-09-09 15:54:41 +08:00
25 changed files with 3747 additions and 2126 deletions
@@ -29,6 +29,24 @@ function fakeApi() {
}
describe("ChromiumCdp", () => {
it("changes attachment identity only across real debugger attachments", async () => {
const { api, onEvent, onDetach } = fakeApi();
const cdp = new ChromiumCdp(api);
expect(cdp.getAttachmentId(4)).toBeUndefined();
await cdp.ensureAttached(4);
const first = cdp.getAttachmentId(4);
expect(first).toBeTruthy();
await cdp.ensureAttached(4);
onEvent.fire({ tabId: 4 }, "Target.attachedToTarget", { sessionId: "child" });
expect(cdp.getAttachmentId(4)).toBe(first);
onDetach.fire({ tabId: 4 }, "target_closed");
expect(cdp.getAttachmentId(4)).toBeUndefined();
await cdp.ensureAttached(4);
expect(cdp.getAttachmentId(4)).not.toBe(first);
await cdp.detach(4);
expect(cdp.getAttachmentId(4)).toBeUndefined();
});
it("discovers multiple iframe targets and recursively routes nested OOPIF commands", async () => {
const { api, onEvent } = fakeApi();
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
@@ -164,6 +164,7 @@ async function settleBeforeDeadline(promises: Promise<void>[], deadline: number)
export class ChromiumCdp {
private readonly api: CdpDebuggerApi;
private readonly attachedTabs = new Set<number>();
private readonly attachmentIds = new Map<number, string>();
private readonly attachInFlight = new Map<number, Promise<void>>();
private readonly detachInFlight = new Map<number, Promise<void>>();
private readonly tabOwners = new Map<number, Set<string>>();
@@ -198,6 +199,11 @@ export class ChromiumCdp {
this.bindFrameTargetHandler();
}
/** Identity of this actual debugger attachment, independent of frame topology. */
getAttachmentId(tabId: number): string | undefined {
return this.attachmentIds.get(tabId);
}
/** Attach to `tabId` if we haven't already in this driver. */
async ensureAttached(tabId: number): Promise<void> {
// Returning a tab clears the cache before Chrome finishes detaching.
@@ -217,6 +223,7 @@ export class ChromiumCdp {
await this.enableConsoleDomains(tabId);
await this.enableNetworkDomainBestEffort(tabId);
this.attachedTabs.add(tabId);
this.attachmentIds.set(tabId, crypto.randomUUID());
await this.enableFrameDiscovery({ tabId }).catch((err) => {
console.debug("[bsk cdp] frame discovery unavailable", { tabId, err });
});
@@ -444,6 +451,7 @@ export class ChromiumCdp {
this.attachInFlight.delete(tabId);
if (!this.attachedTabs.has(tabId)) return;
this.attachedTabs.delete(tabId);
this.attachmentIds.delete(tabId);
this.clearDialogState(tabId);
this.clearConsoleState(tabId);
this.clearNetworkState(tabId);
@@ -502,6 +510,7 @@ export class ChromiumCdp {
this.attachInFlight.clear();
this.tabOwners.clear();
this.attachedTabs.clear();
this.attachmentIds.clear();
this.dialogBuffers.clear();
this.dialogSequences.clear();
this.consoleBuffers.clear();
@@ -848,6 +857,7 @@ export class ChromiumCdp {
const listener = (source: chrome.debugger.Debuggee, _reason: string) => {
if (typeof source.tabId === "number") {
this.attachedTabs.delete(source.tabId);
this.attachmentIds.delete(source.tabId);
this.attachInFlight.delete(source.tabId);
this.tabOwners.delete(source.tabId);
this.clearDialogState(source.tabId);
File diff suppressed because it is too large Load Diff
@@ -1,10 +1,8 @@
// @vitest-environment node
import { describe, expect, it } from "vitest";
import type { CdpFrame, CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
import { GeometryContext } from "../geometry/frame-context";
import type { CdpRunner } from "../shared";
import { captureViewModel } from "../vom/capture";
import { captureFrameData } from "../vom/frame-capture";
import { captureObservationFacts } from "../vom/capture-coordinator";
type Send = <T = Record<string, unknown>>(
method: string,
@@ -132,6 +130,7 @@ describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate c
return send(method, params, sessionFor(target));
},
getFrameGraph: async () => graph,
getAttachmentId: () => rootSession,
};
const oracles = new Map<string, Oracle>();
for (const frame of frames) {
@@ -180,14 +179,12 @@ describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate c
);
oracles.set(frame.frameId, reply.result.value);
}
const geometry = new GeometryContext(cdp, 1, graph);
const captured = await captureViewModel(cdp, 1, { geometry });
await captureFrameData(cdp, 1, captured, undefined, geometry);
expect(captured.frameGeometryIssues).toEqual([]);
const facts = await captureObservationFacts(cdp, 1);
expect(facts.issues).toEqual([]);
for (const frame of frames) {
const node = captured.frameNodes
?.get(frame.frameId)
?.find((node) => node.attrs.id === "probe");
const node = facts.documents
.find((doc) => doc.frame.frameId === frame.frameId)
?.domNodes.find((node) => node.attrs.id === "probe");
expect(node, `missing probe in ${names.get(frame.frameId) || "root"}`).toBeDefined();
const local = oracles.get(frame.frameId)!.probe;
const top = { ...local };
@@ -206,7 +203,7 @@ describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate c
frame: names.get(frame.frameId),
local,
top,
viewport: captured.viewport,
viewport: facts.viewport,
}),
).not.toBeNull();
for (const key of ["x", "y", "w", "h"] as const) {
+38 -225
View File
@@ -1,3 +1,4 @@
import type { CapturedSceneInput } from "./vom/facts";
// Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`,
// and semantic `tool.observe` (design §7). Each handler resolves the target
// tab (defaulting to the Agent Window's active tab when omitted) and
@@ -35,7 +36,6 @@ import { attachDialogs, markDialogCursor } from "./dialogs";
import { rpcError } from "./errors";
import { resolveNodeGeometry } from "./frame-geometry";
import { screenshotPageRect } from "./geometry/coordinate-types";
import { cssViewport, GeometryContext } from "./geometry/frame-context";
import {
type ChromeTabsApi,
enforceToolTargetScope,
@@ -49,15 +49,9 @@ import {
type ToolEffect,
} from "./shared";
import { resolveSnapshotRef } from "./snapshot-ref";
import {
type CapturedNode,
type CapturedSurfaceProbe,
type CapturedViewModel,
captureViewModel,
collectOverlayExcludedBackendIds,
probeHoverSurfaces,
} from "./vom/capture";
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
import { type CapturedNode, type CapturedSurfaceProbe, probeHoverSurfaces } from "./vom/capture";
import { captureObservationFacts, semanticCapture } from "./vom/capture-coordinator";
import type { FrameDocument as CapturedFrameDocument } from "./vom/frame-document";
import { withOverlayBypass } from "./vom/hover-perception";
import { probeTooltipNames } from "./vom/name-enrichment";
import {
@@ -531,7 +525,7 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A
function buildConditionalSurfaces(
nodes: VomNode[],
captured: CapturedViewModel,
captured: CapturedSceneInput,
probes: CapturedSurfaceProbe[],
): CondSurface[] {
if (probes.length === 0) return [];
@@ -661,161 +655,9 @@ export interface BuildVomSceneOptions {
export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>;
function legacyFrameDocuments(
axNodes: CdpAxNode[],
captured: CapturedViewModel,
pageUrl?: string,
rootTarget: VomFrameDocument["target"] = { tabId: 0 },
): VomFrameDocument[] {
const rootFrameId = captured.rootFrameId ?? "root";
const documents: VomFrameDocument[] = [
{
frameId: rootFrameId,
contextScopeId: rootFrameId,
target: rootTarget,
...(pageUrl ? { url: pageUrl } : {}),
axNodes: [],
domNodes: captured.nodes.map((node) => ({ ...node, frameId: node.frameId ?? rootFrameId })),
},
];
const pending = [...captured.iframeNodes.entries()];
let progress = true;
let nextSyntheticFrame = 1;
while (pending.length > 0 && progress) {
progress = false;
for (let index = pending.length - 1; index >= 0; index -= 1) {
const [ownerBackendNodeId, domNodes] = pending[index];
const parent = documents.find((document) =>
document.domNodes.some((node) => node.backendNodeId === ownerBackendNodeId),
);
if (!parent) continue;
const frameId =
domNodes.find((node) => node.frameId)?.frameId ?? `legacy-frame-${nextSyntheticFrame++}`;
documents.push({
frameId,
parentFrameId: parent.frameId,
ownerBackendNodeId,
contextScopeId: frameId,
target: parent.target,
axNodes: [],
domNodes: domNodes.map((node) => ({ ...node, frameId: node.frameId ?? frameId })),
});
pending.splice(index, 1);
progress = true;
}
}
const documentByFrameId = new Map(documents.map((document) => [document.frameId, document]));
const backendFrame = new Map<number, string>();
const childFrameByOwner = new Map<number, string>();
for (const document of documents) {
if (document.ownerBackendNodeId !== undefined) {
childFrameByOwner.set(document.ownerBackendNodeId, document.frameId);
}
for (const node of document.domNodes) backendFrame.set(node.backendNodeId, document.frameId);
}
const axById = new Map(axNodes.map((node) => [node.nodeId, node]));
const ownership = new Map<string, string>();
const resolving = new Set<string>();
const frameForAx = (node: CdpAxNode): string => {
const cached = ownership.get(node.nodeId);
if (cached) return cached;
let frameId: string | undefined;
if (node.frameId && documentByFrameId.has(node.frameId)) frameId = node.frameId;
if (!frameId && typeof node.backendDOMNodeId === "number") {
frameId = backendFrame.get(node.backendDOMNodeId);
}
if (!frameId && node.parentId && !resolving.has(node.nodeId)) {
const parent = axById.get(node.parentId);
if (parent) {
frameId =
typeof parent.backendDOMNodeId === "number"
? childFrameByOwner.get(parent.backendDOMNodeId)
: undefined;
if (!frameId) {
resolving.add(node.nodeId);
frameId = frameForAx(parent);
resolving.delete(node.nodeId);
}
}
}
frameId ??= rootFrameId;
ownership.set(node.nodeId, frameId);
return frameId;
};
for (const node of axNodes) frameForAx(node);
for (const node of axNodes) {
const frameId = ownership.get(node.nodeId) ?? rootFrameId;
const document = documentByFrameId.get(frameId) ?? documents[0];
document.axNodes.push({
...node,
frameId,
...(node.parentId && ownership.get(node.parentId) === frameId
? { parentId: node.parentId }
: { parentId: undefined }),
...(node.childIds
? { childIds: node.childIds.filter((childId) => ownership.get(childId) === frameId) }
: {}),
});
}
return documents;
}
function withLegacyBackendIds(scene: VomScene): VomScene {
const used = new Set<number>();
const idMap = new Map<number, number>();
let nextVirtualId = -1;
for (const node of scene.nodes) {
const preferred = node.backendNodeId;
const id = preferred !== undefined && !used.has(preferred) ? preferred : nextVirtualId--;
used.add(id);
idMap.set(node.id, id);
}
return {
...scene,
nodes: scene.nodes.map((node) => ({
...node,
id: idMap.get(node.id) as number,
parentId: node.parentId === null ? null : (idMap.get(node.parentId) ?? null),
...(node.domParentId !== undefined
? { domParentId: node.domParentId === null ? null : (idMap.get(node.domParentId) ?? null) }
: {}),
...(node.domAncestorIds
? { domAncestorIds: node.domAncestorIds.flatMap((id) => idMap.get(id) ?? []) }
: {}),
})),
...(scene.surfaces
? {
surfaces: scene.surfaces.map((surface) => ({
...surface,
triggerId: idMap.get(surface.triggerId) ?? surface.triggerId,
})),
}
: {}),
...(scene.activeScopeBlocks
? {
activeScopeBlocks: scene.activeScopeBlocks.map((block) => ({
...block,
triggerId: idMap.get(block.triggerId) ?? block.triggerId,
})),
}
: {}),
};
}
export function buildVomScene(
axNodes: CdpAxNode[],
captured: CapturedViewModel,
options: BuildVomSceneOptions = {},
): VomScene {
return withLegacyBackendIds(
buildFrameVomScene(legacyFrameDocuments(axNodes, captured, options.pageUrl), captured, options),
);
}
export function buildFrameVomScene(
documents: VomFrameDocument[],
captured: CapturedViewModel,
captured: CapturedSceneInput,
options: BuildVomSceneOptions = {},
): VomScene {
const scene = buildSemanticVomScene({
@@ -831,7 +673,7 @@ export function buildFrameVomScene(
function attachCapturedSceneAnnotations(
scene: VomScene,
documents: VomFrameDocument[],
captured: CapturedViewModel,
captured: CapturedSceneInput,
surfaceProbes: CapturedSurfaceProbe[],
): VomScene {
const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId);
@@ -981,45 +823,6 @@ export async function handleGetHtml(
}
}
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
}
async function fallbackCapturedViewModel(
cdp: CdpRunner,
tabId: number,
geometry: GeometryContext,
signal?: AbortSignal,
): Promise<CapturedViewModel> {
throwIfAborted(signal, "observation");
let viewport = { width: 0, height: 0 };
try {
const metrics = await geometry.layoutMetrics({ tabId });
throwIfAborted(signal, "observation");
const source = cssViewport(metrics);
viewport = { width: source.width, height: source.height };
} catch (error) {
if (isAbortError(error)) throw error;
}
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal);
throwIfAborted(signal, "observation");
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
}
async function captureForVom(
cdp: CdpRunner,
tabId: number,
options: CaptureVomObservationOptions,
geometry: GeometryContext,
): Promise<CapturedViewModel> {
try {
return await captureViewModel(cdp, tabId, { signal: options.signal, geometry });
} catch (error) {
if (isAbortError(error)) throw error;
return fallbackCapturedViewModel(cdp, tabId, geometry, options.signal);
}
}
export interface HoverProbeOutcome {
/** Whether any active hover was dispatched during this observation. */
performed: boolean;
@@ -1052,7 +855,7 @@ const NO_HOVER_PROBES: HoverProbeOutcome = {
async function runHoverProbes(
cdp: CdpRunner,
tabId: number,
captured: CapturedViewModel,
captured: CapturedSceneInput,
documents: VomFrameDocument[],
staticSemantics: ReturnType<typeof resolveSemanticGraph>,
options: CaptureVomObservationOptions,
@@ -1091,21 +894,9 @@ export async function captureVomObservation(
throwIfAborted(options.signal, "observation");
await cdp.ensureAttachedToUrl?.(tabId, url);
throwIfAborted(options.signal, "observation");
const geometry = new GeometryContext(cdp, tabId, undefined, options.signal);
const captured = await captureForVom(cdp, tabId, options, geometry);
const facts = await captureObservationFacts<CdpAxNode>(cdp, tabId, options.signal, url);
const { captured, documents: normalizedDocuments } = semanticCapture(facts);
throwIfAborted(options.signal, "observation");
const documents = await captureFrameData<CdpAxNode>(
cdp,
tabId,
captured,
options.signal,
geometry,
);
throwIfAborted(options.signal, "observation");
const normalizedDocuments =
documents.length === 1 && captured.iframeNodes.size > 0
? legacyFrameDocuments(documents[0].axNodes, captured, url, documents[0].target)
: documents;
const semanticGraph = buildSemanticGraph({
documents: normalizedDocuments,
viewport: captured.viewport,
@@ -1135,19 +926,41 @@ export async function captureVomObservation(
);
// Reserve space using the renderer's character-based token estimate. Like VOM
// headers, this integrity notice remains visible even under a tiny token budget.
const geometryNotice = captured.frameGeometryIssues?.length
? "@warning geometry incomplete: some page or frame content has no top-level coordinates.\n"
: "";
const notices: string[] = [];
if (facts.issues.some((issue) => issue.stage === "geometry"))
notices.push(
"@warning geometry incomplete: some page or frame content has no top-level coordinates.",
);
const incompleteStages = ["dom", "ax", "forms", "ownership"].filter((stage) =>
facts.issues.some((issue) => issue.stage === stage),
);
if (incompleteStages.length)
notices.push(
`@warning observation incomplete: some ${incompleteStages.join(", ")} data is unavailable or omitted.`,
);
if (
facts.issues.some(
(issue) => issue.stage === "identity" && issue.reason !== "identity-unverified",
)
)
notices.push(
"@warning observation incomplete: some documents were omitted because their identity changed or could not be revalidated.",
);
if (facts.issues.some((issue) => issue.reason === "identity-unverified"))
notices.push(
"@warning document identity unverified: some retained documents could not be checked for changes during capture.",
);
const captureNotice = notices.join("\n");
const rendered = renderVom(decoratedScene, {
maxDepth: options.maxDepth,
maxTokens:
!geometryNotice || options.maxTokens === undefined
!captureNotice || options.maxTokens === undefined
? options.maxTokens
: Math.max(0, options.maxTokens - Math.ceil(geometryNotice.length / 4)),
: Math.max(0, options.maxTokens - Math.ceil((captureNotice.length + 1) / 4)),
redactValues: options.redactValues,
activeRegionPolicy: options.activeRegionPolicy,
});
if (geometryNotice) rendered.text += `\n${geometryNotice.trimEnd()}`;
if (captureNotice) rendered.text += `\n${captureNotice}`;
throwIfAborted(options.signal, "observation");
return projectRecordSafeObservation({
rootFrameId: captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root",
+1
View File
@@ -55,6 +55,7 @@ export interface CdpRunner {
sendToTarget?<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T>;
detach?(tabId: number): Promise<void>;
getFrameGraph?(tabId: number): Promise<CdpFrameGraph>;
getAttachmentId?(tabId: number): string | undefined;
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
trackSessionTab?(sessionId: string, tabId: number): void;
releaseSessionTab?(sessionId: string, tabId: number): Promise<void>;
File diff suppressed because it is too large Load Diff
@@ -1,7 +1,7 @@
import { describe, expect, it, vi } from "vitest";
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
import type { CdpRunner } from "../../shared";
import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
import { collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
import { captureObservationFacts, semanticCapture } from "../capture-coordinator";
// Minimal but format-accurate captureSnapshot reply: a body with one
// fixed full-screen overlay div carrying a password input.
@@ -27,6 +27,7 @@ function fakeSnapshotReply() {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 2],
nodeType: [1, 1, 1, 1],
@@ -74,6 +75,7 @@ function hoverTriggerSnapshotReply() {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1],
nodeType: [1, 1, 1],
@@ -118,6 +120,7 @@ function twoHoverTriggerSnapshotReply() {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 1],
nodeType: [1, 1, 1, 1],
@@ -171,6 +174,7 @@ function nestedHoverTriggerSnapshotReply() {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 2, 3],
nodeType: [1, 1, 1, 1, 1],
@@ -248,13 +252,11 @@ function makeCdp(snapshot: unknown) {
};
}
describe("captureViewModel", () => {
describe("snapshot capture and hover", () => {
it("parses nodes, attrs, rects, paint order and styles", async () => {
const { nodes, viewport, iframeNodes } = await captureViewModel(
makeCdp(fakeSnapshotReply()),
4,
);
expect(iframeNodes.size).toBe(0);
const { nodes, viewport } = semanticCapture(
await captureObservationFacts(makeCdp(fakeSnapshotReply()), 4),
).captured;
expect(viewport).toEqual({ width: 1000, height: 800 });
const div = nodes.find((n) => n.backendNodeId === 12);
expect(div).toMatchObject({ tag: "div", position: "fixed", parentBackendNodeId: 11 });
@@ -266,35 +268,11 @@ describe("captureViewModel", () => {
it("never hovers the page while capturing", async () => {
const cdp = makeCdp(fakeSnapshotReply());
await captureViewModel(cdp, 4);
semanticCapture(await captureObservationFacts(cdp, 4)).captured;
expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything());
});
it("batch-enriches form controls without per-node object resolution", async () => {
const cdp = makeCdp(fakeSnapshotReply());
const result = await captureViewModel(cdp, 4);
const input = result.nodes.find((node) => node.backendNodeId === 13);
expect(input).toMatchObject({
formState: "filled",
formPlaceholder: "secret",
attrs: { type: "password" },
});
expect(input?.formValue).toBeUndefined();
expect(input?.formDefaultValue).toBeUndefined();
expect(cdp.send).toHaveBeenCalledWith(
4,
"Runtime.evaluate",
expect.objectContaining({
serializationOptions: expect.objectContaining({ serialization: "deep" }),
}),
);
expect(cdp.send).toHaveBeenCalledWith(4, "Runtime.releaseObjectGroup", expect.anything());
expect(cdp.send).not.toHaveBeenCalledWith(4, "DOM.resolveNode", expect.anything());
expect(cdp.send).not.toHaveBeenCalledWith(4, "Runtime.callFunctionOn", expect.anything());
});
it("runs hover surface probes when explicitly enabled", async () => {
let hoverStateCalls = 0;
const cdp = {
@@ -336,7 +314,7 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const captured = await captureViewModel(cdp, 4);
const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
expect(surfaceProbes).toEqual([
@@ -398,7 +376,7 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const captured = await captureViewModel(cdp, 4);
const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
expect(surfaceProbes).toEqual([
@@ -445,7 +423,7 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const captured = await captureViewModel(cdp, 4);
const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
await probeHoverSurfaces(cdp, 4, captured.nodes);
expect(hoverMoves).toBe(1);
@@ -477,6 +455,7 @@ describe("captureViewModel", () => {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 0, 3, 4],
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
@@ -507,7 +486,7 @@ describe("captureViewModel", () => {
],
};
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
const ids = nodes.map((n) => n.backendNodeId);
// Real page nodes survive.
expect(ids).toContain(12);
@@ -538,6 +517,7 @@ describe("captureViewModel", () => {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 0, 3, 4],
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
@@ -568,7 +548,9 @@ describe("captureViewModel", () => {
],
};
const { excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
const { excludedBackendNodeIds } = semanticCapture(
await captureObservationFacts(makeCdp(snapshot), 4),
).captured;
expect(excludedBackendNodeIds).toEqual(new Set([13, 14, 15]));
});
@@ -589,6 +571,7 @@ describe("captureViewModel", () => {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i(OVERLAY_HOST_NAME)],
@@ -611,7 +594,9 @@ describe("captureViewModel", () => {
],
};
const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
const { nodes, excludedBackendNodeIds } = semanticCapture(
await captureObservationFacts(makeCdp(snapshot), 4),
).captured;
expect(nodes.map((n) => n.backendNodeId)).not.toContain(13);
expect(excludedBackendNodeIds).toEqual(new Set([13]));
});
@@ -634,6 +619,7 @@ describe("captureViewModel", () => {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("div")],
@@ -656,7 +642,9 @@ describe("captureViewModel", () => {
],
};
const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
const { nodes, excludedBackendNodeIds } = semanticCapture(
await captureObservationFacts(makeCdp(snapshot), 4),
).captured;
expect(nodes.map((n) => n.backendNodeId)).not.toContain(13);
expect(excludedBackendNodeIds).toEqual(new Set([13]));
});
@@ -679,6 +667,7 @@ describe("captureViewModel", () => {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("div")],
@@ -701,7 +690,7 @@ describe("captureViewModel", () => {
},
],
};
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
expect(nodes.find((n) => n.backendNodeId === 12)?.cursor).toBe("pointer");
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
});
@@ -713,6 +702,7 @@ describe("captureViewModel", () => {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 1, 1],
nodeName: [i("html"), i("body"), i("button"), i("button"), i("button")],
@@ -739,7 +729,7 @@ describe("captureViewModel", () => {
],
};
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
expect(nodes.find((node) => node.backendNodeId === 12)?.rendered).toBe(true);
expect(nodes.find((node) => node.backendNodeId === 13)?.rendered).toBe(false);
@@ -761,51 +751,12 @@ describe("captureViewModel", () => {
throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { nodes } = await captureViewModel(cdp, 4);
const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
const div = nodes.find((n) => n.backendNodeId === 12);
expect(div?.localRect?.y).toBe(-200);
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
});
it("normalizes snapshot-owned scroll before using a stale CSS metrics fallback", async () => {
const snapshot = {
strings: ["html", "button"],
documents: [
{
frameId: "root",
scrollOffsetX: 40,
scrollOffsetY: 200,
nodes: {
parentIndex: [-1, 0],
nodeName: [0, 1],
backendNodeId: [10, 11],
attributes: [[], []],
},
layout: { nodeIndex: [1], bounds: [[200, 800, 240, 80]] },
},
],
};
const cdp: CdpRunner = {
send: vi.fn(async (_tab: number, method: string) => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 2000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 999, pageY: 999 },
};
return {};
}) as CdpRunner["send"],
};
const captured = await captureViewModel(cdp, 4);
expect(captured.nodes.find((node) => node.backendNodeId === 11)).toMatchObject({
localRect: { x: 80, y: 300, w: 120, h: 40 },
rect: { x: 80, y: 300, w: 120, h: 40 },
rendered: true,
});
expect(captured.frameGeometryIssues).toEqual([]);
});
it("normalizes raw snapshot bounds before viewport clipping", async () => {
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
const i = (s: string) => S.indexOf(s);
@@ -813,6 +764,7 @@ describe("captureViewModel", () => {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("div")],
@@ -849,7 +801,7 @@ describe("captureViewModel", () => {
throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { nodes } = await captureViewModel(cdp, 4);
const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
expect(nodes.find((n) => n.backendNodeId === 12)?.rect).toEqual({
x: 0,
y: 0,
@@ -885,6 +837,7 @@ describe("captureViewModel", () => {
strings: S,
documents: [
{
frameId: "root",
scrollOffsetX: 0,
scrollOffsetY: 0,
nodes: {
@@ -925,7 +878,7 @@ describe("captureViewModel", () => {
throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { nodes } = await captureViewModel(cdp, 4);
const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
const btn = nodes.find((n) => n.backendNodeId === 12);
expect(btn?.textContent).toBe("登录");
const link = nodes.find((n) => n.backendNodeId === 14);
@@ -935,7 +888,7 @@ describe("captureViewModel", () => {
expect(body?.textContent).toBeUndefined();
});
it("parses iframe sub-documents and returns iframeNodes keyed by iframe backendNodeId", async () => {
it("retains explicit iframe document ownership and projected nodes", async () => {
const S = [
"html",
"body",
@@ -1023,30 +976,21 @@ describe("captureViewModel", () => {
throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const {
nodes,
iframeNodes,
frameNodes,
frameOwnerBackendNodeIds,
frameParentIds,
rootFrameId,
} = await captureViewModel(cdp, 4);
// Main frame has 4 nodes; iframe is included
expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
// iframeNodes keyed by the <iframe> element's backendNodeId=13
expect(iframeNodes.size).toBe(1);
expect(iframeNodes.has(13)).toBe(true);
const subNodes = iframeNodes.get(13)!;
const facts = await captureObservationFacts(cdp, 4);
const root = facts.documents.find((doc) => doc.frame.frameId === "main-frame")!;
const child = facts.documents.find((doc) => doc.frame.frameId === "child-frame")!;
expect(facts.documents).toHaveLength(2);
expect(root.domNodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
const subNodes = child.domNodes;
const input = subNodes.find((n) => n.backendNodeId === 101);
expect(input?.tag).toBe("input");
expect(input?.attrs.type).toBe("text");
expect(input?.ownerFrameBackendNodeId).toBe(13);
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
expect(rootFrameId).toBe("main-frame");
expect(frameNodes?.get("child-frame")).toBe(subNodes);
expect(frameOwnerBackendNodeIds?.get("child-frame")).toBe(13);
expect(frameParentIds?.get("child-frame")).toBe("main-frame");
expect(facts.rootFrameId).toBe("main-frame");
expect(child.frame.ownerBackendNodeId).toBe(13);
expect(child.frame.parentFrameId).toBe("main-frame");
});
it("projects iframe documents independently of owner snapshot geometry", async () => {
@@ -1067,6 +1011,7 @@ describe("captureViewModel", () => {
strings: S,
documents: [
{
frameId: "root",
scrollOffsetX: 0,
scrollOffsetY: 0,
nodes: {
@@ -1084,6 +1029,7 @@ describe("captureViewModel", () => {
},
},
{
frameId: "child",
scrollOffsetX: 0,
scrollOffsetY: 0,
nodes: {
@@ -1122,8 +1068,10 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { iframeNodes } = await captureViewModel(cdp, 4);
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
const { documents } = await captureObservationFacts(cdp, 4);
const input = documents
.find((doc) => doc.frame.frameId === "child")
?.domNodes?.find((n) => n.backendNodeId === 101);
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
expect(input?.rect).toEqual({ x: 120, y: 320, w: 120, h: 60 });
@@ -1147,6 +1095,7 @@ describe("captureViewModel", () => {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("iframe")],
@@ -1168,6 +1117,9 @@ describe("captureViewModel", () => {
},
},
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "child",
nodes: {
parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("iframe")],
@@ -1175,8 +1127,6 @@ describe("captureViewModel", () => {
attributes: [[], [], []],
contentDocumentIndex: { index: [2], value: [2] },
},
scrollOffsetX: 0,
scrollOffsetY: 0,
layout: {
nodeIndex: [1, 2],
styles: [
@@ -1191,14 +1141,15 @@ describe("captureViewModel", () => {
},
},
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: "nested",
nodes: {
parentIndex: [-1, 0],
nodeName: [i("body"), i("input")],
backendNodeId: [30, 31],
attributes: [[], [i("type"), i("text")]],
},
scrollOffsetX: 0,
scrollOffsetY: 0,
layout: {
nodeIndex: [1],
styles: [[i("static"), i("auto")]],
@@ -1250,9 +1201,13 @@ describe("captureViewModel", () => {
) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { iframeNodes } = await captureViewModel(cdp, 4);
const nestedIframe = iframeNodes.get(13)?.find((n) => n.backendNodeId === 23);
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31);
const { documents } = await captureObservationFacts(cdp, 4);
const nestedIframe = documents
.find((doc) => doc.frame.frameId === "child")
?.domNodes?.find((n) => n.backendNodeId === 23);
const input = documents
.find((doc) => doc.frame.frameId === "nested")
?.domNodes?.find((n) => n.backendNodeId === 31);
expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 });
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
@@ -1268,6 +1223,7 @@ describe("captureViewModel", () => {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("div")],
@@ -1304,7 +1260,7 @@ describe("captureViewModel", () => {
throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { nodes } = await captureViewModel(cdp, 4);
const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 / 2 - 100);
});
@@ -1330,200 +1286,3 @@ describe("captureViewModel", () => {
expect(excluded).toEqual(new Set([200, 201, 202]));
});
});
// Six sibling frames and one nested frame exercise scheduling through the real capture entry.
function siblingCaptureFixture(
beforeReply: (method: string, params: Record<string, unknown>) => Promise<void> = async () => {},
) {
const document = (id: number, owners: number[], childIndexes: number[]) => ({
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: `frame-${id}`,
nodes: {
parentIndex: [-1, ...owners.map(() => 0)],
nodeName: [0, ...owners.map(() => 1)],
backendNodeId: [1000 + id, ...owners],
attributes: [[], ...owners.map(() => [])],
contentDocumentIndex: { index: owners.map((_, i) => i + 1), value: childIndexes },
},
layout: {
nodeIndex: [0, ...owners.map((_, i) => i + 1)],
bounds: [[0, 0, 200, 100], ...owners.map(() => [0, 0, 200, 100])],
},
});
const snapshot = {
strings: ["body", "iframe"],
documents: [
document(0, [100, 101, 102, 103, 104, 105], [1, 2, 3, 4, 5, 6]),
document(1, [200], [7]),
...Array.from({ length: 6 }, (_, i) => document(i + 2, [], [])),
],
};
let active = 0;
let peak = 0;
const send = vi.fn(async (_tabId: number, method: string, params: object = {}) => {
const args = params as Record<string, unknown>;
active++;
peak = Math.max(peak, active);
try {
await beforeReply(method, args);
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
if (method === "DOM.getBoxModel")
return { model: { content: [0, 0, 200, 0, 200, 100, 0, 100] } };
if (method === "DOM.resolveNode") return { object: { objectId: String(args.backendNodeId) } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 200, height: 100 } } };
return {};
} finally {
active--;
}
});
return {
cdp: { send: send as CdpRunner["send"] },
send,
peak: () => peak,
active: () => active,
measured: () =>
send.mock.calls
.filter(([, method]) => method === "DOM.getBoxModel")
.map(([, , params]) => (params as { backendNodeId: number }).backendNodeId),
};
}
describe("sibling frame measurement scheduling", () => {
it("batches same-target owner sizes without changing projected results", async () => {
const fixture = siblingCaptureFixture();
const original = fixture.cdp.send;
const methods: string[] = [];
fixture.cdp.send = async (tabId, method, params) => {
methods.push(method);
if (method === "Runtime.evaluate")
return {
result: {
deepSerializedValue: {
type: "array",
value: [100, 101, 102, 103, 104, 105, 200].map((backendNodeId) => ({
type: "array",
value: [
{ type: "node", value: { backendNodeId } },
{ type: "string", value: JSON.stringify({ width: 200, height: 100 }) },
],
})),
},
},
} as never;
return original(tabId, method, params);
};
const captured = await captureViewModel(fixture.cdp, 4);
expect(captured.frameGeometryIssues).toEqual([]);
expect([...captured.iframeNodes.keys()]).toEqual([100, 200, 101, 102, 103, 104, 105]);
expect(captured.iframeNodes.get(200)?.[0].rect).toEqual({ x: 0, y: 0, w: 200, h: 100 });
expect(methods.filter((method) => method === "DOM.getBoxModel")).toHaveLength(7);
expect(methods.filter((method) => method === "Runtime.evaluate")).toHaveLength(1);
expect(methods.filter((method) => method === "Runtime.releaseObjectGroup")).toHaveLength(1);
expect(methods).not.toContain("DOM.resolveNode");
expect(methods).not.toContain("Runtime.callFunctionOn");
});
it("bounds concurrent reads, fills free slots and preserves depth-first output despite reordered replies", async () => {
const pending = new Map<number, () => void>();
const fixture = siblingCaptureFixture(async (method, params) => {
if (method === "DOM.getBoxModel" && Number(params.backendNodeId) < 200)
await new Promise<void>((resolve) => pending.set(Number(params.backendNodeId), resolve));
});
const capture = captureViewModel(fixture.cdp, 4);
await vi.waitFor(() => expect(pending.size).toBe(4));
expect(fixture.measured()).toEqual([100, 101, 102, 103]);
pending.get(103)!();
await vi.waitFor(() => expect(pending.has(104)).toBe(true));
pending.get(104)!();
await vi.waitFor(() => expect(pending.has(105)).toBe(true));
expect(fixture.measured()).not.toContain(200);
for (const id of [105, 102, 101, 100]) pending.get(id)!();
const captured = await capture;
expect(captured.frameGeometryIssues).toEqual([]);
expect(fixture.peak()).toBe(4);
expect(fixture.active()).toBe(0);
expect(fixture.measured()).toEqual([100, 101, 102, 103, 104, 105, 200]);
expect([...captured.iframeNodes.keys()]).toEqual([100, 200, 101, 102, 103, 104, 105]);
expect(captured.iframeNodes.get(200)?.[0].rect).toEqual({ x: 0, y: 0, w: 200, h: 100 });
for (const method of [
"DOM.getBoxModel",
"DOM.resolveNode",
"Runtime.callFunctionOn",
"Runtime.releaseObject",
])
expect(fixture.send.mock.calls.filter(([, name]) => name === method)).toHaveLength(7);
});
it("does not measure descendants of a failed owner and retains sibling geometry and local nodes", async () => {
const fixture = siblingCaptureFixture(async (method, params) => {
if (method === "DOM.resolveNode" && params.backendNodeId === 100)
throw new Error("owner replaced");
});
const captured = await captureViewModel(fixture.cdp, 4);
expect(captured.frameGeometryIssues).toEqual([
{
status: "unavailable",
source: { target: { tabId: 4 }, frameId: "frame-1" },
ownerBackendNodeId: 100,
},
{
status: "blocked",
source: { target: { tabId: 4 }, frameId: "frame-7" },
cause: captured.frameGeometryIssues?.[0],
},
]);
expect(fixture.measured()).not.toContain(200);
expect(captured.iframeNodes.get(200)?.[0]).toMatchObject({
tag: "body",
rect: null,
localRect: { x: 0, y: 0, w: 200, h: 100 },
});
expect(captured.iframeNodes.get(101)?.[0].rect).toEqual({ x: 0, y: 0, w: 200, h: 100 });
expect([...captured.iframeNodes.keys()]).toEqual([100, 200, 101, 102, 103, 104, 105]);
});
it("stops scheduling on cancellation and waits for resolved objects to be released", async () => {
const controller = new AbortController();
const resolutions = new Map<string, () => void>();
const releases = new Map<string, () => void>();
const fixture = siblingCaptureFixture(async (method, params) => {
if (method === "DOM.resolveNode")
await new Promise<void>((resolve) =>
resolutions.set(String(params.backendNodeId), resolve),
);
if (method === "Runtime.releaseObject")
await new Promise<void>((resolve) => releases.set(String(params.objectId), resolve));
});
let settled = false;
const capture = captureViewModel(fixture.cdp, 4, { signal: controller.signal });
const rejected = expect(capture).rejects.toMatchObject({ name: "AbortError" });
void capture.then(
() => {
settled = true;
},
() => {
settled = true;
},
);
await vi.waitFor(() => expect(resolutions.size).toBe(4));
controller.abort();
for (const resolve of resolutions.values()) resolve();
await vi.waitFor(() => expect(releases.size).toBe(4));
expect(settled).toBe(false);
expect(fixture.measured()).toEqual([100, 101, 102, 103]);
for (const release of releases.values()) release();
await rejected;
expect(fixture.active()).toBe(0);
expect(fixture.send.mock.calls.some(([, method]) => method === "Runtime.callFunctionOn")).toBe(
false,
);
});
});
@@ -0,0 +1,127 @@
import { describe, expect, it, vi } from "vitest";
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import { createCaptureCheckpoint } from "../capture-abort";
import { buildDocumentIndex, type DecodedNode } from "../facts";
import { decodeDocument, REQUESTED_STYLES } from "../snapshot";
function node(id: number, parent: number | null): DecodedNode {
return {
backendNodeId: id,
parentBackendNodeId: parent,
tag: "div",
attrs: {},
paintOrder: 0,
position: "static",
pointerEvents: "auto",
};
}
describe("document facts", () => {
it.each([
1_000, 10_000, 100_000,
])("indexes deep and wide inputs of %i without copying ancestors", async (count) => {
for (const wide of [false, true]) {
let reads = 0;
const input = Array.from({ length: count }, (_, i) => {
const item = node(i, i ? (wide ? 0 : i - 1) : null);
if (i === 0) item.attrs[OVERLAY_HOST_MARKER_ATTR] = "";
Object.defineProperty(item, "parentBackendNodeId", {
get: () => {
reads++;
return i ? (wide ? 0 : i - 1) : null;
},
});
return item;
}).reverse(); // parent need not precede child
const index = await buildDocumentIndex(input);
expect(index.nodes.size).toBe(count);
expect(index.excludedBackendNodeIds.size).toBe(count);
expect(reads).toBeLessThanOrEqual(count * 8);
expect(index.nodes.get(count - 1)).toBe(input[0]);
}
});
it("handles cycles and orphans and propagates overlay through shadow roots", async () => {
const host = { ...node(1, null), attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } };
const input = [
node(3, 2),
{ ...node(2, 1), tag: "#document-fragment" },
host,
node(4, 99),
node(5, 6),
node(6, 5),
node(7, 6),
];
const index = await buildDocumentIndex(input);
expect([...index.excludedBackendNodeIds].sort()).toEqual([1, 2, 3]);
expect(index.nodes.size).toBe(input.length);
});
it("does not propagate a descendant overlay backwards into a cycle", async () => {
const overlay = { ...node(3, 1), attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } };
const index = await buildDocumentIndex([overlay, node(1, 2), node(2, 1)]);
expect([...index.excludedBackendNodeIds]).toEqual([3]);
});
it("decodes current observation bounds, styles and attributes without projecting coordinates", async () => {
const strings = ["div", "aria-hidden", "true", "inert", "", "visible", "0", "static", "auto"];
const styles = REQUESTED_STYLES.map((style) =>
style === "visibility" ? 5 : style === "opacity" ? 6 : style === "position" ? 7 : 8,
);
const { nodes } = await decodeDocument(
{
nodes: { backendNodeId: [1], parentIndex: [-1], nodeName: [0], attributes: [[1, 2, 3, 4]] },
layout: { nodeIndex: [0], bounds: [[10, 20, 120, 40]], styles: [styles] },
},
strings,
);
expect(nodes[0].attrs).toEqual({ "aria-hidden": "true", inert: "" });
expect(nodes[0].layout).toEqual({
boundsSpace: "snapshot-document-layout",
bounds: [10, 20, 120, 40],
styles: {
position: "static",
"pointer-events": "auto",
cursor: "auto",
visibility: "visible",
opacity: "0",
},
});
expect(nodes[0]).not.toHaveProperty("rect");
expect(nodes[0]).not.toHaveProperty("excluded");
});
it("keeps short blocks synchronous and checks cancellation after yielding", async () => {
const clock = vi.spyOn(performance, "now").mockReturnValue(0);
try {
const controller = new AbortController();
const checkpoint = createCaptureCheckpoint(controller.signal);
for (let i = 0; i < 20; i++) expect(checkpoint()).toBeUndefined();
clock.mockReturnValue(9);
const pending = checkpoint();
expect(pending).toBeInstanceOf(Promise);
controller.abort();
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
} finally {
clock.mockRestore();
}
});
it("checks cancellation at bounded indexing blocks", async () => {
const controller = new AbortController();
const input = Array.from({ length: 100_000 }, (_, i) => node(i, i ? i - 1 : null));
let reads = 0;
for (const item of input)
Object.defineProperty(item, "attrs", {
get: () => {
reads++;
if (reads === 100) controller.abort();
return {};
},
});
await expect(buildDocumentIndex(input, controller.signal)).rejects.toMatchObject({
name: "AbortError",
});
expect(reads).toBeLessThanOrEqual(356);
});
});
@@ -1,6 +1,7 @@
import { describe, expect, it, vi } from "vitest";
import { type CdpRunner, cdpRunnerForTarget } from "../../shared";
import { captureViewModel } from "../capture";
import { enrichFormControlStates } from "../form-capture";
import { decodeDocument } from "../snapshot";
interface Control {
id: number;
@@ -77,21 +78,32 @@ function reply(entries: Array<[number, unknown]>) {
};
}
function fakeCdp(groups: Control[][], evaluate: (params: Record<string, unknown>) => unknown) {
async function fakeCdp(
groups: Control[][],
evaluate: (params: Record<string, unknown>) => unknown,
) {
const snap = snapshot(groups);
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
if (method === "DOMSnapshot.captureSnapshot") return snap;
if (method === "Runtime.evaluate") return evaluate(params as Record<string, unknown>);
return {};
});
return { send, cdp: { send: send as CdpRunner["send"] } };
const nodes = await decodeControls(snap);
return { send, cdp: { send: send as CdpRunner["send"] }, nodes };
}
async function decodeControls(snap: ReturnType<typeof snapshot>) {
return Promise.all(
snap.documents.map(async (doc) =>
(await decodeDocument(doc, snap.strings)).nodes.map((node) => ({ ...node, rect: null })),
),
);
}
describe("form state identity", () => {
it("matches reordered controls by identity and preserves controls missing from the batch", async () => {
// The snapshot includes a shadow input. The runtime query skips it,
// encounters a new input, and sees the remaining inputs in a new order.
const { cdp } = fakeCdp(
const { cdp, nodes } = await fakeCdp(
[
[
{ id: 10, value: "" },
@@ -106,21 +118,21 @@ describe("form state identity", () => {
[11, state("alice@example.com")],
]),
);
const captured = await captureViewModel(cdp, 7);
expect(captured.nodes.find((n) => n.backendNodeId === 10)).toMatchObject({
await enrichFormControlStates(cdp, 7, nodes);
expect(nodes[0].find((n) => n.backendNodeId === 10)).toMatchObject({
formValue: "",
formState: "empty",
formPlaceholder: "snapshot placeholder",
});
expect(captured.nodes.find((n) => n.backendNodeId === 11)).toMatchObject({
expect(nodes[0].find((n) => n.backendNodeId === 11)).toMatchObject({
formValue: "alice@example.com",
});
expect(captured.nodes.find((n) => n.backendNodeId === 12)).toMatchObject({ formValue: "上海" });
expect(captured.nodes.some((n) => n.backendNodeId === 99)).toBe(false);
expect(nodes[0].find((n) => n.backendNodeId === 12)).toMatchObject({ formValue: "上海" });
expect(nodes[0].some((n) => n.backendNodeId === 99)).toBe(false);
});
it("does not shift state between frames when one frame is absent from the runtime batch", async () => {
const { cdp } = fakeCdp(
const { cdp, nodes } = await fakeCdp(
[
[],
[{ id: 11, value: "inaccessible frame" }],
@@ -133,8 +145,8 @@ describe("form state identity", () => {
[21, state("second frame")],
]),
);
const captured = await captureViewModel(cdp, 7);
const controls = [...captured.iframeNodes.values()].flat();
await enrichFormControlStates(cdp, 7, nodes);
const controls = nodes.slice(1).flat();
expect(controls.find((n) => n.backendNodeId === 11)?.formValue).toBe("inaccessible frame");
expect(controls.find((n) => n.backendNodeId === 21)?.formValue).toBe("second frame");
expect(controls.find((n) => n.backendNodeId === 31)?.formValue).toBe("third frame");
@@ -142,7 +154,6 @@ describe("form state identity", () => {
it("keeps identical backend ids in different OOPIF targets separate", async () => {
const sendToTarget = vi.fn(async (target, method) => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot([[{ id: 11, value: "old" }]]);
if (method === "Runtime.evaluate") return reply([[11, state(target.sessionId)]]);
return {};
});
@@ -151,8 +162,9 @@ describe("form state identity", () => {
});
const cdp = { send, sendToTarget } as unknown as CdpRunner;
for (const sessionId of ["left-frame", "right-frame"]) {
const captured = await captureViewModel(cdpRunnerForTarget(cdp, { tabId: 7, sessionId }), 7);
expect(captured.nodes.find((n) => n.backendNodeId === 11)?.formValue).toBe(sessionId);
const nodes = await decodeControls(snapshot([[{ id: 11, value: "old" }]]));
await enrichFormControlStates(cdpRunnerForTarget(cdp, { tabId: 7, sessionId }), 7, nodes);
expect(nodes[0].find((n) => n.backendNodeId === 11)?.formValue).toBe(sessionId);
expect(sendToTarget).toHaveBeenCalledWith(
{ tabId: 7, sessionId },
"Runtime.releaseObjectGroup",
@@ -168,7 +180,7 @@ describe("form state identity", () => {
"malformed",
"invalid-state",
])("preserves snapshot data when enrichment is %s", async (mode) => {
const { cdp, send } = fakeCdp([[{ id: 11, value: "snapshot value" }]], () => {
const { cdp, send, nodes } = await fakeCdp([[{ id: 11, value: "snapshot value" }]], () => {
if (mode === "unsupported") throw new Error("serialization unsupported");
if (mode === "legacy") return { result: { value: { controls: [state("wrong value")] } } };
const result = reply([
@@ -178,8 +190,8 @@ describe("form state identity", () => {
result.result.deepSerializedValue.value[0].value[1].value = "not json";
return result;
});
const captured = await captureViewModel(cdp, 7);
expect(captured.nodes.find((n) => n.backendNodeId === 11)).toMatchObject({
expect(await enrichFormControlStates(cdp, 7, nodes)).toBe(mode !== "unsupported");
expect(nodes[0].find((n) => n.backendNodeId === 11)).toMatchObject({
formValue: "snapshot value",
formPlaceholder: "snapshot placeholder",
});
@@ -188,18 +200,34 @@ describe("form state identity", () => {
it("releases the batch's remote objects when capture is cancelled", async () => {
const controller = new AbortController();
const { cdp, send } = fakeCdp([[{ id: 11, value: "old" }]], () => {
const { cdp, send, nodes } = await fakeCdp([[{ id: 11, value: "old" }]], () => {
controller.abort();
return reply([[11, state("new")]]);
});
await expect(captureViewModel(cdp, 7, { signal: controller.signal })).rejects.toMatchObject({
await expect(enrichFormControlStates(cdp, 7, nodes, controller.signal)).rejects.toMatchObject({
name: "AbortError",
});
expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
});
it("does not report success when cancelled during object cleanup", async () => {
const controller = new AbortController();
const nodes = await decodeControls(snapshot([[{ id: 11, value: "old" }]]));
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method) => {
if (method === "Runtime.evaluate") return reply([[11, state("new")]]);
if (method === "Runtime.releaseObjectGroup") controller.abort();
return {};
}) as CdpRunner["send"],
};
await expect(enrichFormControlStates(cdp, 7, nodes, controller.signal)).rejects.toMatchObject({
name: "AbortError",
});
expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
});
it("skips entries without node identity while still applying valid entries", async () => {
const { cdp } = fakeCdp(
const { cdp, nodes } = await fakeCdp(
[
[
{ id: 11, value: "snapshot value" },
@@ -216,20 +244,20 @@ describe("form state identity", () => {
return result;
},
);
const captured = await captureViewModel(cdp, 7);
expect(captured.nodes.find((n) => n.backendNodeId === 11)?.formValue).toBe("snapshot value");
expect(captured.nodes.find((n) => n.backendNodeId === 12)?.formValue).toBe("live value");
await enrichFormControlStates(cdp, 7, nodes);
expect(nodes[0].find((n) => n.backendNodeId === 11)?.formValue).toBe("snapshot value");
expect(nodes[0].find((n) => n.backendNodeId === 12)?.formValue).toBe("live value");
});
it.each([
"password",
"text",
])("redacts a sensitive %s control after identity matching", async (type) => {
const { cdp } = fakeCdp([[{ id: 11, value: "secret", type }]], () =>
const { cdp, nodes } = await fakeCdp([[{ id: 11, value: "secret", type }]], () =>
reply([[11, { ...state("secret"), sensitive: true }]]),
);
const captured = await captureViewModel(cdp, 7);
const control = captured.nodes.find((n) => n.backendNodeId === 11);
await enrichFormControlStates(cdp, 7, nodes);
const control = nodes[0].find((n) => n.backendNodeId === 11);
expect(control?.formState).toBe("filled");
expect(control?.formValue).toBeUndefined();
expect(control?.formDefaultValue).toBeUndefined();
@@ -246,7 +274,7 @@ describe("form state identity", () => {
defaultValue: "",
placeholder: "",
}));
const { cdp, send } = fakeCdp([controls], (params) => {
const { cdp, send, nodes } = await fakeCdp([controls], (params) => {
const doc = {
querySelectorAll: (selector: string) =>
selector === "input,textarea,select" ? elements : [],
@@ -257,9 +285,9 @@ describe("form state identity", () => {
expect(entries).toHaveLength(250);
return reply(entries.map(([element, json]) => [element.backendNodeId, JSON.parse(json)]));
});
const captured = await captureViewModel(cdp, 7);
expect(captured.nodes.find((n) => n.backendNodeId === 250)?.formValue).toBe("live 250");
expect(captured.nodes.find((n) => n.backendNodeId === 251)?.formValue).toBe("old");
await enrichFormControlStates(cdp, 7, nodes);
expect(nodes[0].find((n) => n.backendNodeId === 250)?.formValue).toBe("live 250");
expect(nodes[0].find((n) => n.backendNodeId === 251)?.formValue).toBe("old");
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1);
expect(
send.mock.calls.filter(([, method]) => method === "Runtime.releaseObjectGroup"),
@@ -1,283 +0,0 @@
import { describe, expect, it, vi } from "vitest";
import type { CdpRunner } from "../../shared";
import type { CapturedNode, CapturedViewModel } from "../capture";
import { captureFrameData } from "../frame-capture";
function ownerNode(backendNodeId: number, x: number): CapturedNode {
return {
backendNodeId,
parentBackendNodeId: 1,
frameId: "main",
ownerFrameBackendNodeId: null,
tag: "iframe",
attrs: {},
rect: { x, y: 100, w: 300, h: 200 },
localRect: { x, y: 100, w: 300, h: 200 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
};
}
function childSnapshot(frameId: string, backendNodeId: number) {
const strings = [frameId, "body", "button", "static", "auto", "pointer"];
return {
strings,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId,
nodes: {
parentIndex: [-1, 0],
nodeName: [1, 2],
backendNodeId: [backendNodeId - 1, backendNodeId],
attributes: [[], []],
},
layout: {
nodeIndex: [0, 1],
styles: [
[3, 4, 4],
[3, 4, 5],
],
bounds: [
[0, 0, 300, 200],
[10, 20, 100, 40],
],
paintOrders: [0, 1],
},
},
],
};
}
describe("captureFrameData", () => {
it("retains nested owner failures when merging a captured OOPIF", async () => {
const owner = ownerNode(10, 50);
const captured: CapturedViewModel = {
nodes: [owner],
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
frameNodes: new Map([["main", [owner]]]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const child = childSnapshot("child", 101);
const nested = childSnapshot("nested", 201);
const document = child.documents[0];
const snapshot = {
strings: child.strings,
documents: [
{
...document,
nodes: { ...document.nodes, contentDocumentIndex: { index: [1], value: [1] } },
},
nested.documents[0],
],
};
const reply = async (_target: unknown, method: string) => {
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 300, clientHeight: 200 },
};
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "DOM.getBoxModel")
return { model: { content: [50, 100, 350, 100, 350, 300, 50, 300] } };
if (method === "DOM.resolveNode") throw new Error("nested owner replaced");
return {};
};
const cdp: CdpRunner = {
send: vi.fn(reply) as CdpRunner["send"],
sendToTarget: vi.fn(reply) as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
})),
};
await captureFrameData(cdp, 4, captured);
expect(captured.frameGeometryIssues).toEqual([
{
status: "unavailable",
source: { target: { tabId: 4, sessionId: "child-session" }, frameId: "nested" },
ownerBackendNodeId: 101,
},
]);
expect(captured.frameNodes?.get("nested")?.[1]).toMatchObject({
rect: null,
localRect: { x: 10, y: 20, w: 100, h: 40 },
});
});
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
const leftOwner = ownerNode(10, 50);
const rightOwner = ownerNode(20, 500);
const captured: CapturedViewModel = {
nodes: [leftOwner, rightOwner],
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
frameNodes: new Map([["main", [leftOwner, rightOwner]]]),
frameOwnerBackendNodeIds: new Map(),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const sendToTarget = vi.fn(async (target, method) => {
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 },
};
}
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
return target.sessionId === "left-session"
? childSnapshot("left", 101)
: childSnapshot("right", 201);
}
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
throw new Error(`unexpected ${method}`);
});
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method, params) => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
if (method === "DOM.getBoxModel") {
const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId;
const x = backendNodeId === 10 ? 50 : 500;
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
}
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "left",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "left-session" },
},
{
frameId: "right",
parentFrameId: "main",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "right-session" },
},
],
})),
};
const trees = await captureFrameData(cdp, 4, captured);
expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]);
expect(
captured.frameNodes?.get("left")?.find((node) => node.backendNodeId === 101)?.rect,
).toEqual({ x: 60, y: 120, w: 100, h: 40 });
expect(
captured.frameNodes?.get("right")?.find((node) => node.backendNodeId === 201)?.rect,
).toEqual({ x: 510, y: 120, w: 100, h: 40 });
expect(captured.frameOwnerBackendNodeIds).toEqual(
new Map([
["left", 10],
["right", 20],
]),
);
expect(captured.frameParentIds).toEqual(
new Map([
["left", "main"],
["right", "main"],
]),
);
});
it("keeps OOPIF semantics when viewport projection is unavailable", async () => {
const owner = { ...ownerNode(10, 50), rect: null, localRect: null };
const captured: CapturedViewModel = {
nodes: [owner],
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
frameNodes: new Map([["main", [owner]]]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method) => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable");
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Page.getLayoutMetrics") throw new Error("viewport unavailable");
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("child", 101);
if (method === "Accessibility.getFullAXTree") {
return {
nodes: [
{
nodeId: "button",
backendDOMNodeId: 101,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Continue" },
},
],
};
}
throw new Error(`unexpected ${method}`);
}) as unknown as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
})),
};
const documents = await captureFrameData(cdp, 4, captured);
const childDocument = documents.find((document) => document.frameId === "child");
expect(childDocument?.axNodes).toEqual([
expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }),
]);
expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual(
expect.objectContaining({ rect: null, localRect: null, rendered: true }),
);
expect(captured.frameGeometryIssues).toContainEqual({
status: "unavailable",
source: { target: { tabId: 4, sessionId: "child-session" }, frameId: "child" },
reason: "snapshot-coordinates-unavailable",
});
});
});
@@ -1,7 +1,12 @@
import { describe, expect, it } from "vitest";
import { describe, expect, it, vi } from "vitest";
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
import type { CapturedNode, CapturedViewModel } from "../capture";
import { buildFrameDocuments, type FrameAxBatch, type FrameOwnedAxNode } from "../frame-document";
import type { CapturedNode } from "../capture";
import {
buildFrameDocuments,
type FrameAxBatch,
type FrameDomInput,
type FrameOwnedAxNode,
} from "../frame-document";
function domNode(frameId: string, backendNodeId: number): CapturedNode {
return {
@@ -17,19 +22,16 @@ function domNode(frameId: string, backendNodeId: number): CapturedNode {
};
}
function captured(frameNodes: Map<string, CapturedNode[]>): CapturedViewModel {
function captured(frameNodes: Map<string, CapturedNode[]>): FrameDomInput {
return {
nodes: frameNodes.get("main") ?? [],
viewport: { width: 800, height: 600 },
iframeNodes: new Map(),
frameNodes,
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
}
describe("buildFrameDocuments", () => {
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", () => {
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", async () => {
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
@@ -54,7 +56,7 @@ describe("buildFrameDocuments", () => {
},
];
const documents = buildFrameDocuments(
const documents = await buildFrameDocuments(
graph,
batches,
captured(
@@ -72,7 +74,7 @@ describe("buildFrameDocuments", () => {
expect(childNodes.find((node) => node.nodeId === "virtual")?.frameId).toBe("child");
});
it("deduplicates overlapping same-target AX results using the strongest ownership", () => {
it("deduplicates overlapping same-target AX results using the strongest ownership", async () => {
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
@@ -92,7 +94,7 @@ describe("buildFrameDocuments", () => {
},
];
const documents = buildFrameDocuments(
const documents = await buildFrameDocuments(
graph,
batches,
captured(
@@ -107,7 +109,7 @@ describe("buildFrameDocuments", () => {
expect(documents.find((document) => document.frameId === "child")?.axNodes).toHaveLength(1);
});
it("uses target-scoped backend ownership when OOPIF backend ids collide", () => {
it("uses target-scoped backend ownership when OOPIF backend ids collide", async () => {
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
@@ -129,7 +131,7 @@ describe("buildFrameDocuments", () => {
nodes: [{ nodeId: `${frame.frameId}-node`, backendDOMNodeId: 7 }],
}));
const documents = buildFrameDocuments(
const documents = await buildFrameDocuments(
graph,
batches,
captured(
@@ -149,7 +151,7 @@ describe("buildFrameDocuments", () => {
);
});
it("uses captured frame parent identity instead of guessing from backend ids", () => {
it("uses captured frame parent identity instead of guessing from backend ids", async () => {
const frameNodes = new Map([
["main", [domNode("main", 1)]],
["left", [domNode("left", 7)]],
@@ -157,18 +159,51 @@ describe("buildFrameDocuments", () => {
["child", [domNode("child", 9)]],
]);
const capture = captured(frameNodes);
capture.frameOwnerBackendNodeIds = new Map([["child", 7]]);
capture.frameParentIds = new Map([["child", "left"]]);
const frames = ["main", "left", "right", "child"].map((frameId) => ({
frame: { frameId, target: { tabId: 4 } },
frame: {
frameId,
target: { tabId: 4 },
...(frameId === "child" ? { parentFrameId: "left", ownerBackendNodeId: 7 } : {}),
},
nodes: [],
}));
const documents = buildFrameDocuments(null, frames, capture);
const documents = await buildFrameDocuments(null, frames, capture);
expect(documents.find((document) => document.frameId === "child")).toMatchObject({
parentFrameId: "left",
ownerBackendNodeId: 7,
});
});
it("rejects explicit AX frame ownership in a different target", async () => {
const frames = [
{ frameId: "main", target: { tabId: 4 } },
{ frameId: "remote", target: { tabId: 4, sessionId: "remote" } },
];
const unresolved = vi.fn();
const documents = await buildFrameDocuments(
{ rootFrameId: "main", frames },
[{ frame: frames[0], nodes: [{ nodeId: "wrong", frameId: "remote" }] }],
captured(new Map()),
undefined,
unresolved,
);
expect(documents.every((doc) => doc.axNodes.length === 0)).toBe(true);
expect(unresolved).toHaveBeenCalledOnce();
});
it("resolves a reverse-ordered deep AX parent chain without recursion", async () => {
const frame = { frameId: "main", target: { tabId: 4 } };
const nodes: FrameOwnedAxNode[] = Array.from({ length: 10000 }, (_, i) => ({
nodeId: String(i),
...(i ? { parentId: String(i - 1) } : { frameId: "main" }),
})).reverse();
const documents = await buildFrameDocuments(
{ rootFrameId: "main", frames: [frame] },
[{ frame, nodes }],
captured(new Map()),
);
expect(documents[0].axNodes).toHaveLength(10000);
expect(documents[0].axNodes[0].frameId).toBe("main");
});
});
@@ -0,0 +1,40 @@
function captureAbortError(): Error {
const error = new Error("observation aborted");
error.name = "AbortError";
return error;
}
export function isAbortError(error: unknown): boolean {
return (
typeof error === "object" &&
error !== null &&
(error as { name?: string }).name === "AbortError"
);
}
export function throwIfAborted(signal: AbortSignal | undefined): void {
if (signal?.aborted) throw captureAbortError();
}
/** Check at bounded work blocks; yield only after a processing slice is spent.
* Each caller owns its deadline, so unrelated captures do not share state. */
export function createCaptureCheckpoint(signal?: AbortSignal): () => Promise<void> | undefined {
throwIfAborted(signal);
let deadline = performance.now() + 8;
return () => {
throwIfAborted(signal);
if (performance.now() < deadline) return;
return new Promise<void>((resolve) => {
const channel = new MessageChannel();
channel.port1.onmessage = () => {
channel.port1.close();
channel.port2.close();
resolve();
};
channel.port2.postMessage(null);
}).then(() => {
throwIfAborted(signal);
deadline = performance.now() + 8;
});
};
}
@@ -0,0 +1,413 @@
import {
type CdpFrame,
type CdpFrameGraph,
type CdpTarget,
cdpTargetKey,
} from "@/browser-driver/frame-graph";
import { cssViewport, GeometryContext } from "../geometry/frame-context";
import { type CdpRunner, cdpRunnerForTarget } from "../shared";
import { collectOverlayExcludedBackendIds } from "./capture";
import {
isAbortError as isCaptureAbort,
throwIfAborted as throwCaptureAborted,
} from "./capture-abort";
import { verifyDocumentIdentity } from "./document-identity";
import type { CapturedSceneInput, DocumentIdentity } from "./facts";
import { buildDocumentIndex, type CaptureIssue, type ObservationFacts } from "./facts";
import { enrichFormControlStates } from "./form-capture";
import {
buildFrameDocuments,
type FrameAxBatch,
type FrameDocument,
type FrameOwnedAxNode,
} from "./frame-document";
import { type NormalizedFrameDocument, normalizeSnapshot } from "./normalize";
import { describeSnapshotFrames, REQUESTED_STYLES, type SnapshotReply } from "./snapshot";
interface TargetBatch<T extends FrameOwnedAxNode> {
target: CdpTarget;
frames: CdpFrame[];
snapshotAttachmentId?: string;
documents: NormalizedFrameDocument[];
ax: FrameAxBatch<T>[];
fallbackExcluded: Set<number>;
snapshotFrameIds?: Set<string>;
rootFrameId?: string;
}
/** A fixed worker pool scoped to one capture; each worker issues at most one
* collection request at a time. Geometry has its own measurement-phase bound. */
async function collectTasks<T>(
items: readonly T[],
collect: (item: T) => Promise<void>,
signal?: AbortSignal,
): Promise<void> {
let cursor = 0;
let failed = false;
let failure: unknown;
await Promise.all(
Array.from({ length: Math.min(4, items.length) }, async () => {
try {
while (cursor < items.length && !failed && !signal?.aborted) {
const item = items[cursor++];
await collect(item);
}
} catch (error) {
if (!failed) failure = error;
failed = true;
}
}),
);
// A worker's rejection must not let the capture outlive its own task scope.
// In-flight reads and their finally blocks settle before cancellation/failure.
throwCaptureAborted(signal);
if (failed) throw failure;
}
export async function captureObservationFacts<T extends FrameOwnedAxNode>(
cdp: CdpRunner,
tabId: number,
signal?: AbortSignal,
pageUrl?: string,
): Promise<ObservationFacts<T>> {
const startedAt = Date.now();
throwCaptureAborted(signal);
let graph: CdpFrameGraph | undefined;
try {
graph = await cdp.getFrameGraph?.(tabId);
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
throwCaptureAborted(signal);
const geometry = new GeometryContext(cdp, tabId, graph, signal);
const issues: CaptureIssue[] = [];
const graphFrames = new Map(graph?.frames.map((frame) => [frame.frameId, frame]) ?? []);
const groups = new Map<string, TargetBatch<T>>();
for (const frame of graph?.frames ?? []) {
const key = cdpTargetKey(frame.target);
let group = groups.get(key);
if (!group) {
group = {
target: frame.target,
frames: [],
documents: [],
ax: [],
fallbackExcluded: new Set(),
};
groups.set(key, group);
}
group.frames.push(
!frame.target.sessionId && frame.frameId === graph?.rootFrameId && !frame.url && pageUrl
? { ...frame, url: pageUrl }
: frame,
);
}
if (!groups.has(cdpTargetKey({ tabId })))
groups.set(cdpTargetKey({ tabId }), {
target: { tabId },
frames: [],
documents: [],
ax: [],
fallbackExcluded: new Set(),
});
const batches = [...groups.values()];
const unavailable = (batch: TargetBatch<T>, stage: CaptureIssue["stage"], frameId?: string) =>
issues.push({ target: batch.target, frameId, stage, reason: "capture-unavailable" });
let firstFailure: unknown;
const axReady = new Set<TargetBatch<T>>();
await collectTasks(
batches,
async (batch) => {
const scoped = cdpRunnerForTarget(cdp, batch.target);
try {
throwCaptureAborted(signal);
await scoped.send(tabId, "DOMSnapshot.enable", {});
throwCaptureAborted(signal);
batch.snapshotAttachmentId = cdp.getAttachmentId?.(tabId);
const snapshot = await scoped.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
computedStyles: REQUESTED_STYLES,
includePaintOrder: true,
includeDOMRects: true,
});
throwCaptureAborted(signal);
const observed = describeSnapshotFrames(snapshot, batch.target, graphFrames, pageUrl);
batch.snapshotFrameIds = observed.ids;
batch.rootFrameId = observed.rootFrameId;
// Keep graph-only frames for AX fallback, never in preference to observed membership.
batch.frames = [
...observed.frames,
...batch.frames.filter((frame) => !observed.ids.has(frame.frameId)),
];
batch.documents = await normalizeSnapshot(
snapshot,
batch.target,
batch.frames.filter((frame) => observed.ids.has(frame.frameId)),
geometry,
issues,
signal,
observed.rootFrameId,
);
const formsAvailable = await enrichFormControlStates(
scoped,
tabId,
batch.documents.map((doc) => doc.nodes),
signal,
);
throwCaptureAborted(signal);
if (!formsAvailable) unavailable(batch, "forms");
const present = new Set(batch.documents.map((doc) => doc.frame.frameId));
for (const frame of batch.frames)
if (!present.has(frame.frameId)) unavailable(batch, "dom", frame.frameId);
} catch (error) {
throwCaptureAborted(signal);
if (isCaptureAbort(error)) throw error;
firstFailure ??= error;
unavailable(batch, "dom");
batch.fallbackExcluded = await collectOverlayExcludedBackendIds(scoped, tabId, signal);
}
if (!batch.frames.length && !graph)
batch.frames = [
{
frameId: "root",
target: batch.target,
...(!batch.target.sessionId && pageUrl ? { url: pageUrl } : {}),
},
];
try {
throwCaptureAborted(signal);
await scoped.send(tabId, "Accessibility.enable", {});
} catch (error) {
throwCaptureAborted(signal);
if (isCaptureAbort(error)) throw error;
firstFailure ??= error;
unavailable(batch, "ax");
return;
}
axReady.add(batch);
},
signal,
);
// Schedule AX by frame across all ready targets, rather than nesting pools.
// Keep result slots in source order even when CDP replies arrive out of order.
const axTasks: { batch: TargetBatch<T>; frame: CdpFrame; result?: FrameAxBatch<T> }[] =
batches.flatMap((batch) =>
axReady.has(batch) ? batch.frames.map((frame) => ({ batch, frame })) : [],
);
await collectTasks(
axTasks,
async (task) => {
const { batch, frame } = task;
try {
throwCaptureAborted(signal);
const result = await cdpRunnerForTarget(cdp, batch.target).send<{ nodes?: T[] }>(
tabId,
"Accessibility.getFullAXTree",
frame.frameId === "root" ? {} : { frameId: frame.frameId },
);
throwCaptureAborted(signal);
task.result = { frame, nodes: result.nodes ?? [] };
} catch (error) {
throwCaptureAborted(signal);
if (isCaptureAbort(error)) throw error;
firstFailure ??= error;
unavailable(batch, "ax", frame.frameId);
}
},
signal,
);
for (const task of axTasks) if (task.result) task.batch.ax.push(task.result);
let viewport = { width: 0, height: 0 };
try {
const measured = cssViewport(await geometry.layoutMetrics({ tabId }));
viewport = { width: measured.width, height: measured.height };
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
// A snapshot claim outranks an old graph hint. Conflicting actual claims
// cannot be resolved by completion order or by overwriting a frameId map.
const claims = new Map<string, string>();
const invalid = new Set<string>();
for (const batch of batches) {
const targetKey = cdpTargetKey(batch.target);
for (const id of batch.snapshotFrameIds ?? []) {
const previous = claims.get(id);
if (previous !== undefined && previous !== targetKey) invalid.add(id);
claims.set(id, targetKey);
}
const accepted = new Set(batch.frames.map((frame) => frame.frameId));
for (const id of batch.snapshotFrameIds ?? []) if (!accepted.has(id)) invalid.add(id);
}
const currentFrames = batches.flatMap((batch) =>
batch.frames.filter(
(frame) =>
!claims.has(frame.frameId) || claims.get(frame.frameId) === cdpTargetKey(batch.target),
),
);
const rootBatch = batches.find((batch) => !batch.target.sessionId);
const rootFrameId =
rootBatch?.rootFrameId ?? graph?.rootFrameId ?? rootBatch?.frames[0]?.frameId ?? "root";
const ownershipInvalid = new Set(invalid);
const identities = new Map<string, DocumentIdentity>();
const selected = new Map(currentFrames.map((frame) => [frame.frameId, frame]));
const checks = batches.flatMap((batch) => {
const docs = new Map(batch.documents.map((doc) => [doc.frame.frameId, doc]));
return batch.frames
.filter(
(frame) =>
!invalid.has(frame.frameId) &&
cdpTargetKey(selected.get(frame.frameId)!.target) === cdpTargetKey(batch.target),
)
.map((frame) => ({ batch, frame, doc: docs.get(frame.frameId) }));
});
await collectTasks(
checks,
async ({ batch, frame, doc }) => {
throwCaptureAborted(signal);
if (!batch.snapshotAttachmentId || doc?.documentElementBackendNodeId === undefined) {
issues.push({
target: frame.target,
frameId: frame.frameId,
stage: "identity",
reason: "identity-unverified",
});
return;
}
const identity: DocumentIdentity = {
attachmentId: batch.snapshotAttachmentId,
target: frame.target,
frameId: frame.frameId,
documentElementBackendNodeId: doc.documentElementBackendNodeId,
};
const verified = await verifyDocumentIdentity(cdp, identity, signal);
const status = cdp.getAttachmentId?.(tabId) === identity.attachmentId ? verified : "changed";
if (status === "current") identities.set(frame.frameId, identity);
else {
invalid.add(frame.frameId);
issues.push({
target: frame.target,
frameId: frame.frameId,
stage: "identity",
reason: status === "changed" ? "document-changed" : "identity-unavailable",
});
}
},
signal,
);
const children = new Map<string, string[]>();
for (const frame of currentFrames) {
if (!frame.parentFrameId) continue;
const siblings = children.get(frame.parentFrameId);
if (siblings) siblings.push(frame.frameId);
else children.set(frame.parentFrameId, [frame.frameId]);
}
const blocked = [...invalid];
for (let i = 0; i < blocked.length; i++) {
for (const child of children.get(blocked[i]) ?? []) {
if (!invalid.has(child)) {
invalid.add(child);
blocked.push(child);
}
}
}
for (const frame of currentFrames) {
if (ownershipInvalid.has(frame.frameId))
issues.push({
target: frame.target,
frameId: frame.frameId,
stage: "ownership",
reason: "frame-ownership-unresolved",
});
}
if (invalid.has(rootFrameId))
throw new Error(
ownershipInvalid.has(rootFrameId)
? "observation root document ownership is ambiguous"
: "observation document identity changed or could not be verified; observe again",
);
const frames = currentFrames.filter((frame) => !invalid.has(frame.frameId));
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
const belongs = (frame: CdpFrame) => {
const current = frameById.get(frame.frameId);
return current && cdpTargetKey(current.target) === cdpTargetKey(frame.target);
};
const decoded = new Map(
batches
.flatMap((batch) => batch.documents)
.filter((doc) => belongs(doc.frame))
.map((doc) => [doc.frame.frameId, doc]),
);
const ax = batches.flatMap((batch) => batch.ax).filter((batch) => belongs(batch.frame));
const documents = await buildFrameDocuments(
{ rootFrameId, frames },
ax,
{
nodes: [],
rootFrameId,
frameNodes: new Map([...decoded].map(([id, doc]) => [id, doc.nodes])),
frameExcludedBackendNodeIds: new Map(
[...decoded].map(([id, doc]) => [id, doc.index.excludedBackendNodeIds]),
),
},
signal,
(frame) =>
issues.push({
target: frame.target,
frameId: frame.frameId,
stage: "ownership",
reason: "frame-ownership-unresolved",
}),
);
const facts: ObservationFacts<T>["documents"][number][] = [];
for (const document of documents) {
const doc = decoded.get(document.frameId);
const index = doc?.index ?? (await buildDocumentIndex([], signal));
const fallback = groups.get(cdpTargetKey(document.target))?.fallbackExcluded;
const {
domNodes,
axNodes,
contextScopeId: _scope,
excludedBackendNodeIds: _excluded,
...frame
} = document;
facts.push({
frame,
identity: identities.get(document.frameId),
index: fallback?.size ? { ...index, excludedBackendNodeIds: fallback } : index,
domNodes,
axNodes,
});
}
throwCaptureAborted(signal);
if (firstFailure && facts.every((doc) => !doc.domNodes.length && !doc.axNodes.length))
throw firstFailure;
return { rootFrameId, viewport, documents: facts, issues, startedAt, finishedAt: Date.now() };
}
/** Existing semantic consumers get a narrow view; no raw snapshot
* data can leak through recording's allowlisted output DTO. */
export function semanticCapture<T extends FrameOwnedAxNode>(
facts: ObservationFacts<T>,
): { captured: CapturedSceneInput; documents: FrameDocument<T>[] } {
const documents = facts.documents.map((doc) => ({
...doc.frame,
contextScopeId: doc.frame.frameId,
domNodes: doc.domNodes.filter(
(node) => !doc.index.excludedBackendNodeIds.has(node.backendNodeId),
),
axNodes: doc.axNodes,
excludedBackendNodeIds: doc.index.excludedBackendNodeIds,
}));
const root = documents.find((doc) => doc.frameId === facts.rootFrameId);
return {
documents,
captured: {
nodes: root?.domNodes ?? [],
viewport: facts.viewport,
rootFrameId: facts.rootFrameId,
excludedBackendNodeIds: root?.excludedBackendNodeIds ?? new Set(),
},
};
}
@@ -0,0 +1,37 @@
import type { Rect } from "@browser-skill/vom";
export interface CapturedNode {
backendNodeId: number;
parentBackendNodeId: number | null;
frameId?: string;
/** Owning iframe backend node id; `null` for the top-level document. */
ownerFrameBackendNodeId?: number | null;
tag: string;
attrs: Record<string, string>;
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
rect: Rect | null;
/** Frame-local viewport-relative CSS px before top-level projection. */
localRect?: Rect | null;
paintOrder: number;
position: string;
pointerEvents: string;
/**
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
* is actually an interactive control — used by the adapter to surface
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
* `textContent`: the live parser always sets it, hand-built fixtures may not.
*/
cursor?: string;
/**
* Whether the live DOM snapshot provides a painted, non-hidden box for this
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
* nodes remain authoritative even when they are outside the viewport.
*/
rendered?: boolean;
textContent?: string;
formState?: "empty" | "filled" | "default";
formValue?: string;
formDefaultValue?: string;
formPlaceholder?: string;
}
+12 -825
View File
@@ -1,339 +1,22 @@
// CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics
// → CapturedNode[] + Viewport. This is the ONLY VOM module that touches
// raw CDP. The captureSnapshot reply is columnar (parallel arrays + a
// shared string table); requested computed styles are decoded through
// STYLE_COL so capture and visibility policy share one explicit contract.
import type { Rect, Viewport } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph";
// Browser-side hover probes and overlay exclusion fallback. Static collection
// lives in capture-coordinator; snapshot decoding has no browser dependency.
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
import {
type FrameProjectionIssue,
type FrameProjectionState,
projectSnapshotRect,
type SnapshotCoordinates,
snapshotCoordinates,
snapshotViewportRect,
} from "../geometry/coordinate-types";
import {
GeometryContext,
type LayoutMetrics,
snapshotLayoutScale,
} from "../geometry/frame-context";
import { OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
import type { CdpRunner } from "../shared";
import type { CapturedNode } from "./capture-types";
import type { CapturedSurfaceProbe } from "./facts";
export type {
CapturedNode,
CapturedSurfaceProbe,
} from "./facts";
import { isAbortError, throwIfAborted } from "./capture-abort";
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
const STYLE_COL = Object.fromEntries(
REQUESTED_STYLES.map((name, index) => [name, index]),
) as Record<(typeof REQUESTED_STYLES)[number], number>;
export interface CapturedNode {
backendNodeId: number;
parentBackendNodeId: number | null;
frameId?: string;
/** Owning iframe backend node id; `null` for the top-level document. */
ownerFrameBackendNodeId?: number | null;
tag: string;
attrs: Record<string, string>;
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
rect: Rect | null;
/** Frame-local viewport-relative CSS px before top-level projection. */
localRect?: Rect | null;
paintOrder: number;
position: string;
pointerEvents: string;
/**
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
* is actually an interactive control — used by the adapter to surface
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
* `textContent`: the live parser always sets it, hand-built fixtures may not.
*/
cursor?: string;
/**
* Whether the live DOM snapshot provides a painted, non-hidden box for this
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
* nodes remain authoritative even when they are outside the viewport.
*/
rendered?: boolean;
textContent?: string;
formState?: "empty" | "filled" | "default";
formValue?: string;
formDefaultValue?: string;
formPlaceholder?: string;
}
export type CapturedIframeNodes = Map<number, CapturedNode[]>;
export interface CapturedSurfaceProbe {
triggerBackendNodeId: number;
triggerPoint?: { x: number; y: number };
triggerAction: "hover" | "focus" | string;
subItems: string[];
confidence?: "high" | "medium" | "low";
}
export interface CapturedViewModel {
nodes: CapturedNode[];
viewport: Viewport;
iframeNodes: CapturedIframeNodes;
frameNodes?: Map<string, CapturedNode[]>;
frameOwnerBackendNodeIds?: Map<string, number>;
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
frameParentIds?: Map<string, string>;
rootFrameId?: string;
/** Failed boundaries and descendants blocked by them; clipping is not a failure. */
frameGeometryIssues?: FrameProjectionIssue[];
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
excludedBackendNodeIds: Set<number>;
}
export interface CaptureViewModelOptions {
signal?: AbortSignal;
geometry?: GeometryContext;
target?: CdpTarget;
}
function captureAbortError(): Error {
const error = new Error("observation aborted");
error.name = "AbortError";
return error;
}
function isAbortError(error: unknown): boolean {
return (
typeof error === "object" &&
error !== null &&
(error as { name?: string }).name === "AbortError"
);
}
function throwIfAborted(signal: AbortSignal | undefined): void {
if (signal?.aborted) throw captureAbortError();
}
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
interface SparseArray {
index: number[];
value: number[];
}
interface RareBooleanData {
index: number[];
}
interface SnapshotDocument {
frameId?: string | number;
scrollOffsetX?: number;
scrollOffsetY?: number;
nodes?: {
parentIndex?: number[];
nodeName?: number[];
backendNodeId?: number[];
attributes?: number[][];
/**
* Per-node text value (index into strings), set for `#text` / CDATA nodes.
* Element nodes carry -1. Same length as `backendNodeId`.
*/
nodeValue?: number[];
/** Maps node array index → index into `documents[]` for frame content. */
contentDocumentIndex?: SparseArray;
inputValue?: SparseArray;
textValue?: SparseArray;
inputChecked?: RareBooleanData;
optionSelected?: RareBooleanData;
};
layout?: {
nodeIndex?: number[];
styles?: number[][];
bounds?: number[][];
paintOrders?: number[];
};
}
function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
if (typeof document.frameId === "string") return document.frameId || undefined;
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
return undefined;
}
interface SnapshotReply {
strings?: string[];
documents?: SnapshotDocument[];
}
function isFormControlTag(tag: string): boolean {
return tag === "input" || tag === "textarea" || tag === "select";
}
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
}
function snapshotFormState(
value: string | undefined,
defaultValue: string,
hasDefaultValue: boolean,
sensitive: boolean,
): CapturedNode["formState"] {
if (sensitive) {
if (value === undefined && !hasDefaultValue) return undefined;
return (value ?? defaultValue) === "" ? "empty" : "filled";
}
if (value === undefined) return undefined;
if (value === "") return "empty";
return value === defaultValue ? "default" : "filled";
}
const MAX_FORM_ENRICH_CONTROLS = 250;
interface CapturedFormState {
value?: string;
defaultValue?: string;
placeholder?: string;
state?: "empty" | "filled" | "default";
sensitive?: boolean;
}
interface DeepSerializedValue {
type: string;
value?: unknown;
}
function formStateBatchExpression(maxControls: number): string {
return `(() => {
const maxControls = ${JSON.stringify(maxControls)};
let remaining = maxControls;
const controlSelector = "input,textarea,select";
const controlState = (el) => {
const tag = el.tagName.toLowerCase();
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
const sensitive = type === "password" || type === "credit-card";
const rawValue = typeof el.value === "string" ? el.value : "";
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
return {
state,
sensitive,
placeholder,
...(sensitive ? {} : { value: rawValue, defaultValue }),
};
};
const controls = [];
const collect = (doc) => {
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
if (remaining <= 0) break;
// Deep serialization supplies the node's backend id. Keep the
// state as JSON so decoding needs no general-purpose V8 deserializer.
controls.push([el, JSON.stringify(controlState(el))]);
remaining -= 1;
}
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
if (remaining <= 0) break;
let childDoc = null;
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
if (childDoc) collect(childDoc);
}
};
collect(document);
return controls;
})()`;
}
function formStatesByBackendId(result: RuntimeEvaluateReply): Map<number, CapturedFormState> {
const states = new Map<number, CapturedFormState>();
const serialized = result.result?.deepSerializedValue;
if (serialized?.type !== "array" || !Array.isArray(serialized.value)) return states;
for (const entry of serialized.value as DeepSerializedValue[]) {
if (entry?.type !== "array" || !Array.isArray(entry.value)) continue;
const [element, json] = entry.value as DeepSerializedValue[];
if (element?.type !== "node" || json?.type !== "string" || typeof json.value !== "string") {
continue;
}
const backendNodeId = (element.value as { backendNodeId?: number } | undefined)?.backendNodeId;
if (typeof backendNodeId !== "number" || !Number.isSafeInteger(backendNodeId)) continue;
try {
const state = JSON.parse(json.value) as CapturedFormState | null;
if (
!state ||
!["empty", "filled", "default"].includes(state.state ?? "") ||
typeof state.sensitive !== "boolean" ||
typeof state.placeholder !== "string" ||
(!state.sensitive &&
(typeof state.value !== "string" || typeof state.defaultValue !== "string"))
) {
continue;
}
states.set(backendNodeId, state);
} catch {
// A malformed entry must not overwrite the snapshot's own state.
}
}
return states;
}
function applyFormStates(nodes: CapturedNode[], states: Map<number, CapturedFormState>): void {
for (const node of nodes) {
if (!isFormControlTag(node.tag)) continue;
const state = states.get(node.backendNodeId);
if (!state) continue;
node.formState = state.state;
node.formPlaceholder = state.placeholder ?? "";
if (state.sensitive || isSensitiveFormControl(node)) {
delete node.formValue;
delete node.formDefaultValue;
delete node.attrs.value;
} else {
node.formDefaultValue = state.defaultValue ?? "";
if (state.value !== undefined) node.formValue = state.value;
}
}
}
async function enrichFormControlStates(
cdp: CdpRunner,
tabId: number,
frameNodeGroups: CapturedNode[][],
signal?: AbortSignal,
): Promise<void> {
const hasControls = frameNodeGroups.some((nodes) =>
nodes.some((node) => isFormControlTag(node.tag)),
);
if (!hasControls) return;
const objectGroup = `bsk-vom-forms-${crypto.randomUUID()}`;
try {
throwIfAborted(signal);
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
objectGroup,
serializationOptions: {
serialization: "deep",
maxDepth: 3,
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
},
});
throwIfAborted(signal);
// Backend ids are scoped to this capture's CDP target. OOPIF targets
// use their own captureViewModel call and never share this lookup.
const states = formStatesByBackendId(result);
for (const nodes of frameNodeGroups) {
applyFormStates(nodes, states);
}
} catch (error) {
if (isAbortError(error)) throw error;
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
} finally {
await cdp.send(tabId, "Runtime.releaseObjectGroup", { objectGroup }).catch(() => undefined);
}
}
interface RuntimeEvaluateReply {
result?: {
value?: unknown;
deepSerializedValue?: DeepSerializedValue;
};
}
@@ -352,36 +35,6 @@ interface CdpDomNode {
shadowRoots?: CdpDomNode[];
}
interface ParseDocumentResult {
nodes: CapturedNode[];
excludedBackendNodeIds: Set<number>;
}
interface FrameContext {
frameId?: string;
ownerFrameBackendNodeId: number | null;
projection: FrameProjectionState;
target: CdpTarget;
coordinates: SnapshotCoordinates | null;
layoutUnitsPerCssPixel: number | null;
}
function str(strings: string[], idx: number | undefined): string {
if (idx === undefined || idx < 0) return "";
return strings[idx] ?? "";
}
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
const out = new Map<number, number>();
if (!sparse?.index || !sparse.value) return out;
for (let i = 0; i < sparse.index.length; i++) {
const nodeIndex = sparse.index[i];
const docIndex = sparse.value[i];
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
}
return out;
}
function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<number>): void {
if (!node) return;
if (typeof node.backendNodeId === "number") {
@@ -790,469 +443,3 @@ export async function collectOverlayExcludedBackendIds(
}
return excluded;
}
/**
* Parse one DOMSnapshot document into CapturedNode[].
*
* @param doc - the raw DOMSnapshot document object
* @param strings - the shared string table for the whole snapshot
* @param context - frame coordinate context; output rects are top viewport-relative
*/
function parseDocumentNodes(
doc: SnapshotDocument,
strings: string[],
context: FrameContext,
): ParseDocumentResult {
const dn = doc.nodes;
const dl = doc.layout;
if (!dn?.backendNodeId) {
return { nodes: [], excludedBackendNodeIds: new Set() };
}
const count = dn.backendNodeId.length;
const layoutByNode = new Map<number, number>();
(dl?.nodeIndex ?? []).forEach((nodeIdx, layoutIdx) => layoutByNode.set(nodeIdx, layoutIdx));
// The agent's own overlay (WXT shadow host, marked with OVERLAY_HOST_MARKER_ATTR)
// is injected into the page and its open shadow root
// is inlined by DOMSnapshot as descendants of the host. Its fixed
// full-viewport click-blocker would otherwise dominate occlusion detection
// and hide the real page, so mark the host + its whole subtree excluded.
// parentIndex always references an earlier node, so one forward pass suffices.
const excluded = new Array<boolean>(count).fill(false);
const excludedBackendNodeIds = new Set<number>();
for (let n = 0; n < count; n++) {
const parentIdx = dn.parentIndex?.[n] ?? -1;
const inherited = parentIdx >= 0 && excluded[parentIdx];
let isOverlayHost = inherited;
if (!isOverlayHost) {
const pairs = dn.attributes?.[n] ?? [];
const attrNames: string[] = [];
for (let a = 0; a + 1 < pairs.length; a += 2) {
attrNames.push(str(strings, pairs[a]));
}
isOverlayHost = isOverlayHostNode(str(strings, dn.nodeName?.[n]), attrNames);
}
if (!isOverlayHost) continue;
excluded[n] = true;
const backendNodeId = dn.backendNodeId[n];
if (backendNodeId !== undefined) {
excludedBackendNodeIds.add(backendNodeId);
}
}
const posCol = STYLE_COL.position;
const peCol = STYLE_COL["pointer-events"];
const cursorCol = STYLE_COL.cursor;
const visibilityCol = STYLE_COL.visibility;
const opacityCol = STYLE_COL.opacity;
const inputValues = sparseIndexMap(dn.inputValue);
const textValues = sparseIndexMap(dn.textValue);
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
// Collect visible text from #text child nodes so element CapturedNodes
// carry a textContent value usable as a button/link label fallback.
// nodeValue is a parallel array: string index for text nodes, -1 otherwise.
const nodeTextContent = new Map<number, string>();
if (dn.nodeValue) {
for (let n = 0; n < count; n++) {
const nvIdx = dn.nodeValue[n] ?? -1;
if (nvIdx < 0) continue;
const text = str(strings, nvIdx).trim();
if (!text) continue;
const parentIdx = dn.parentIndex?.[n] ?? -1;
if (parentIdx >= 0) {
const existing = nodeTextContent.get(parentIdx);
nodeTextContent.set(parentIdx, existing ? `${existing} ${text}` : text);
}
}
}
// Build CapturedNodes.
const nodes: CapturedNode[] = [];
for (let n = 0; n < count; n++) {
if (excluded[n]) continue;
const backendNodeId = dn.backendNodeId[n];
const parentIdx = dn.parentIndex?.[n] ?? -1;
const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null;
const tag = str(strings, dn.nodeName?.[n]).toLowerCase();
const attrs: Record<string, string> = {};
const pairs = dn.attributes?.[n] ?? [];
for (let a = 0; a + 1 < pairs.length; a += 2) {
attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]);
}
let rect: Rect | null = null;
let localRect: Rect | null = null;
let paintOrder = 0;
let position = "static";
let pointerEvents = "auto";
let cursor = "auto";
let rendered = false;
const li = layoutByNode.get(n);
if (li !== undefined) {
const b = dl?.bounds?.[li];
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
const input = snapshotViewportRect(
b,
{ target: context.target, frameId: context.frameId },
context.coordinates,
);
if (input) {
const { x, y, width: w, height: h } = input.rect;
localRect = { x, y, w, h };
}
const bounds =
input && context.projection.status === "available"
? projectSnapshotRect(input, context.projection.projection)
: null;
rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
}
paintOrder = dl?.paintOrders?.[li] ?? 0;
const styleRow = dl?.styles?.[li] ?? [];
position = str(strings, styleRow[posCol]) || "static";
pointerEvents = str(strings, styleRow[peCol]) || "auto";
cursor = str(strings, styleRow[cursorCol]) || "auto";
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
const opacity = str(strings, styleRow[opacityCol]) || "1";
rendered =
!!b &&
b.length >= 4 &&
b.slice(0, 4).every(Number.isFinite) &&
b[2] > 0 &&
b[3] > 0 &&
visibility !== "hidden" &&
visibility !== "collapse" &&
(Number.parseFloat(opacity) || 0) > 0;
}
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
// geometry and are never queried by callers.
if (tag.startsWith("#")) continue;
const textContent = nodeTextContent.get(n);
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
const rawFormValue =
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
const sensitive = isSensitiveFormControl({ tag, attrs });
const formDefaultValue = attrs.value ?? "";
const formValue = sensitive ? undefined : rawFormValue;
const formState = snapshotFormState(
rawFormValue,
formDefaultValue,
Object.prototype.hasOwnProperty.call(attrs, "value"),
sensitive,
);
if (sensitive) delete attrs.value;
nodes.push({
backendNodeId,
parentBackendNodeId,
...(context.frameId ? { frameId: context.frameId } : {}),
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
tag,
attrs,
rect,
localRect,
paintOrder,
position,
pointerEvents,
cursor,
rendered,
textContent,
...(formValue !== undefined ? { formValue } : {}),
...(tag === "input" || tag === "textarea"
? {
formPlaceholder: attrs.placeholder ?? "",
...(!sensitive ? { formDefaultValue } : {}),
...(formState ? { formState } : {}),
}
: {}),
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
});
}
return { nodes, excludedBackendNodeIds };
}
interface ParseFrameDocumentsResult {
frameGeometryIssues: FrameProjectionIssue[];
iframeNodes: CapturedIframeNodes;
frameOwnerBackendNodeIds: Map<string, number>;
frameParentIds: Map<string, string>;
excludedBackendNodeIds: Set<number>;
}
async function parseChildFrameDocuments(
documents: SnapshotDocument[],
strings: string[],
parentDocIndex: number,
parentContext: FrameContext,
geometry: GeometryContext,
signal?: AbortSignal,
visited = new Set<number>(),
): Promise<ParseFrameDocumentsResult> {
const frameGeometryIssues: FrameProjectionIssue[] = [];
const iframeNodes: CapturedIframeNodes = new Map();
const frameOwnerBackendNodeIds = new Map<string, number>();
const frameParentIds = new Map<string, string>();
const excludedBackendNodeIds = new Set<number>();
const parentDoc = documents[parentDocIndex];
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
if (cdi.size === 0) {
return {
iframeNodes,
frameOwnerBackendNodeIds,
frameParentIds,
excludedBackendNodeIds,
frameGeometryIssues,
};
}
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
const children = [...cdi].flatMap(([nodeArrayIdx, childDocIndex]) => {
const childDoc = documents[childDocIndex];
const iframeBackendId = parentBackendIds[nodeArrayIdx];
if (visited.has(childDocIndex) || !childDoc || iframeBackendId === undefined) return [];
const source = { target: parentContext.target, frameId: snapshotFrameId(childDoc, strings) };
const coordinates = snapshotCoordinates(childDoc, parentContext.layoutUnitsPerCssPixel);
const parent = parentContext.projection;
const projection: FrameProjectionState =
parent.status === "available"
? coordinates
? { status: "unavailable", source, ownerBackendNodeId: iframeBackendId }
: { status: "unavailable", source, reason: "snapshot-coordinates-unavailable" }
: { status: "blocked", source, cause: parent.status === "blocked" ? parent.cause : parent };
return [
{
childDocIndex,
childDoc,
iframeBackendId,
source,
coordinates,
projection: projection as FrameProjectionState,
},
];
});
const parentProjection =
parentContext.projection.status === "available"
? parentContext.projection.projection.geometry
: null;
if (parentProjection) {
// Measure only direct siblings. Recursion starts after these workers finish,
// so descendants never hold or recursively acquire a measurement slot.
const edge = parentProjection.edges[0];
const clips = edge
? [edge.destinationQuad, ...(edge.destinationClips ?? [])]
: parentProjection.sourceClips;
let cursor = 0;
let failure: unknown;
await Promise.all(
Array.from({ length: Math.min(4, children.length) }, async () => {
while (cursor < children.length && !signal?.aborted && !failure) {
const child = children[cursor++];
if (!child.coordinates) continue;
try {
// Owner quads are target-relative; do not apply the parent transform twice.
child.projection = await geometry.snapshotProjection(
child.source,
child.iframeBackendId,
clips,
parentProjection.topViewport,
);
} catch (error) {
failure ??= error;
}
}
}),
);
// Join all active reads, including object cleanup, before propagating abort.
if (failure) throw failure;
}
throwIfAborted(signal);
// Completion order must not change document traversal or result insertion order.
for (const {
childDocIndex,
childDoc,
iframeBackendId,
source,
projection,
coordinates,
} of children) {
throwIfAborted(signal);
if (projection.status !== "available") frameGeometryIssues.push(projection);
const childContext: FrameContext = {
frameId: source.frameId,
target: source.target,
ownerFrameBackendNodeId: iframeBackendId,
projection,
coordinates,
layoutUnitsPerCssPixel: parentContext.layoutUnitsPerCssPixel,
};
const nextVisited = new Set(visited);
nextVisited.add(childDocIndex);
const parsed = parseDocumentNodes(childDoc, strings, childContext);
iframeNodes.set(iframeBackendId, parsed.nodes);
if (childContext.frameId) {
frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId);
if (parentContext.frameId) frameParentIds.set(childContext.frameId, parentContext.frameId);
}
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
const nested = await parseChildFrameDocuments(
documents,
strings,
childDocIndex,
childContext,
geometry,
signal,
nextVisited,
);
frameGeometryIssues.push(...nested.frameGeometryIssues);
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
iframeNodes.set(nestedFrameId, nestedNodes);
}
for (const [frameId, ownerBackendNodeId] of nested.frameOwnerBackendNodeIds) {
frameOwnerBackendNodeIds.set(frameId, ownerBackendNodeId);
}
for (const [frameId, parentFrameId] of nested.frameParentIds) {
frameParentIds.set(frameId, parentFrameId);
}
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
}
return {
iframeNodes,
frameOwnerBackendNodeIds,
frameParentIds,
excludedBackendNodeIds,
frameGeometryIssues,
};
}
export async function captureViewModel(
cdp: CdpRunner,
tabId: number,
options: CaptureViewModelOptions = {},
): Promise<CapturedViewModel> {
throwIfAborted(options.signal);
const target = options.target ?? { tabId };
const geometry = options.geometry ?? new GeometryContext(cdp, tabId, undefined, options.signal);
let metrics: LayoutMetrics = {};
try {
metrics = await geometry.layoutMetrics(target);
} catch (error) {
if (isAbortError(error)) throw error;
console.debug("[bsk capture] layout metrics unavailable", error);
}
throwIfAborted(options.signal);
const viewport: Viewport = {
width: metrics.cssLayoutViewport?.clientWidth ?? 0,
height: metrics.cssLayoutViewport?.clientHeight ?? 0,
};
const layoutUnitsPerCssPixel = snapshotLayoutScale(metrics);
await cdp.send(tabId, "DOMSnapshot.enable", {});
throwIfAborted(options.signal);
const snap = await cdp.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
computedStyles: REQUESTED_STYLES,
includePaintOrder: true,
includeDOMRects: true,
});
throwIfAborted(options.signal);
const strings = snap.strings ?? [];
const documents = snap.documents ?? [];
const doc0 = documents[0];
if (!doc0?.nodes?.backendNodeId) {
return {
nodes: [],
viewport,
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
};
}
const source = { target, frameId: snapshotFrameId(doc0, strings) };
const coordinates = snapshotCoordinates(doc0, layoutUnitsPerCssPixel, {
x: metrics.cssLayoutViewport?.pageX,
y: metrics.cssLayoutViewport?.pageY,
});
const topContext: FrameContext = {
frameId: snapshotFrameId(doc0, strings),
ownerFrameBackendNodeId: null,
target,
projection:
coordinates &&
[viewport.width, viewport.height].every((size) => Number.isFinite(size) && size > 0)
? {
status: "available",
projection: {
source,
geometry: { sourceClips: [], edges: [], topViewport: viewport },
},
}
: { status: "unavailable", source, reason: "snapshot-coordinates-unavailable" },
coordinates,
layoutUnitsPerCssPixel,
};
const mainParsed = parseDocumentNodes(doc0, strings, topContext);
const nodes = mainParsed.nodes;
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
const ownerIds = new Set<number>();
for (const document of documents) {
for (const index of document.nodes?.contentDocumentIndex?.index ?? []) {
const id = document.nodes?.backendNodeId?.[index];
if (id !== undefined) ownerIds.add(id);
}
}
geometry.registerSnapshotOwners(target, ownerIds);
const frameParsed = await parseChildFrameDocuments(
documents,
strings,
0,
topContext,
geometry,
options.signal,
);
const iframeNodes = frameParsed.iframeNodes;
for (const id of frameParsed.excludedBackendNodeIds) {
excludedBackendNodeIds.add(id);
}
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()], options.signal);
throwIfAborted(options.signal);
const frameNodes = new Map<string, CapturedNode[]>();
if (topContext.frameId) frameNodes.set(topContext.frameId, nodes);
for (const iframeFrameNodes of iframeNodes.values()) {
const frameId = iframeFrameNodes.find((node) => node.frameId)?.frameId;
if (frameId) frameNodes.set(frameId, iframeFrameNodes);
}
for (const [frameId, ownerBackendNodeId] of frameParsed.frameOwnerBackendNodeIds) {
if (!frameNodes.has(frameId)) {
frameNodes.set(frameId, iframeNodes.get(ownerBackendNodeId) ?? []);
}
}
return {
nodes,
viewport,
iframeNodes,
frameNodes,
frameGeometryIssues: [
...(topContext.projection.status === "available" ? [] : [topContext.projection]),
...frameParsed.frameGeometryIssues,
],
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
frameParentIds: frameParsed.frameParentIds,
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
excludedBackendNodeIds,
};
}
@@ -0,0 +1,53 @@
import type { CdpRunner } from "../shared";
import { sendToCdpTarget } from "../shared";
import { isAbortError, throwIfAborted } from "./capture-abort";
import type { DocumentIdentity } from "./facts";
/** Compare the snapshot root with the current root in that exact frame.
* Deep serialization supplies the backend ID without a separate describeNode. */
export async function verifyDocumentIdentity(
cdp: CdpRunner,
identity: DocumentIdentity,
signal?: AbortSignal,
): Promise<"current" | "changed" | "unavailable"> {
const attached = () => cdp.getAttachmentId?.(identity.target.tabId) === identity.attachmentId;
throwIfAborted(signal);
if (!attached()) return "changed";
const objectGroup = `bsk-document-identity-${crypto.randomUUID()}`;
const send = <T>(method: string, params: object) => {
throwIfAborted(signal);
return sendToCdpTarget<T>(cdp, identity.target, method, params);
};
try {
const world = await send<{ executionContextId: number }>("Page.createIsolatedWorld", {
frameId: identity.frameId,
worldName: "bsk-document-identity",
});
const reply = await send<{
result?: { deepSerializedValue?: { type: string; value?: { backendNodeId?: number } } };
}>("Runtime.evaluate", {
expression: "document.documentElement",
contextId: world.executionContextId,
objectGroup,
serializationOptions: {
serialization: "deep",
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
},
});
if (!attached()) return "changed";
const root = reply.result?.deepSerializedValue;
if (root?.type === "null") return "changed";
const id = root?.type === "node" ? root.value?.backendNodeId : undefined;
if (id === undefined) return "unavailable";
return id === identity.documentElementBackendNodeId ? "current" : "changed";
} catch (error) {
throwIfAborted(signal);
if (isAbortError(error)) throw error;
return attached() ? "unavailable" : "changed";
} finally {
await sendToCdpTarget(cdp, identity.target, "Runtime.releaseObjectGroup", {
objectGroup,
}).catch(() => {});
throwIfAborted(signal);
}
}
+155
View File
@@ -0,0 +1,155 @@
import type { Viewport } from "@browser-skill/vom";
import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph";
import { isOverlayHostNode } from "@/lib/overlay-bridge";
import type { FrameProjectionIssue } from "../geometry/coordinate-types";
import { createCaptureCheckpoint } from "./capture-abort";
import type { CapturedNode } from "./capture-types";
import type { FrameOwnedAxNode } from "./frame-document";
/** Bounds retain their raw snapshot document layout units until normalization. */
export interface SnapshotLayout {
readonly boundsSpace: "snapshot-document-layout";
bounds?: number[];
styles: Readonly<Record<string, string>>;
}
export interface DecodedNode extends Omit<CapturedNode, "rect" | "localRect" | "rendered"> {
nodeType?: number;
layout?: SnapshotLayout;
}
export interface NodeFacts extends CapturedNode {
nodeType?: number;
layout?: SnapshotLayout;
}
export interface DocumentIndex<T extends DecodedNode = NodeFacts> {
readonly nodes: ReadonlyMap<number, T>;
readonly excludedBackendNodeIds: ReadonlySet<number>;
}
export interface DecodedDocument {
nodes: DecodedNode[];
}
/** All snapshot nodes participate, including document and shadow roots. This
* preserves overlay propagation when the semantic adapter omits those nodes. */
export async function buildDocumentIndex<T extends DecodedNode>(
input: readonly T[],
signal?: AbortSignal,
): Promise<DocumentIndex<T>> {
const checkpoint = createCaptureCheckpoint(signal);
const nodes = new Map<number, T>();
const overlayByNode = new Map<number, boolean>();
const excludedBackendNodeIds = new Set<number>();
for (let i = 0; i < input.length; i++) {
if (i % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const node = input[i];
nodes.set(node.backendNodeId, node);
}
let work = 0;
for (const node of input) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
if (overlayByNode.has(node.backendNodeId)) continue;
const path: DecodedNode[] = [];
const visiting = new Set<number>();
let current: DecodedNode | undefined = node;
let overlay = false;
while (current && !overlayByNode.has(current.backendNodeId)) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
if (visiting.has(current.backendNodeId)) {
// Only an overlay inside the cycle may mark the cycle and its descendants.
overlay = path
.slice(path.findIndex((item) => item.backendNodeId === current!.backendNodeId))
.some((item) => isOverlayHostNode(item.tag, Object.keys(item.attrs)));
break;
}
visiting.add(current.backendNodeId);
path.push(current);
if (current.parentBackendNodeId === null) {
current = undefined;
break;
}
current = nodes.get(current.parentBackendNodeId);
}
if (current && overlayByNode.has(current.backendNodeId))
overlay = overlayByNode.get(current.backendNodeId)!;
for (let i = path.length - 1; i >= 0; i--) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const item = path[i];
overlay = overlay || isOverlayHostNode(item.tag, Object.keys(item.attrs));
overlayByNode.set(item.backendNodeId, overlay);
if (overlay) excludedBackendNodeIds.add(item.backendNodeId);
}
}
return { nodes, excludedBackendNodeIds };
}
export interface DocumentIdentity {
attachmentId: string;
target: CdpTarget;
frameId: string;
documentElementBackendNodeId: number;
}
/** Narrow input shared by the existing semantic scene/hover consumers. */
export interface CapturedSceneInput {
nodes: CapturedNode[];
viewport: Viewport;
rootFrameId?: string;
excludedBackendNodeIds: ReadonlySet<number>;
}
export interface CaptureIssue {
/** Owner failure or inherited blockage; absent for other geometry failures. */
projectionIssue?: FrameProjectionIssue;
target: CdpTarget;
frameId?: string;
stage: "dom" | "ax" | "identity" | "ownership" | "geometry" | "forms";
reason:
| "document-changed"
| "identity-unavailable"
| "identity-unverified"
| "capture-unavailable"
| "frame-ownership-unresolved"
| "geometry-unavailable";
}
export interface DocumentFacts<T extends FrameOwnedAxNode> {
readonly frame: CdpFrame;
readonly identity?: DocumentIdentity;
readonly index: DocumentIndex;
readonly domNodes: CapturedNode[];
readonly axNodes: T[];
}
export interface ObservationFacts<T extends FrameOwnedAxNode> {
readonly rootFrameId: string;
readonly viewport: Viewport;
readonly documents: readonly DocumentFacts<T>[];
readonly issues: readonly CaptureIssue[];
readonly startedAt: number;
readonly finishedAt: number;
}
export interface CapturedSurfaceProbe {
triggerBackendNodeId: number;
triggerPoint?: { x: number; y: number };
triggerAction: "hover" | "focus" | string;
subItems: string[];
confidence?: "high" | "medium" | "low";
}
export type { CapturedNode } from "./capture-types";
@@ -0,0 +1,155 @@
import type { CdpRunner } from "../shared";
import { isAbortError, throwIfAborted } from "./capture-abort";
import type { CapturedNode } from "./capture-types";
interface RuntimeEvaluateReply {
result?: { deepSerializedValue?: DeepSerializedValue };
}
const MAX_FORM_ENRICH_CONTROLS = 250;
interface CapturedFormState {
value?: string;
defaultValue?: string;
placeholder?: string;
state?: "empty" | "filled" | "default";
sensitive?: boolean;
}
interface DeepSerializedValue {
type: string;
value?: unknown;
}
function formStateBatchExpression(maxControls: number): string {
return `(() => {
const maxControls = ${JSON.stringify(maxControls)};
let remaining = maxControls;
const controlSelector = "input,textarea,select";
const controlState = (el) => {
const tag = el.tagName.toLowerCase();
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
const sensitive = type === "password" || type === "credit-card";
const rawValue = typeof el.value === "string" ? el.value : "";
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
return {
state,
sensitive,
placeholder,
...(sensitive ? {} : { value: rawValue, defaultValue }),
};
};
const controls = [];
const collect = (doc) => {
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
if (remaining <= 0) break;
// Deep serialization supplies the node's backend id. Keep the
// state as JSON so decoding needs no general-purpose V8 deserializer.
controls.push([el, JSON.stringify(controlState(el))]);
remaining -= 1;
}
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
if (remaining <= 0) break;
let childDoc = null;
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
if (childDoc) collect(childDoc);
}
};
collect(document);
return controls;
})()`;
}
function formStatesByBackendId(result: RuntimeEvaluateReply): Map<number, CapturedFormState> {
const states = new Map<number, CapturedFormState>();
const serialized = result.result?.deepSerializedValue;
if (serialized?.type !== "array" || !Array.isArray(serialized.value)) return states;
for (const entry of serialized.value as DeepSerializedValue[]) {
if (entry?.type !== "array" || !Array.isArray(entry.value)) continue;
const [element, json] = entry.value as DeepSerializedValue[];
if (element?.type !== "node" || json?.type !== "string" || typeof json.value !== "string") {
continue;
}
const backendNodeId = (element.value as { backendNodeId?: number } | undefined)?.backendNodeId;
if (typeof backendNodeId !== "number" || !Number.isSafeInteger(backendNodeId)) continue;
try {
const state = JSON.parse(json.value) as CapturedFormState | null;
if (
!state ||
!["empty", "filled", "default"].includes(state.state ?? "") ||
typeof state.sensitive !== "boolean" ||
typeof state.placeholder !== "string" ||
(!state.sensitive &&
(typeof state.value !== "string" || typeof state.defaultValue !== "string"))
) {
continue;
}
states.set(backendNodeId, state);
} catch {
// A malformed entry must not overwrite the snapshot's own state.
}
}
return states;
}
function applyFormStates(nodes: CapturedNode[], states: Map<number, CapturedFormState>): void {
for (const node of nodes) {
if (!["input", "textarea", "select"].includes(node.tag)) continue;
const state = states.get(node.backendNodeId);
if (!state) continue;
node.formState = state.state;
node.formPlaceholder = state.placeholder ?? "";
if (
state.sensitive ||
(node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password")
) {
delete node.formValue;
delete node.formDefaultValue;
delete node.attrs.value;
} else {
node.formDefaultValue = state.defaultValue ?? "";
if (state.value !== undefined) node.formValue = state.value;
}
}
}
export async function enrichFormControlStates(
cdp: CdpRunner,
tabId: number,
frameNodeGroups: CapturedNode[][],
signal?: AbortSignal,
): Promise<boolean> {
const hasControls = frameNodeGroups.some((nodes) =>
nodes.some((node) => ["input", "textarea", "select"].includes(node.tag)),
);
if (!hasControls) return true;
const objectGroup = `bsk-vom-forms-${crypto.randomUUID()}`;
try {
throwIfAborted(signal);
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
objectGroup,
serializationOptions: {
serialization: "deep",
maxDepth: 3,
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
},
});
throwIfAborted(signal);
// Backend ids are scoped to this capture's CDP target. OOPIF targets
// have separate enrichment calls and never share this lookup.
const states = formStatesByBackendId(result);
for (const nodes of frameNodeGroups) {
applyFormStates(nodes, states);
}
return true;
} catch (error) {
if (isAbortError(error)) throw error;
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
return false;
} finally {
await cdp.send(tabId, "Runtime.releaseObjectGroup", { objectGroup }).catch(() => undefined);
throwIfAborted(signal);
}
}
@@ -1,178 +0,0 @@
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
import { type GeometryProjection, projectRectToViewport } from "../geometry";
import { GeometryContext } from "../geometry/frame-context";
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
import {
buildFrameDocuments,
type FrameAxBatch,
type FrameDocument,
type FrameOwnedAxNode,
} from "./frame-document";
export type FrameAxNode = FrameOwnedAxNode;
export type CapturedFrameDocument<T extends FrameAxNode> = FrameDocument<T>;
function throwIfAborted(signal?: AbortSignal): void {
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
}
function isAbortError(err: unknown): boolean {
return err instanceof Error && err.name === "AbortError";
}
function transformFrameNodes(
nodes: CapturedNode[],
projection: GeometryProjection | null,
): CapturedNode[] {
return nodes.map((node) => ({
...node,
rect: projection
? (() => {
const bounds = projectRectToViewport(node.rect, projection);
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
})()
: null,
}));
}
async function captureMissingFrameDocuments(
cdp: CdpRunner,
tabId: number,
graph: CdpFrameGraph,
captured: CapturedViewModel,
geometry: GeometryContext,
signal?: AbortSignal,
): Promise<void> {
if (!captured.frameNodes) captured.frameNodes = new Map();
if (!captured.frameOwnerBackendNodeIds) captured.frameOwnerBackendNodeIds = new Map();
if (!captured.frameParentIds) captured.frameParentIds = new Map();
for (const frame of graph.frames) {
if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue;
try {
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
signal,
geometry,
target: frame.target,
});
if (child.frameGeometryIssues?.length) {
(captured.frameGeometryIssues ??= []).push(...child.frameGeometryIssues);
}
let projection: GeometryProjection | null = null;
try {
projection = await geometry.targetProjection(frame.frameId);
} catch (err) {
if (isAbortError(err)) throw err;
console.debug("[bsk observation] child frame geometry projection failed", {
frameId: frame.frameId,
err,
});
}
const rootFrameId = child.rootFrameId ?? frame.frameId;
const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
if (rootFrameId !== frame.frameId && !childFrames.has(frame.frameId)) {
childFrames.set(
frame.frameId,
child.nodes.map((node) => ({ ...node, frameId: frame.frameId })),
);
}
for (const [childFrameId, nodes] of childFrames) {
captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
}
if (frame.ownerBackendNodeId !== undefined) {
captured.iframeNodes.set(
frame.ownerBackendNodeId,
captured.frameNodes.get(frame.frameId) ?? [],
);
captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
}
if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId);
for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) {
captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId);
}
for (const [childFrameId, parentFrameId] of child.frameParentIds ?? []) {
captured.frameParentIds.set(childFrameId, parentFrameId);
}
} catch (err) {
if (isAbortError(err)) throw err;
console.debug("[bsk observation] child frame DOM capture failed", {
frameId: frame.frameId,
err,
});
}
}
}
async function captureAxTrees<T extends FrameAxNode>(
cdp: CdpRunner,
tabId: number,
discoveredFrames: CdpFrame[],
captured: CapturedViewModel,
signal?: AbortSignal,
): Promise<FrameAxBatch<T>[]> {
let frames = [...discoveredFrames];
const capturedFrameIds = [...(captured.frameNodes?.keys() ?? [])];
if (frames.length === 0) {
const rootFrameId = captured.rootFrameId ?? capturedFrameIds[0] ?? "root";
frames = [{ frameId: rootFrameId, target: { tabId } }];
}
for (const frameId of capturedFrameIds) {
if (!frames.some((frame) => frame.frameId === frameId)) {
frames.push({ frameId, target: { tabId } });
}
}
let firstFailure: unknown;
const trees = await Promise.all(
frames.map(async (frame): Promise<FrameAxBatch<T>> => {
try {
throwIfAborted(signal);
await sendToCdpTarget(cdp, frame.target, "Accessibility.enable", {});
const result = await sendToCdpTarget<{ nodes?: T[] }>(
cdp,
frame.target,
"Accessibility.getFullAXTree",
frame.frameId === "root" ? {} : { frameId: frame.frameId },
);
throwIfAborted(signal);
return {
frame,
nodes: result.nodes ?? [],
};
} catch (err) {
if (isAbortError(err)) throw err;
firstFailure ??= err;
console.debug("[bsk observation] frame AX capture failed", {
frameId: frame.frameId,
err,
});
return { frame, nodes: [] };
}
}),
);
const capturedNodeCount = [...captured.iframeNodes.values()].reduce(
(count, nodes) => count + nodes.length,
captured.nodes.length,
);
if (firstFailure && capturedNodeCount === 0 && trees.every((tree) => tree.nodes.length === 0)) {
throw firstFailure;
}
return trees;
}
export async function captureFrameData<T extends FrameAxNode>(
cdp: CdpRunner,
tabId: number,
captured: CapturedViewModel,
signal?: AbortSignal,
geometry = new GeometryContext(cdp, tabId, undefined, signal),
): Promise<CapturedFrameDocument<T>[]> {
const graph = await geometry.graph();
const frames = graph?.frames ?? [];
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, geometry, signal);
throwIfAborted(signal);
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
return buildFrameDocuments(graph, batches, captured);
}
+92 -61
View File
@@ -4,7 +4,8 @@ import {
type CdpTarget,
cdpTargetKey,
} from "@/browser-driver/frame-graph";
import type { CapturedNode, CapturedViewModel } from "./capture";
import { createCaptureCheckpoint } from "./capture-abort";
import type { CapturedNode } from "./capture-types";
export interface FrameOwnedAxNode {
nodeId: string;
@@ -23,6 +24,15 @@ export interface FrameDocument<T extends FrameOwnedAxNode> extends CdpFrame {
contextScopeId: string;
axNodes: T[];
domNodes: CapturedNode[];
excludedBackendNodeIds?: ReadonlySet<number>;
}
/** Explicit snapshot document data used to join DOM and AX ownership. */
export interface FrameDomInput {
nodes: CapturedNode[];
rootFrameId?: string;
frameNodes?: ReadonlyMap<string, CapturedNode[]>;
frameExcludedBackendNodeIds?: ReadonlyMap<string, ReadonlySet<number>>;
}
interface Ownership {
@@ -47,35 +57,25 @@ function targetBackendKey(target: CdpTarget, backendNodeId: number): string {
function frameList<T extends FrameOwnedAxNode>(
graph: CdpFrameGraph | null,
batches: FrameAxBatch<T>[],
captured: CapturedViewModel,
): CdpFrame[] {
const frames = new Map<string, CdpFrame>();
for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame);
for (const batch of batches) {
if (!frames.has(batch.frame.frameId)) frames.set(batch.frame.frameId, batch.frame);
}
for (const frameId of captured.frameNodes?.keys() ?? []) {
if (!frames.has(frameId))
frames.set(frameId, { frameId, target: { tabId: batches[0]?.frame.target.tabId ?? 0 } });
}
return [...frames.values()].map((frame) => {
const ownerBackendNodeId =
frame.ownerBackendNodeId ?? captured.frameOwnerBackendNodeIds?.get(frame.frameId);
const parentFrameId = frame.parentFrameId ?? captured.frameParentIds?.get(frame.frameId);
return {
...frame,
...(ownerBackendNodeId !== undefined ? { ownerBackendNodeId } : {}),
...(parentFrameId ? { parentFrameId } : {}),
};
});
return [...frames.values()];
}
export function buildFrameDocuments<T extends FrameOwnedAxNode>(
export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
graph: CdpFrameGraph | null,
batches: FrameAxBatch<T>[],
captured: CapturedViewModel,
): FrameDocument<T>[] {
const frames = frameList(graph, batches, captured);
captured: FrameDomInput,
signal?: AbortSignal,
unresolved?: (frame: CdpFrame) => void,
): Promise<FrameDocument<T>[]> {
const checkpoint = createCaptureCheckpoint(signal);
let work = 0;
const frames = frameList(graph, batches);
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId;
const domNodesForFrame = (frameId: string): CapturedNode[] =>
@@ -83,54 +83,80 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
const backendOwner = new Map<string, string>();
for (const frame of frames) {
for (const node of domNodesForFrame(frame.frameId)) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId);
}
}
const candidates = new Map<string, OwnedCandidate<T>>();
for (const batch of batches) {
const nodeById = new Map(batch.nodes.map((node) => [node.nodeId, node]));
const ownershipByNodeId = new Map<string, Ownership>();
const resolving = new Set<string>();
const resolveOwnership = (node: T): Ownership => {
const cached = ownershipByNodeId.get(node.nodeId);
if (cached) return cached;
if (node.frameId && frameById.has(node.frameId)) {
const ownership = { frameId: node.frameId, strength: 4 };
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
}
if (typeof node.backendDOMNodeId === "number") {
const frameId = backendOwner.get(
targetBackendKey(batch.frame.target, node.backendDOMNodeId),
);
if (frameId) {
const ownership = { frameId, strength: 3 };
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
}
}
if (node.parentId && !resolving.has(node.nodeId)) {
const parent = nodeById.get(node.parentId);
if (parent) {
resolving.add(node.nodeId);
const inherited = resolveOwnership(parent);
resolving.delete(node.nodeId);
const ownership = {
frameId: inherited.frameId,
strength: Math.min(2, inherited.strength),
};
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
}
}
const ownership = { frameId: batch.frame.frameId, strength: 1 };
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
};
const nodeById = new Map<string, T>();
for (const node of batch.nodes) {
const ownership = resolveOwnership(node);
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
nodeById.set(node.nodeId, node);
}
let reported = false;
const ownershipByNodeId = new Map<string, Ownership>();
for (const node of batch.nodes) {
const path: T[] = [];
const visiting = new Set<string>();
let current: T | undefined = node;
let ownership: Ownership = { frameId: batch.frame.frameId, strength: 1 };
while (current) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const cached = ownershipByNodeId.get(current.nodeId);
if (cached) {
ownership = cached;
break;
}
if (visiting.has(current.nodeId)) break;
visiting.add(current.nodeId);
if (current.frameId) {
const frame = frameById.get(current.frameId);
// An explicit contradiction must not fall through to a guessed owner.
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(batch.frame.target)) {
if (!reported) {
unresolved?.(batch.frame);
reported = true;
}
ownership = { frameId: "", strength: 0 };
break;
}
ownership = { frameId: frame.frameId, strength: 4 };
ownershipByNodeId.set(current.nodeId, ownership);
break;
}
const frameId =
typeof current.backendDOMNodeId === "number"
? backendOwner.get(targetBackendKey(batch.frame.target, current.backendDOMNodeId))
: undefined;
if (frameId) {
ownership = { frameId, strength: 3 };
ownershipByNodeId.set(current.nodeId, ownership);
break;
}
path.push(current);
current = current.parentId ? nodeById.get(current.parentId) : undefined;
}
for (let i = path.length - 1; i >= 0; i--) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
ownership = { frameId: ownership.frameId, strength: Math.min(2, ownership.strength) };
ownershipByNodeId.set(path[i].nodeId, ownership);
}
if (!ownershipByNodeId.has(node.nodeId)) ownershipByNodeId.set(node.nodeId, ownership);
if (!ownership.frameId) continue;
const key = targetNodeKey(batch.frame.target, node.nodeId);
const existing = candidates.get(key);
if (!existing || ownership.strength > existing.ownership.strength) {
@@ -144,6 +170,10 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
);
const axNodesByFrame = new Map<string, T[]>();
for (const candidate of candidates.values()) {
if (work++ % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const { node, target, ownership } = candidate;
const parentFrameId = node.parentId
? ownershipByTargetNode.get(targetNodeKey(target, node.parentId))
@@ -169,5 +199,6 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
contextScopeId: frame.frameId,
axNodes: axNodesByFrame.get(frame.frameId) ?? [],
domNodes: domNodesForFrame(frame.frameId),
excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(frame.frameId),
}));
}
+325
View File
@@ -0,0 +1,325 @@
import type { Viewport } from "@browser-skill/vom";
import { type CdpFrame, type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
import { type GeometryProjection, projectRectToViewport } from "../geometry";
import {
type FrameProjectionState,
projectSnapshotRect,
type SnapshotCoordinates,
snapshotCoordinates,
snapshotViewportRect,
} from "../geometry/coordinate-types";
import {
GeometryContext,
type LayoutMetrics,
snapshotLayoutScale,
} from "../geometry/frame-context";
import {
createCaptureCheckpoint,
isAbortError as isCaptureAbort,
throwIfAborted as throwCaptureAborted,
} from "./capture-abort";
import { buildDocumentIndex, type CaptureIssue, type DocumentIndex, type NodeFacts } from "./facts";
import {
decodeDocument,
type SnapshotDocument,
type SnapshotReply,
snapshotFrameId,
} from "./snapshot";
export interface FrameContext {
frameId?: string;
ownerFrameBackendNodeId: number | null;
projection: FrameProjectionState | null;
targetProjection?: GeometryProjection | null;
target: CdpTarget;
coordinates: SnapshotCoordinates | null;
}
export interface NormalizedDocument {
nodes: NodeFacts[];
index: DocumentIndex;
documentElementBackendNodeId?: number;
}
/** Interpret one document using a supplied projection. No live reads or frame scheduling. */
export async function normalizeDocument(
doc: SnapshotDocument,
strings: string[],
context: FrameContext,
signal?: AbortSignal,
): Promise<NormalizedDocument> {
const decoded = await decodeDocument(doc, strings, signal);
const checkpoint = createCaptureCheckpoint(signal);
const nodes: NodeFacts[] = [];
for (let i = 0; i < decoded.nodes.length; i++) {
if (i % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const node = decoded.nodes[i];
const bounds = node.layout?.bounds ?? [];
const input = snapshotViewportRect(
bounds,
{ target: context.target, frameId: context.frameId },
context.coordinates,
);
const local = input?.rect;
let rect =
input && context.projection?.status === "available"
? projectSnapshotRect(input, context.projection.projection)
: null;
if (rect && context.targetProjection !== undefined)
rect = context.targetProjection
? projectRectToViewport(
{ x: rect.x, y: rect.y, w: rect.width, h: rect.height },
context.targetProjection,
)
: null;
const visibility = node.layout?.styles.visibility || "visible";
const opacity = node.layout?.styles.opacity || "1";
nodes.push({
...node,
...(context.frameId ? { frameId: context.frameId } : {}),
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
localRect: local ? { x: local.x, y: local.y, w: local.width, h: local.height } : null,
rect: rect ? { x: rect.x, y: rect.y, w: rect.width, h: rect.height } : null,
rendered:
bounds.length >= 4 &&
bounds.slice(0, 4).every(Number.isFinite) &&
bounds[2] > 0 &&
bounds[3] > 0 &&
visibility !== "hidden" &&
visibility !== "collapse" &&
(Number.parseFloat(opacity) || 0) > 0,
});
}
const index = await buildDocumentIndex(nodes, signal);
return {
nodes: nodes.filter(
(node) => !node.tag.startsWith("#") && !index.excludedBackendNodeIds.has(node.backendNodeId),
),
index,
documentElementBackendNodeId: nodes.find(
(node) =>
node.nodeType === 1 &&
node.parentBackendNodeId !== null &&
index.nodes.get(node.parentBackendNodeId)?.nodeType === 9,
)?.backendNodeId,
};
}
export interface NormalizedFrameDocument extends NormalizedDocument {
frame: CdpFrame;
}
/** One target's snapshot. Source ownership is checked before interpreting any
* coordinates. Snapshot and live quad coordinates are deliberately not conflated. */
export async function normalizeSnapshot(
snapshot: SnapshotReply,
target: CdpTarget,
frames: readonly CdpFrame[],
geometry: GeometryContext,
issues: CaptureIssue[],
signal?: AbortSignal,
rootFrameId?: string,
): Promise<NormalizedFrameDocument[]> {
const strings = snapshot.strings ?? [];
const raw = snapshot.documents ?? [];
const ownerIds = new Set<number>();
for (const document of raw) {
for (const index of document.nodes?.contentDocumentIndex?.index ?? []) {
const id = document.nodes?.backendNodeId?.[index];
if (id !== undefined) ownerIds.add(id);
}
}
geometry.registerSnapshotOwners(target, ownerIds);
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
const sources = new Map<string, SnapshotDocument>();
for (const doc of raw) {
const id = snapshotFrameId(doc, strings);
if (!id || !frameById.has(id) || sources.has(id)) {
issues.push({
target,
frameId: id,
stage: "ownership",
reason: "frame-ownership-unresolved",
});
continue;
}
sources.set(id, doc);
}
let metrics: LayoutMetrics = {};
try {
metrics = await geometry.layoutMetrics(target);
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
const viewport: Viewport = {
width: metrics.cssLayoutViewport?.clientWidth ?? 0,
height: metrics.cssLayoutViewport?.clientHeight ?? 0,
};
const layoutUnitsPerCssPixel = snapshotLayoutScale(metrics);
const viewportAvailable = [viewport.width, viewport.height].every(
(size) => Number.isFinite(size) && size > 0,
);
const children = new Map<string, CdpFrame[]>();
const pending: CdpFrame[] = [];
for (const frame of frames) {
if (!frame.parentFrameId || !frameById.has(frame.parentFrameId)) pending.push(frame);
else {
const siblings = children.get(frame.parentFrameId);
if (siblings) siblings.push(frame);
else children.set(frame.parentFrameId, [frame]);
}
}
const projections = new Map<string, FrameProjectionState | null>();
const result: NormalizedFrameDocument[] = [];
let targetProjection: GeometryProjection | null = null;
const targetRoot = rootFrameId ? frameById.get(rootFrameId) : undefined;
if (target.sessionId && targetRoot) {
try {
const known = await geometry.frame(targetRoot.frameId);
if (known && cdpTargetKey(known.target) === cdpTargetKey(target))
targetProjection = await geometry.targetProjection(targetRoot.frameId);
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
}
const checkpoint = createCaptureCheckpoint(signal);
const visited = new Set<string>();
const remaining = frames[Symbol.iterator]();
for (let cursor = 0; visited.size < frames.length; cursor++) {
if (cursor >= pending.length) {
let next = remaining.next();
while (!next.done && visited.has(next.value.frameId)) next = remaining.next();
if (next.done) break;
pending.push(next.value);
}
if (cursor % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const frame = pending[cursor];
if (visited.has(frame.frameId)) continue;
visited.add(frame.frameId);
for (const child of children.get(frame.frameId) ?? []) pending.push(child);
const doc = sources.get(frame.frameId);
if (!doc) continue;
const source = { target, frameId: frame.frameId };
const coordinates = snapshotCoordinates(
doc,
layoutUnitsPerCssPixel,
frame.frameId === rootFrameId
? { x: metrics.cssLayoutViewport?.pageX, y: metrics.cssLayoutViewport?.pageY }
: undefined,
);
let state: FrameProjectionState | null =
frame.frameId === rootFrameId
? viewportAvailable && coordinates
? {
status: "available",
projection: {
source,
geometry: { sourceClips: [], edges: [], topViewport: viewport },
},
}
: { status: "unavailable", source, reason: "snapshot-coordinates-unavailable" }
: (projections.get(frame.frameId) ?? null);
if (!state && frame.frameId !== rootFrameId)
issues.push({
target,
frameId: frame.frameId,
stage: "ownership",
reason: "frame-ownership-unresolved",
});
if (!coordinates && (!state || state.status === "available"))
state = { status: "unavailable", source, reason: "snapshot-coordinates-unavailable" };
projections.set(frame.frameId, state);
const projection = state?.status === "available" ? state.projection : null;
if (!projection || (target.sessionId && !targetProjection))
issues.push({
target,
frameId: frame.frameId,
stage: "geometry",
reason: "geometry-unavailable",
...(state && state.status !== "available" ? { projectionIssue: state } : {}),
});
// Only direct siblings share this pool. Keep normalization in its existing
// breadth-first order, independent of owner read completion order.
const siblings = (children.get(frame.frameId) ?? []).filter((child) =>
sources.has(child.frameId),
);
const edge = projection?.geometry.edges[0];
const clips = edge
? [edge.destinationQuad, ...(edge.destinationClips ?? [])]
: (projection?.geometry.sourceClips ?? []);
let next = 0;
let failure: unknown;
await Promise.all(
Array.from({ length: Math.min(4, siblings.length) }, async () => {
while (next < siblings.length && !signal?.aborted && !failure) {
const child = siblings[next++];
const childSource = { target, frameId: child.frameId };
if (!projection) {
projections.set(
child.frameId,
state && state.status !== "available"
? {
status: "blocked",
source: childSource,
cause: state.status === "blocked" ? state.cause : state,
}
: null,
);
continue;
}
if (child.ownerBackendNodeId === undefined) {
projections.set(child.frameId, null);
continue;
}
if (!snapshotCoordinates(sources.get(child.frameId)!, layoutUnitsPerCssPixel)) {
projections.set(child.frameId, {
status: "unavailable",
source: childSource,
reason: "snapshot-coordinates-unavailable",
});
continue;
}
try {
projections.set(
child.frameId,
await geometry.snapshotProjection(
childSource,
child.ownerBackendNodeId,
clips,
viewport,
),
);
} catch (error) {
failure ??= error;
}
}
}),
);
// Join active reads and their cleanup before returning cancellation.
if (failure) throw failure;
throwCaptureAborted(signal);
const normalized = await normalizeDocument(
doc,
strings,
{
frameId: frame.frameId,
ownerFrameBackendNodeId: frame.ownerBackendNodeId ?? null,
projection: state,
target,
...(target.sessionId ? { targetProjection } : {}),
coordinates,
},
signal,
);
result.push({ ...normalized, frame });
}
return result;
}
@@ -2,7 +2,10 @@ import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { readRecordingDocumentIdentity } from "@/shared/recording-document-identity";
import type { CapturedSurfaceProbe } from "./capture";
import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture";
import type {
FrameDocument as CapturedFrameDocument,
FrameOwnedAxNode as FrameAxNode,
} from "./frame-document";
/** Frame identity needed to resolve an `@eN` that lives in an iframe. */
export interface CaptureVomFrame {
@@ -61,8 +61,9 @@ export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGrap
dom,
sourceOrder: index,
excluded:
document.frameId === rootFrameId &&
input.excludedBackendNodeIds?.has(dom.backendNodeId) === true,
document.excludedBackendNodeIds?.has(dom.backendNodeId) === true ||
(document.frameId === rootFrameId &&
input.excludedBackendNodeIds?.has(dom.backendNodeId) === true),
};
nodes.set(id, node);
frameNodeIds.push(id);
@@ -90,9 +91,10 @@ export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGrap
ax,
sourceOrder: document.domNodes.length + index,
excluded:
document.frameId === rootFrameId &&
typeof ax.backendDOMNodeId === "number" &&
input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true,
(document.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true ||
(document.frameId === rootFrameId &&
input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true)),
});
frameNodeIds.push(id);
}
+291
View File
@@ -0,0 +1,291 @@
import { type CdpFrame, type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
import { createCaptureCheckpoint } from "./capture-abort";
import type { CapturedNode } from "./capture-types";
import type { DecodedDocument, DecodedNode } from "./facts";
export const REQUESTED_STYLES = [
"position",
"pointer-events",
"cursor",
"visibility",
"opacity",
] as const;
const STYLE_COL = Object.fromEntries(
REQUESTED_STYLES.map((name, index) => [name, index]),
) as Record<(typeof REQUESTED_STYLES)[number], number>;
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
interface SparseArray {
index: number[];
value: number[];
}
interface RareBooleanData {
index: number[];
}
export interface SnapshotDocument {
frameId?: string | number;
documentURL?: number;
scrollOffsetX?: number;
scrollOffsetY?: number;
nodes?: {
parentIndex?: number[];
nodeName?: number[];
nodeType?: number[];
backendNodeId?: number[];
attributes?: number[][];
/**
* Per-node text value (index into strings), set for `#text` / CDATA nodes.
* Element nodes carry -1. Same length as `backendNodeId`.
*/
nodeValue?: number[];
/** Maps node array index → index into `documents[]` for frame content. */
contentDocumentIndex?: SparseArray;
inputValue?: SparseArray;
textValue?: SparseArray;
inputChecked?: RareBooleanData;
optionSelected?: RareBooleanData;
};
layout?: {
nodeIndex?: number[];
styles?: number[][];
bounds?: number[][];
paintOrders?: number[];
};
}
export function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
if (typeof document.frameId === "string") return document.frameId || undefined;
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
return undefined;
}
export interface SnapshotReply {
strings?: string[];
documents?: SnapshotDocument[];
}
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
}
function snapshotFormState(
value: string | undefined,
defaultValue: string,
hasDefaultValue: boolean,
sensitive: boolean,
): CapturedNode["formState"] {
if (sensitive) {
if (value === undefined && !hasDefaultValue) return undefined;
return (value ?? defaultValue) === "" ? "empty" : "filled";
}
if (value === undefined) return undefined;
if (value === "") return "empty";
return value === defaultValue ? "default" : "filled";
}
function str(strings: string[], idx: number | undefined): string {
if (idx === undefined || idx < 0) return "";
return strings[idx] ?? "";
}
export function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
const out = new Map<number, number>();
if (!sparse?.index || !sparse.value) return out;
for (let i = 0; i < sparse.index.length; i++) {
const nodeIndex = sparse.index[i];
const docIndex = sparse.value[i];
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
}
return out;
}
export async function decodeDocument(
doc: SnapshotDocument,
strings: string[],
signal?: AbortSignal,
): Promise<DecodedDocument> {
const checkpoint = createCaptureCheckpoint(signal);
const dn = doc.nodes;
const dl = doc.layout;
if (!dn?.backendNodeId) {
return { nodes: [] };
}
const count = dn.backendNodeId.length;
const layoutByNode = new Map<number, number>();
for (let i = 0; i < (dl?.nodeIndex?.length ?? 0); i++) {
if (i % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
layoutByNode.set(dl!.nodeIndex![i], i);
}
const inputValues = sparseIndexMap(dn.inputValue);
const textValues = sparseIndexMap(dn.textValue);
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
// Collect visible text from #text child nodes so element CapturedNodes
// carry a textContent value usable as a button/link label fallback.
// nodeValue is a parallel array: string index for text nodes, -1 otherwise.
const nodeTextContent = new Map<number, string[]>();
if (dn.nodeValue) {
for (let n = 0; n < count; n++) {
if (n % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const nvIdx = dn.nodeValue[n] ?? -1;
if (nvIdx < 0) continue;
const text = str(strings, nvIdx).trim();
if (!text) continue;
const parentIdx = dn.parentIndex?.[n] ?? -1;
if (parentIdx >= 0) {
const existing = nodeTextContent.get(parentIdx);
if (existing) existing.push(text);
else nodeTextContent.set(parentIdx, [text]);
}
}
}
// Decode fields without assigning a coordinate projection or semantic policy.
const nodes: DecodedNode[] = [];
for (let n = 0; n < count; n++) {
if (n % 256 === 0) {
const pending = checkpoint();
if (pending) await pending;
}
const backendNodeId = dn.backendNodeId[n];
const parentIdx = dn.parentIndex?.[n] ?? -1;
const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null;
const tag = str(strings, dn.nodeName?.[n]).toLowerCase();
const attrs: Record<string, string> = {};
const pairs = dn.attributes?.[n] ?? [];
for (let a = 0; a + 1 < pairs.length; a += 2) {
attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]);
}
const li = layoutByNode.get(n);
const styleRow = li === undefined ? [] : (dl?.styles?.[li] ?? []);
const styles: Record<string, string> = {};
for (const name of REQUESTED_STYLES) styles[name] = str(strings, styleRow[STYLE_COL[name]]);
const layout =
li === undefined
? undefined
: {
boundsSpace: "snapshot-document-layout" as const,
bounds: dl?.bounds?.[li],
styles,
};
const textContent = nodeTextContent.get(n)?.join(" ");
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
const rawFormValue =
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
const sensitive = isSensitiveFormControl({ tag, attrs });
const formDefaultValue = attrs.value ?? "";
const formValue = sensitive ? undefined : rawFormValue;
const formState = snapshotFormState(
rawFormValue,
formDefaultValue,
Object.prototype.hasOwnProperty.call(attrs, "value"),
sensitive,
);
if (sensitive) delete attrs.value;
nodes.push({
backendNodeId,
nodeType: dn.nodeType?.[n],
parentBackendNodeId,
tag,
attrs,
layout,
paintOrder: li === undefined ? 0 : (dl?.paintOrders?.[li] ?? 0),
position: styles.position || "static",
pointerEvents: styles["pointer-events"] || "auto",
cursor: styles.cursor || "auto",
textContent,
...(formValue !== undefined ? { formValue } : {}),
...(tag === "input" || tag === "textarea"
? {
formPlaceholder: attrs.placeholder ?? "",
...(!sensitive ? { formDefaultValue } : {}),
...(formState ? { formState } : {}),
}
: {}),
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
});
}
return { nodes };
}
/** Snapshot membership and same-target edges are current response evidence.
* The earlier graph only supplies compatible metadata and cross-target edges. */
export function describeSnapshotFrames(
snapshot: SnapshotReply,
target: CdpTarget,
hintById: ReadonlyMap<string, CdpFrame>,
pageUrl?: string,
): { frames: CdpFrame[]; ids: Set<string>; rootFrameId?: string } {
const raw = snapshot.documents ?? [];
const strings = snapshot.strings ?? [];
const ids = new Set<string>();
const invalid = new Set<string>();
const frames = raw.map((doc) => {
const frameId = snapshotFrameId(doc, strings);
if (!frameId) return undefined;
if (ids.has(frameId)) invalid.add(frameId);
ids.add(frameId);
const hint = hintById.get(frameId);
const compatible =
hint && cdpTargetKey(hint.target) === cdpTargetKey(target) ? hint : undefined;
const url =
(doc.documentURL === undefined ? undefined : strings[doc.documentURL]) || compatible?.url;
return { frameId, target, ...(url ? { url } : {}) } as CdpFrame;
});
const root = frames[0];
if (root) {
const hint = hintById.get(root.frameId);
const parent = hint?.parentFrameId ? hintById.get(hint.parentFrameId) : undefined;
if (
target.sessionId &&
hint &&
cdpTargetKey(hint.target) === cdpTargetKey(target) &&
parent &&
cdpTargetKey(parent.target) !== cdpTargetKey(target)
) {
root.parentFrameId = parent.frameId;
root.ownerBackendNodeId = hint.ownerBackendNodeId;
}
if (!target.sessionId && !root.url && pageUrl) root.url = pageUrl;
}
const owned = new Set<string>();
for (let i = 0; i < raw.length; i++) {
const edges = raw[i].nodes?.contentDocumentIndex;
const parent = frames[i];
if (!parent || !edges) continue;
for (let e = 0; e < edges.index.length; e++) {
const child = frames[edges.value[e]];
const owner = raw[i].nodes?.backendNodeId?.[edges.index[e]];
if (!child || owner === undefined) continue;
if (child === root || child === parent || owned.has(child.frameId)) {
invalid.add(child.frameId);
continue;
}
owned.add(child.frameId);
child.parentFrameId = parent.frameId;
child.ownerBackendNodeId = owner;
}
}
return {
frames: frames.filter((frame): frame is CdpFrame => !!frame && !invalid.has(frame.frameId)),
ids,
rootFrameId: root?.frameId,
};
}