mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
refactor(observation): isolate document processing from capture orchestration
This commit is contained in:
Ljy-0827
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,3 @@
|
||||
import type { CapturedSceneInput } from "./vom/facts";
|
||||
// Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`,
|
||||
// and semantic `tool.observe` (design §7). Each handler resolves the target
|
||||
// tab (defaulting to the Agent Window's active tab when omitted) and
|
||||
@@ -36,6 +35,7 @@ import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import { rpcError } from "./errors";
|
||||
import { resolveNodeGeometry } from "./frame-geometry";
|
||||
import { screenshotPageRect } from "./geometry/coordinate-types";
|
||||
import { cssViewport, GeometryContext } from "./geometry/frame-context";
|
||||
import {
|
||||
type ChromeTabsApi,
|
||||
enforceToolTargetScope,
|
||||
@@ -49,9 +49,15 @@ import {
|
||||
type ToolEffect,
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import { type CapturedNode, type CapturedSurfaceProbe, probeHoverSurfaces } from "./vom/capture";
|
||||
import { captureObservationFacts, semanticCapture } from "./vom/capture-coordinator";
|
||||
import type { FrameDocument as CapturedFrameDocument } from "./vom/frame-document";
|
||||
import {
|
||||
type CapturedNode,
|
||||
type CapturedSurfaceProbe,
|
||||
type CapturedViewModel,
|
||||
captureViewModel,
|
||||
collectOverlayExcludedBackendIds,
|
||||
probeHoverSurfaces,
|
||||
} from "./vom/capture";
|
||||
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
|
||||
import { withOverlayBypass } from "./vom/hover-perception";
|
||||
import { probeTooltipNames } from "./vom/name-enrichment";
|
||||
import {
|
||||
@@ -525,7 +531,7 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A
|
||||
|
||||
function buildConditionalSurfaces(
|
||||
nodes: VomNode[],
|
||||
captured: CapturedSceneInput,
|
||||
captured: CapturedViewModel,
|
||||
probes: CapturedSurfaceProbe[],
|
||||
): CondSurface[] {
|
||||
if (probes.length === 0) return [];
|
||||
@@ -655,9 +661,163 @@ export interface BuildVomSceneOptions {
|
||||
|
||||
export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>;
|
||||
|
||||
function legacyFrameDocuments(
|
||||
axNodes: CdpAxNode[],
|
||||
captured: CapturedViewModel,
|
||||
pageUrl?: string,
|
||||
rootTarget: VomFrameDocument["target"] = { tabId: 0 },
|
||||
): VomFrameDocument[] {
|
||||
const rootFrameId = captured.rootFrameId ?? "root";
|
||||
const documents: VomFrameDocument[] = [
|
||||
{
|
||||
frameId: rootFrameId,
|
||||
contextScopeId: rootFrameId,
|
||||
target: rootTarget,
|
||||
...(pageUrl ? { url: pageUrl } : {}),
|
||||
axNodes: [],
|
||||
domNodes: captured.nodes.map((node) => ({ ...node, frameId: node.frameId ?? rootFrameId })),
|
||||
excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(rootFrameId),
|
||||
},
|
||||
];
|
||||
const pending = [...captured.iframeNodes.entries()];
|
||||
let progress = true;
|
||||
let nextSyntheticFrame = 1;
|
||||
while (pending.length > 0 && progress) {
|
||||
progress = false;
|
||||
for (let index = pending.length - 1; index >= 0; index -= 1) {
|
||||
const [ownerBackendNodeId, domNodes] = pending[index];
|
||||
const parent = documents.find((document) =>
|
||||
document.domNodes.some((node) => node.backendNodeId === ownerBackendNodeId),
|
||||
);
|
||||
if (!parent) continue;
|
||||
const frameId =
|
||||
domNodes.find((node) => node.frameId)?.frameId ?? `legacy-frame-${nextSyntheticFrame++}`;
|
||||
documents.push({
|
||||
frameId,
|
||||
parentFrameId: parent.frameId,
|
||||
ownerBackendNodeId,
|
||||
contextScopeId: frameId,
|
||||
target: parent.target,
|
||||
axNodes: [],
|
||||
domNodes: domNodes.map((node) => ({ ...node, frameId: node.frameId ?? frameId })),
|
||||
excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(frameId),
|
||||
});
|
||||
pending.splice(index, 1);
|
||||
progress = true;
|
||||
}
|
||||
}
|
||||
|
||||
const documentByFrameId = new Map(documents.map((document) => [document.frameId, document]));
|
||||
const backendFrame = new Map<number, string>();
|
||||
const childFrameByOwner = new Map<number, string>();
|
||||
for (const document of documents) {
|
||||
if (document.ownerBackendNodeId !== undefined) {
|
||||
childFrameByOwner.set(document.ownerBackendNodeId, document.frameId);
|
||||
}
|
||||
for (const node of document.domNodes) backendFrame.set(node.backendNodeId, document.frameId);
|
||||
}
|
||||
const axById = new Map(axNodes.map((node) => [node.nodeId, node]));
|
||||
const ownership = new Map<string, string>();
|
||||
const resolving = new Set<string>();
|
||||
const frameForAx = (node: CdpAxNode): string => {
|
||||
const cached = ownership.get(node.nodeId);
|
||||
if (cached) return cached;
|
||||
let frameId: string | undefined;
|
||||
if (node.frameId && documentByFrameId.has(node.frameId)) frameId = node.frameId;
|
||||
if (!frameId && typeof node.backendDOMNodeId === "number") {
|
||||
frameId = backendFrame.get(node.backendDOMNodeId);
|
||||
}
|
||||
if (!frameId && node.parentId && !resolving.has(node.nodeId)) {
|
||||
const parent = axById.get(node.parentId);
|
||||
if (parent) {
|
||||
frameId =
|
||||
typeof parent.backendDOMNodeId === "number"
|
||||
? childFrameByOwner.get(parent.backendDOMNodeId)
|
||||
: undefined;
|
||||
if (!frameId) {
|
||||
resolving.add(node.nodeId);
|
||||
frameId = frameForAx(parent);
|
||||
resolving.delete(node.nodeId);
|
||||
}
|
||||
}
|
||||
}
|
||||
frameId ??= rootFrameId;
|
||||
ownership.set(node.nodeId, frameId);
|
||||
return frameId;
|
||||
};
|
||||
for (const node of axNodes) frameForAx(node);
|
||||
for (const node of axNodes) {
|
||||
const frameId = ownership.get(node.nodeId) ?? rootFrameId;
|
||||
const document = documentByFrameId.get(frameId) ?? documents[0];
|
||||
document.axNodes.push({
|
||||
...node,
|
||||
frameId,
|
||||
...(node.parentId && ownership.get(node.parentId) === frameId
|
||||
? { parentId: node.parentId }
|
||||
: { parentId: undefined }),
|
||||
...(node.childIds
|
||||
? { childIds: node.childIds.filter((childId) => ownership.get(childId) === frameId) }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
return documents;
|
||||
}
|
||||
|
||||
function withLegacyBackendIds(scene: VomScene): VomScene {
|
||||
const used = new Set<number>();
|
||||
const idMap = new Map<number, number>();
|
||||
let nextVirtualId = -1;
|
||||
for (const node of scene.nodes) {
|
||||
const preferred = node.backendNodeId;
|
||||
const id = preferred !== undefined && !used.has(preferred) ? preferred : nextVirtualId--;
|
||||
used.add(id);
|
||||
idMap.set(node.id, id);
|
||||
}
|
||||
return {
|
||||
...scene,
|
||||
nodes: scene.nodes.map((node) => ({
|
||||
...node,
|
||||
id: idMap.get(node.id) as number,
|
||||
parentId: node.parentId === null ? null : (idMap.get(node.parentId) ?? null),
|
||||
...(node.domParentId !== undefined
|
||||
? { domParentId: node.domParentId === null ? null : (idMap.get(node.domParentId) ?? null) }
|
||||
: {}),
|
||||
...(node.domAncestorIds
|
||||
? { domAncestorIds: node.domAncestorIds.flatMap((id) => idMap.get(id) ?? []) }
|
||||
: {}),
|
||||
})),
|
||||
...(scene.surfaces
|
||||
? {
|
||||
surfaces: scene.surfaces.map((surface) => ({
|
||||
...surface,
|
||||
triggerId: idMap.get(surface.triggerId) ?? surface.triggerId,
|
||||
})),
|
||||
}
|
||||
: {}),
|
||||
...(scene.activeScopeBlocks
|
||||
? {
|
||||
activeScopeBlocks: scene.activeScopeBlocks.map((block) => ({
|
||||
...block,
|
||||
triggerId: idMap.get(block.triggerId) ?? block.triggerId,
|
||||
})),
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function buildVomScene(
|
||||
axNodes: CdpAxNode[],
|
||||
captured: CapturedViewModel,
|
||||
options: BuildVomSceneOptions = {},
|
||||
): VomScene {
|
||||
return withLegacyBackendIds(
|
||||
buildFrameVomScene(legacyFrameDocuments(axNodes, captured, options.pageUrl), captured, options),
|
||||
);
|
||||
}
|
||||
|
||||
export function buildFrameVomScene(
|
||||
documents: VomFrameDocument[],
|
||||
captured: CapturedSceneInput,
|
||||
captured: CapturedViewModel,
|
||||
options: BuildVomSceneOptions = {},
|
||||
): VomScene {
|
||||
const scene = buildSemanticVomScene({
|
||||
@@ -673,7 +833,7 @@ export function buildFrameVomScene(
|
||||
function attachCapturedSceneAnnotations(
|
||||
scene: VomScene,
|
||||
documents: VomFrameDocument[],
|
||||
captured: CapturedSceneInput,
|
||||
captured: CapturedViewModel,
|
||||
surfaceProbes: CapturedSurfaceProbe[],
|
||||
): VomScene {
|
||||
const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId);
|
||||
@@ -823,6 +983,45 @@ export async function handleGetHtml(
|
||||
}
|
||||
}
|
||||
|
||||
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
|
||||
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
||||
}
|
||||
|
||||
async function fallbackCapturedViewModel(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
geometry: GeometryContext,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CapturedViewModel> {
|
||||
throwIfAborted(signal, "observation");
|
||||
let viewport = { width: 0, height: 0 };
|
||||
try {
|
||||
const metrics = await geometry.layoutMetrics({ tabId });
|
||||
throwIfAborted(signal, "observation");
|
||||
const source = cssViewport(metrics);
|
||||
viewport = { width: source.width, height: source.height };
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
}
|
||||
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal);
|
||||
throwIfAborted(signal, "observation");
|
||||
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
async function captureForVom(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
options: CaptureVomObservationOptions,
|
||||
geometry: GeometryContext,
|
||||
): Promise<CapturedViewModel> {
|
||||
try {
|
||||
return await captureViewModel(cdp, tabId, { signal: options.signal, geometry });
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
return fallbackCapturedViewModel(cdp, tabId, geometry, options.signal);
|
||||
}
|
||||
}
|
||||
|
||||
export interface HoverProbeOutcome {
|
||||
/** Whether any active hover was dispatched during this observation. */
|
||||
performed: boolean;
|
||||
@@ -855,7 +1054,7 @@ const NO_HOVER_PROBES: HoverProbeOutcome = {
|
||||
async function runHoverProbes(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
captured: CapturedSceneInput,
|
||||
captured: CapturedViewModel,
|
||||
documents: VomFrameDocument[],
|
||||
staticSemantics: ReturnType<typeof resolveSemanticGraph>,
|
||||
options: CaptureVomObservationOptions,
|
||||
@@ -894,9 +1093,21 @@ export async function captureVomObservation(
|
||||
throwIfAborted(options.signal, "observation");
|
||||
await cdp.ensureAttachedToUrl?.(tabId, url);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const facts = await captureObservationFacts<CdpAxNode>(cdp, tabId, options.signal, url);
|
||||
const { captured, documents: normalizedDocuments } = semanticCapture(facts);
|
||||
const geometry = new GeometryContext(cdp, tabId, undefined, options.signal);
|
||||
const captured = await captureForVom(cdp, tabId, options, geometry);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const documents = await captureFrameData<CdpAxNode>(
|
||||
cdp,
|
||||
tabId,
|
||||
captured,
|
||||
options.signal,
|
||||
geometry,
|
||||
);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const normalizedDocuments =
|
||||
documents.length === 1 && captured.iframeNodes.size > 0
|
||||
? legacyFrameDocuments(documents[0].axNodes, captured, url, documents[0].target)
|
||||
: documents;
|
||||
const semanticGraph = buildSemanticGraph({
|
||||
documents: normalizedDocuments,
|
||||
viewport: captured.viewport,
|
||||
@@ -926,7 +1137,7 @@ export async function captureVomObservation(
|
||||
);
|
||||
// Reserve space using the renderer's character-based token estimate. Like VOM
|
||||
// headers, this integrity notice remains visible even under a tiny token budget.
|
||||
const geometryNotice = facts.issues.some((issue) => issue.stage === "geometry")
|
||||
const geometryNotice = captured.frameGeometryIssues?.length
|
||||
? "@warning iframe geometry incomplete: some frame content has no top-level coordinates.\n"
|
||||
: "";
|
||||
const rendered = renderVom(decoratedScene, {
|
||||
|
||||
@@ -1,914 +0,0 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import { cdpTargetKey } from "@/browser-driver/frame-graph";
|
||||
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
|
||||
import type { CdpRunner } from "../../shared";
|
||||
import { captureObservationFacts, semanticCapture } from "../capture-coordinator";
|
||||
import { buildSemanticGraph } from "../semantic-graph/build";
|
||||
import { REQUESTED_STYLES } from "../snapshot";
|
||||
|
||||
function fixture(
|
||||
options: { frames?: CdpFrame[]; fail?: string; omitDocument?: string; overlay?: string } = {},
|
||||
) {
|
||||
const frames = options.frames ?? [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "same",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 3,
|
||||
target: { tabId: 4 },
|
||||
},
|
||||
{
|
||||
frameId: "nested",
|
||||
parentFrameId: "same",
|
||||
ownerBackendNodeId: 13,
|
||||
target: { tabId: 4 },
|
||||
},
|
||||
{
|
||||
frameId: "remote",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 4,
|
||||
target: { tabId: 4, sessionId: "remote" },
|
||||
},
|
||||
];
|
||||
const elements = new Map(
|
||||
frames.map((frame, i) => [frame.frameId, frame.target.sessionId ? 1 : i * 10 + 1]),
|
||||
);
|
||||
const logs: Array<{ target: CdpTarget; method: string; params: Record<string, unknown> }> = [];
|
||||
const send = vi.fn(
|
||||
async <T>(target: CdpTarget, method: string, params: object = {}): Promise<T> => {
|
||||
logs.push({ target, method, params: params as Record<string, unknown> });
|
||||
const args = params as Record<string, string | number>;
|
||||
if (options.fail === `${target.sessionId ?? "main"}:${method}`)
|
||||
throw new Error("fixture failure");
|
||||
let result: unknown = {};
|
||||
if (method === "Page.getLayoutMetrics")
|
||||
result = { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
expect((params as { computedStyles: unknown }).computedStyles).toEqual(REQUESTED_STYLES);
|
||||
result = {
|
||||
strings: [
|
||||
"#document",
|
||||
"html",
|
||||
"button",
|
||||
OVERLAY_HOST_MARKER_ATTR,
|
||||
"",
|
||||
"visible",
|
||||
"1",
|
||||
"static",
|
||||
"auto",
|
||||
],
|
||||
documents: frames
|
||||
.filter(
|
||||
(frame) =>
|
||||
cdpTargetKey(frame.target) === cdpTargetKey(target) &&
|
||||
frame.frameId !== options.omitDocument,
|
||||
)
|
||||
.map((frame) => {
|
||||
const element = elements.get(frame.frameId)!;
|
||||
return {
|
||||
frameId: frame.frameId,
|
||||
nodes: {
|
||||
backendNodeId: [element - 1, element + 10000, element, element + 1],
|
||||
nodeName: [0, 1, 1, 2],
|
||||
nodeType: [9, 10, 1, 1],
|
||||
parentIndex: [-1, 0, 0, 2],
|
||||
attributes: [[], [], [], frame.frameId === options.overlay ? [3, 4] : []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [2, 3],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[10, 20, 100, 40],
|
||||
],
|
||||
styles: [
|
||||
[7, 8, 8, 5, 6],
|
||||
[7, 8, 8, 5, 6],
|
||||
],
|
||||
},
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
if (method === "Accessibility.getFullAXTree") {
|
||||
const frameId = String(args.frameId);
|
||||
result = {
|
||||
nodes: [
|
||||
{
|
||||
nodeId: `${frameId}-button`,
|
||||
frameId,
|
||||
backendDOMNodeId: elements.get(frameId)! + 1,
|
||||
role: { value: "button" },
|
||||
name: { value: frameId },
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
return result as T;
|
||||
},
|
||||
);
|
||||
const cdp: CdpRunner = {
|
||||
getFrameGraph: async () => ({ rootFrameId: frames[0].frameId, frames }),
|
||||
send: (tabId, method, params) =>
|
||||
(send as NonNullable<CdpRunner["sendToTarget"]>)({ tabId }, method, params),
|
||||
sendToTarget: send as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
};
|
||||
return { cdp, logs, elements };
|
||||
}
|
||||
|
||||
describe("captureObservationFacts", () => {
|
||||
it("collects each target once and scopes equal backend IDs", async () => {
|
||||
const { cdp, logs } = fixture();
|
||||
const facts = await captureObservationFacts(cdp, 4);
|
||||
expect(logs.filter((call) => call.method === "DOMSnapshot.captureSnapshot")).toHaveLength(2);
|
||||
expect(logs.filter((call) => call.method === "Accessibility.enable")).toHaveLength(2);
|
||||
expect(logs.filter((call) => call.method === "Accessibility.getFullAXTree")).toHaveLength(4);
|
||||
const main = facts.documents.find((doc) => doc.frame.frameId === "main")!;
|
||||
const remote = facts.documents.find((doc) => doc.frame.frameId === "remote")!;
|
||||
expect(main.index.nodes.get(2)).not.toBe(remote.index.nodes.get(2));
|
||||
expect(main.index.nodes.get(2)?.parentBackendNodeId).toBe(1);
|
||||
expect(main.index.nodes.get(10001)?.nodeType).toBe(10);
|
||||
expect(remote.axNodes[0].frameId).toBe("remote");
|
||||
expect(main.index.nodes.get(2)).toBe(main.domNodes.find((node) => node.backendNodeId === 2));
|
||||
expect(facts.finishedAt).toBeGreaterThanOrEqual(facts.startedAt);
|
||||
});
|
||||
|
||||
it("does not retry failed or missing documents and retains valid AX-only semantics", async () => {
|
||||
const { cdp, logs } = fixture({
|
||||
fail: "remote:DOMSnapshot.captureSnapshot",
|
||||
omitDocument: "same",
|
||||
});
|
||||
const facts = await captureObservationFacts(cdp, 4);
|
||||
const remote = facts.documents.find((doc) => doc.frame.frameId === "remote")!;
|
||||
expect(remote.domNodes).toHaveLength(0);
|
||||
expect(remote.axNodes).toHaveLength(1);
|
||||
expect(logs.filter((call) => call.method === "DOMSnapshot.captureSnapshot")).toHaveLength(2);
|
||||
expect(facts.issues).toContainEqual(
|
||||
expect.objectContaining({ frameId: "same", stage: "dom", reason: "capture-unavailable" }),
|
||||
);
|
||||
expect(facts.issues).toContainEqual(
|
||||
expect.objectContaining({ target: { tabId: 4, sessionId: "remote" }, stage: "dom" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("excludes overlay AX in its own document without excluding equal IDs elsewhere", async () => {
|
||||
const facts = await captureObservationFacts(fixture({ overlay: "remote" }).cdp, 4);
|
||||
const { documents } = semanticCapture(facts);
|
||||
const graph = buildSemanticGraph({
|
||||
documents,
|
||||
viewport: facts.viewport,
|
||||
rootFrameId: facts.rootFrameId,
|
||||
});
|
||||
expect(
|
||||
[...graph.nodes.values()].find(
|
||||
(node) => node.frameId === "remote" && node.backendNodeId === 2,
|
||||
)?.excluded,
|
||||
).toBe(true);
|
||||
expect(
|
||||
[...graph.nodes.values()].find((node) => node.frameId === "main" && node.backendNodeId === 2)
|
||||
?.excluded,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"cancel",
|
||||
"worker-abort",
|
||||
] as const)("joins target cleanup and stops claiming targets after %s", async (mode) => {
|
||||
const frames = Array.from({ length: 5 }, (_, i) => ({
|
||||
frameId: `f${i}`,
|
||||
target: { tabId: 4, ...(i ? { sessionId: `s${i}` } : {}) },
|
||||
}));
|
||||
const { cdp } = fixture({ frames });
|
||||
const original = cdp.sendToTarget!;
|
||||
const pending = new Map<string, { resolve: () => void; reject: (error: Error) => void }>();
|
||||
const calls: Array<{ target: string; method: string }> = [];
|
||||
let release: (() => void) | undefined;
|
||||
let released = false;
|
||||
const send: NonNullable<CdpRunner["sendToTarget"]> = async <T>(
|
||||
target: CdpTarget,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => {
|
||||
calls.push({ target: target.sessionId ?? "main", method });
|
||||
if (method === "DOMSnapshot.captureSnapshot" && target.sessionId) {
|
||||
await new Promise<void>((resolve, reject) =>
|
||||
pending.set(target.sessionId!, { resolve, reject }),
|
||||
);
|
||||
}
|
||||
if (method === "Runtime.releaseObjectGroup") {
|
||||
await new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
released = true;
|
||||
}
|
||||
const result = await original<T>(target, method, params);
|
||||
if (method === "DOMSnapshot.captureSnapshot" && !target.sessionId)
|
||||
(result as { strings: string[] }).strings[2] = "input";
|
||||
return result;
|
||||
};
|
||||
cdp.sendToTarget = send;
|
||||
cdp.send = (tabId, method, params) => send({ tabId }, method, params);
|
||||
const controller = new AbortController();
|
||||
let settled = false;
|
||||
const capture = captureObservationFacts(cdp, 4, controller.signal);
|
||||
const rejected = expect(capture).rejects.toMatchObject({ name: "AbortError" });
|
||||
void capture.then(
|
||||
() => {
|
||||
settled = true;
|
||||
},
|
||||
() => {
|
||||
settled = true;
|
||||
},
|
||||
);
|
||||
await vi.waitFor(() => {
|
||||
expect(pending.size).toBe(3);
|
||||
expect(release).toBeDefined();
|
||||
});
|
||||
if (mode === "cancel") controller.abort();
|
||||
// Cancellation may race with an ordinary transport rejection. Without an
|
||||
// external signal, an escaping AbortError still terminates the worker pool.
|
||||
pending
|
||||
.get("s1")!
|
||||
.reject(
|
||||
mode === "cancel"
|
||||
? new Error("transport closed")
|
||||
: new DOMException("target aborted", "AbortError"),
|
||||
);
|
||||
pending.get("s2")!.resolve();
|
||||
pending.get("s3")!.resolve();
|
||||
await vi.waitFor(() => {
|
||||
if (mode === "worker-abort")
|
||||
expect(
|
||||
calls.some(
|
||||
(call) => call.target === "s3" && call.method === "Accessibility.getFullAXTree",
|
||||
),
|
||||
).toBe(true);
|
||||
else
|
||||
expect(
|
||||
calls.some(
|
||||
(call) => call.target === "s3" && call.method === "DOMSnapshot.captureSnapshot",
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
// Allow worker rejection and the outer promise chain to run while cleanup
|
||||
// is deliberately held. The old fail-fast pool settles here.
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
expect(settled).toBe(false);
|
||||
expect(released).toBe(false);
|
||||
expect(calls.some((call) => call.target === "s4")).toBe(false);
|
||||
if (mode === "cancel")
|
||||
expect(
|
||||
calls.some(
|
||||
(call) => call.method === "Accessibility.enable" || call.method === "DOM.getDocument",
|
||||
),
|
||||
).toBe(false);
|
||||
release!();
|
||||
await rejected;
|
||||
expect(released).toBe(true);
|
||||
expect(calls.some((call) => call.target === "s4")).toBe(false);
|
||||
});
|
||||
|
||||
it("bounds collection across many targets and stops scheduling after cancellation", async () => {
|
||||
const frames = Array.from({ length: 12 }, (_, i) => ({
|
||||
frameId: `f${i}`,
|
||||
target: { tabId: 4, ...(i ? { sessionId: `s${i}` } : {}) },
|
||||
}));
|
||||
const { cdp } = fixture({ frames });
|
||||
let active = 0,
|
||||
peak = 0;
|
||||
const original = cdp.sendToTarget!;
|
||||
const send: NonNullable<CdpRunner["sendToTarget"]> = async (target, method, params) => {
|
||||
active++;
|
||||
peak = Math.max(peak, active);
|
||||
try {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1));
|
||||
return await original(target, method, params);
|
||||
} finally {
|
||||
active--;
|
||||
}
|
||||
};
|
||||
cdp.sendToTarget = send;
|
||||
cdp.send = (tabId, method, params) => send({ tabId }, method, params);
|
||||
await captureObservationFacts(cdp, 4);
|
||||
// This fixture has no frame edges, so no concurrent owner requests.
|
||||
expect(peak).toBeLessThanOrEqual(4);
|
||||
expect(peak).toBeGreaterThan(1);
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
await expect(captureObservationFacts(cdp, 4, controller.signal)).rejects.toMatchObject({
|
||||
name: "AbortError",
|
||||
});
|
||||
expect(active).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
function childSnapshot(frameId: string, backendNodeId: number) {
|
||||
const strings = [frameId, "body", "button", "static", "auto", "pointer"];
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
frameId,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [1, 2],
|
||||
backendNodeId: [backendNodeId - 1, backendNodeId],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[3, 4, 4],
|
||||
[3, 4, 5],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 300, 200],
|
||||
[10, 20, 100, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe("OOPIF capture", () => {
|
||||
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
|
||||
const sendToTarget = vi.fn(async (target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return target.sessionId === "left-session"
|
||||
? childSnapshot("left", 101)
|
||||
: childSnapshot("right", 201);
|
||||
}
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
throw new Error(`unexpected ${method}`);
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method, params) => {
|
||||
if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("main", 1);
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
if (method === "DOM.getBoxModel") {
|
||||
const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId;
|
||||
const x = backendNodeId === 10 ? 50 : 500;
|
||||
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
|
||||
}
|
||||
throw new Error(`unexpected root ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "left",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "left-session" },
|
||||
},
|
||||
{
|
||||
frameId: "right",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const { documents: trees } = semanticCapture(await captureObservationFacts(cdp, 4));
|
||||
|
||||
expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]);
|
||||
expect(
|
||||
trees
|
||||
.find((doc) => doc.frameId === "left")
|
||||
?.domNodes?.find((node) => node.backendNodeId === 101)?.rect,
|
||||
).toEqual({ x: 60, y: 120, w: 100, h: 40 });
|
||||
expect(
|
||||
trees
|
||||
.find((doc) => doc.frameId === "right")
|
||||
?.domNodes?.find((node) => node.backendNodeId === 201)?.rect,
|
||||
).toEqual({ x: 510, y: 120, w: 100, h: 40 });
|
||||
expect(
|
||||
new Map(
|
||||
trees
|
||||
.filter((doc) => doc.ownerBackendNodeId !== undefined)
|
||||
.map((doc) => [doc.frameId, doc.ownerBackendNodeId]),
|
||||
),
|
||||
).toEqual(
|
||||
new Map([
|
||||
["left", 10],
|
||||
["right", 20],
|
||||
]),
|
||||
);
|
||||
expect(
|
||||
new Map(
|
||||
trees.filter((doc) => doc.parentFrameId).map((doc) => [doc.frameId, doc.parentFrameId]),
|
||||
),
|
||||
).toEqual(
|
||||
new Map([
|
||||
["left", "main"],
|
||||
["right", "main"],
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("retains nested owner failures when merging a captured OOPIF", async () => {
|
||||
const child = childSnapshot("child", 101);
|
||||
const nested = childSnapshot("nested", 201);
|
||||
const document = child.documents[0];
|
||||
const snapshot = {
|
||||
strings: child.strings,
|
||||
documents: [
|
||||
{
|
||||
...document,
|
||||
nodes: { ...document.nodes, contentDocumentIndex: { index: [1], value: [1] } },
|
||||
},
|
||||
nested.documents[0],
|
||||
],
|
||||
};
|
||||
const reply = async (_target: unknown, method: string) => {
|
||||
if (method === "Page.getLayoutMetrics")
|
||||
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200 } };
|
||||
if (method === "DOMSnapshot.captureSnapshot")
|
||||
return typeof _target === "number" ? childSnapshot("main", 11) : snapshot;
|
||||
if (method === "DOM.getBoxModel")
|
||||
return { model: { content: [50, 100, 350, 100, 350, 300, 50, 300] } };
|
||||
if (method === "DOM.resolveNode") throw new Error("nested owner replaced");
|
||||
return {};
|
||||
};
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(reply) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(reply) as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "nested",
|
||||
parentFrameId: "child",
|
||||
ownerBackendNodeId: 101,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
const captured = await captureObservationFacts(cdp, 4);
|
||||
expect(
|
||||
captured.issues
|
||||
.filter((issue) => issue.projectionIssue)
|
||||
.map((issue) => issue.projectionIssue),
|
||||
).toEqual([
|
||||
{
|
||||
status: "unavailable",
|
||||
source: { target: { tabId: 4, sessionId: "child-session" }, frameId: "nested" },
|
||||
ownerBackendNodeId: 101,
|
||||
},
|
||||
]);
|
||||
expect(
|
||||
captured.documents.find((doc) => doc.frame.frameId === "nested")?.domNodes[1],
|
||||
).toMatchObject({
|
||||
rect: null,
|
||||
localRect: { x: 10, y: 20, w: 100, h: 40 },
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps OOPIF semantics when viewport projection is unavailable", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("main", 1);
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable");
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
|
||||
}
|
||||
throw new Error(`unexpected root ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") throw new Error("viewport unavailable");
|
||||
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("child", 101);
|
||||
if (method === "Accessibility.getFullAXTree") {
|
||||
return {
|
||||
nodes: [
|
||||
{
|
||||
nodeId: "button",
|
||||
backendDOMNodeId: 101,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Continue" },
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const { documents } = semanticCapture(await captureObservationFacts(cdp, 4));
|
||||
const childDocument = documents.find((document) => document.frameId === "child");
|
||||
|
||||
expect(childDocument?.axNodes).toEqual([
|
||||
expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }),
|
||||
]);
|
||||
expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual(
|
||||
expect.objectContaining({ rect: null, localRect: { x: 10, y: 20, w: 100, h: 40 } }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// Six sibling frames and one nested frame exercise scheduling through the real capture entry.
|
||||
function siblingCaptureFixture(
|
||||
beforeReply: (method: string, params: Record<string, unknown>) => Promise<void> = async () => {},
|
||||
) {
|
||||
const document = (id: number, owners: number[], childIndexes: number[]) => ({
|
||||
frameId: `frame-${id}`,
|
||||
nodes: {
|
||||
parentIndex: [-1, ...owners.map(() => 0)],
|
||||
nodeName: [0, ...owners.map(() => 1)],
|
||||
backendNodeId: [1000 + id, ...owners],
|
||||
attributes: [[], ...owners.map(() => [])],
|
||||
contentDocumentIndex: { index: owners.map((_, i) => i + 1), value: childIndexes },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, ...owners.map((_, i) => i + 1)],
|
||||
bounds: [[0, 0, 200, 100], ...owners.map(() => [0, 0, 200, 100])],
|
||||
},
|
||||
});
|
||||
const snapshot = {
|
||||
strings: ["body", "iframe"],
|
||||
documents: [
|
||||
document(0, [100, 101, 102, 103, 104, 105], [1, 2, 3, 4, 5, 6]),
|
||||
document(1, [200], [7]),
|
||||
...Array.from({ length: 6 }, (_, i) => document(i + 2, [], [])),
|
||||
],
|
||||
};
|
||||
let active = 0;
|
||||
let peak = 0;
|
||||
const send = vi.fn(async (_tabId: number, method: string, params: object = {}) => {
|
||||
const args = params as Record<string, unknown>;
|
||||
active++;
|
||||
peak = Math.max(peak, active);
|
||||
try {
|
||||
await beforeReply(method, args);
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Page.getLayoutMetrics")
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
|
||||
if (method === "DOM.getBoxModel")
|
||||
return { model: { content: [0, 0, 200, 0, 200, 100, 0, 100] } };
|
||||
if (method === "DOM.resolveNode") return { object: { objectId: String(args.backendNodeId) } };
|
||||
if (method === "Runtime.callFunctionOn")
|
||||
return { result: { value: { width: 200, height: 100 } } };
|
||||
return {};
|
||||
} finally {
|
||||
active--;
|
||||
}
|
||||
});
|
||||
return {
|
||||
snapshot,
|
||||
cdp: { send: send as CdpRunner["send"] },
|
||||
send,
|
||||
peak: () => peak,
|
||||
active: () => active,
|
||||
measured: () =>
|
||||
send.mock.calls
|
||||
.filter(([, method]) => method === "DOM.getBoxModel")
|
||||
.map(([, , params]) => (params as { backendNodeId: number }).backendNodeId),
|
||||
};
|
||||
}
|
||||
|
||||
describe("snapshot document provenance", () => {
|
||||
it.each([
|
||||
"new-child",
|
||||
"new-root",
|
||||
"stale-owner",
|
||||
])("retains current snapshot evidence with %s graph data", async (mode) => {
|
||||
const f = siblingCaptureFixture();
|
||||
const cdp: CdpRunner = {
|
||||
...f.cdp,
|
||||
getFrameGraph: async () => ({
|
||||
rootFrameId: mode === "new-root" ? "old-root" : "frame-0",
|
||||
frames: [
|
||||
{ frameId: mode === "new-root" ? "old-root" : "frame-0", target: { tabId: 4 } },
|
||||
...(mode === "stale-owner"
|
||||
? [
|
||||
{
|
||||
frameId: "frame-1",
|
||||
parentFrameId: "frame-0",
|
||||
ownerBackendNodeId: 999,
|
||||
target: { tabId: 4 },
|
||||
},
|
||||
]
|
||||
: []),
|
||||
],
|
||||
}),
|
||||
};
|
||||
const facts = await captureObservationFacts(cdp, 4);
|
||||
expect(facts.rootFrameId).toBe("frame-0");
|
||||
expect(facts.documents.find((doc) => doc.frame.frameId === "frame-1")).toMatchObject({
|
||||
frame: { ownerBackendNodeId: 100 },
|
||||
domNodes: [
|
||||
expect.objectContaining({ rect: { x: 0, y: 0, w: 200, h: 100 } }),
|
||||
expect.anything(),
|
||||
],
|
||||
});
|
||||
expect(
|
||||
facts.documents.find((doc) => doc.frame.frameId === "frame-7")?.domNodes[0].rect,
|
||||
).not.toBeNull();
|
||||
expect(f.measured()).not.toContain(999);
|
||||
expect(
|
||||
f.send.mock.calls.filter(([, method]) => method === "DOMSnapshot.captureSnapshot"),
|
||||
).toHaveLength(1);
|
||||
expect(
|
||||
f.send.mock.calls.some(
|
||||
([, method, params]) =>
|
||||
method === "Accessibility.getFullAXTree" &&
|
||||
(params as { frameId?: string }).frameId === "frame-7",
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("uses snapshot URLs before graph URLs and limits the caller URL to the root", async () => {
|
||||
const f = siblingCaptureFixture();
|
||||
f.snapshot.strings.push("https://current.test/page", "https://child.test/page");
|
||||
Object.assign(f.snapshot.documents[0], { documentURL: 2 });
|
||||
Object.assign(f.snapshot.documents[1], { documentURL: 3 });
|
||||
const facts = await captureObservationFacts(
|
||||
{
|
||||
...f.cdp,
|
||||
getFrameGraph: async () => ({
|
||||
rootFrameId: "frame-0",
|
||||
frames: [{ frameId: "frame-0", target: { tabId: 4 }, url: "https://old.test/" }],
|
||||
}),
|
||||
},
|
||||
4,
|
||||
undefined,
|
||||
"https://caller.test/",
|
||||
);
|
||||
expect(facts.documents.find((doc) => doc.frame.frameId === "frame-0")?.frame.url).toBe(
|
||||
"https://current.test/page",
|
||||
);
|
||||
expect(facts.documents.find((doc) => doc.frame.frameId === "frame-1")?.frame.url).toBe(
|
||||
"https://child.test/page",
|
||||
);
|
||||
expect(
|
||||
facts.documents.find((doc) => doc.frame.frameId === "frame-2")?.frame.url,
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("accepts a migrated snapshot claim without merging AX from the old target", async () => {
|
||||
const f = siblingCaptureFixture();
|
||||
const cdp: CdpRunner = {
|
||||
...f.cdp,
|
||||
getFrameGraph: async () => ({
|
||||
rootFrameId: "frame-0",
|
||||
frames: [
|
||||
{ frameId: "frame-0", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "frame-1",
|
||||
parentFrameId: "frame-0",
|
||||
ownerBackendNodeId: 100,
|
||||
target: { tabId: 4, sessionId: "old" },
|
||||
},
|
||||
],
|
||||
}),
|
||||
sendToTarget: async <T>(_target: CdpTarget, method: string): Promise<T> =>
|
||||
(method === "Accessibility.getFullAXTree"
|
||||
? { nodes: [{ nodeId: "stale", role: { value: "button" }, name: { value: "stale" } }] }
|
||||
: {}) as T,
|
||||
};
|
||||
const facts = await captureObservationFacts(cdp, 4);
|
||||
const current = facts.documents.find((doc) => doc.frame.frameId === "frame-1");
|
||||
expect(current?.frame.target).toEqual({ tabId: 4 });
|
||||
expect(current?.domNodes.length).toBeGreaterThan(0);
|
||||
expect(current?.axNodes).toEqual([]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
0, 1,
|
||||
])("rejects duplicate snapshot frame identity at document %i without overwriting", async (index) => {
|
||||
const f = siblingCaptureFixture();
|
||||
f.snapshot.documents.push(f.snapshot.documents[index]);
|
||||
if (index === 0) {
|
||||
await expect(captureObservationFacts(f.cdp, 4)).rejects.toThrow(
|
||||
"root document ownership is ambiguous",
|
||||
);
|
||||
} else {
|
||||
const facts = await captureObservationFacts(f.cdp, 4);
|
||||
expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-1");
|
||||
expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-7");
|
||||
expect(
|
||||
facts.documents.find((doc) => doc.frame.frameId === "frame-2")?.domNodes.length,
|
||||
).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it("retains the caller URL in the root AX-only fallback", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
getFrameGraph: async () => {
|
||||
throw new Error("no graph");
|
||||
},
|
||||
send: async <T>(_tabId: number, method: string): Promise<T> => {
|
||||
if (method === "DOMSnapshot.captureSnapshot") throw new Error("no snapshot");
|
||||
return (
|
||||
method === "Accessibility.getFullAXTree"
|
||||
? { nodes: [{ nodeId: "root", role: { value: "RootWebArea" } }] }
|
||||
: {}
|
||||
) as T;
|
||||
},
|
||||
};
|
||||
const facts = await captureObservationFacts(cdp, 4, undefined, "https://page.test/");
|
||||
expect(facts.documents[0].frame.url).toBe("https://page.test/");
|
||||
expect(facts.documents[0].axNodes).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("retains disconnected document data without inventing top-level geometry", async () => {
|
||||
const f = siblingCaptureFixture();
|
||||
f.snapshot.documents[0].nodes.contentDocumentIndex = { index: [], value: [] };
|
||||
const facts = await captureObservationFacts(f.cdp, 4);
|
||||
expect(
|
||||
facts.documents.find((doc) => doc.frame.frameId === "frame-1")?.domNodes[0],
|
||||
).toMatchObject({ rect: null, localRect: { x: 0, y: 0, w: 200, h: 100 } });
|
||||
expect(f.measured()).toEqual([]);
|
||||
expect(facts.issues).toContainEqual(
|
||||
expect.objectContaining({ frameId: "frame-1", stage: "ownership" }),
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
false,
|
||||
true,
|
||||
])("isolates competing target claims independent of batch order %s", async (reverse) => {
|
||||
const f = siblingCaptureFixture();
|
||||
const remote = {
|
||||
frameId: "frame-1",
|
||||
parentFrameId: "frame-0",
|
||||
ownerBackendNodeId: 100,
|
||||
target: { tabId: 4, sessionId: "remote" },
|
||||
};
|
||||
const main = { frameId: "frame-0", target: { tabId: 4 } };
|
||||
const cdp: CdpRunner = {
|
||||
...f.cdp,
|
||||
getFrameGraph: async () => ({
|
||||
rootFrameId: "frame-0",
|
||||
frames: reverse ? [remote, main] : [main, remote],
|
||||
}),
|
||||
sendToTarget: async <T>(_target: CdpTarget, method: string): Promise<T> =>
|
||||
(method === "DOMSnapshot.captureSnapshot"
|
||||
? { ...f.snapshot, documents: [f.snapshot.documents[1]] }
|
||||
: method === "Page.getLayoutMetrics"
|
||||
? { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } }
|
||||
: {}) as T,
|
||||
};
|
||||
const facts = await captureObservationFacts(cdp, 4);
|
||||
expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-1");
|
||||
expect(facts.documents.map((doc) => doc.frame.frameId)).not.toContain("frame-7");
|
||||
expect(
|
||||
facts.documents.find((doc) => doc.frame.frameId === "frame-2")?.domNodes[0].rect,
|
||||
).not.toBeNull();
|
||||
expect(facts.issues).toContainEqual(
|
||||
expect.objectContaining({ frameId: "frame-1", stage: "ownership" }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("sibling frame measurement scheduling", () => {
|
||||
it("bounds concurrent reads, fills free slots and preserves breadth-first output despite reordered replies", async () => {
|
||||
const pending = new Map<number, () => void>();
|
||||
const fixture = siblingCaptureFixture(async (method, params) => {
|
||||
if (method === "DOM.getBoxModel" && Number(params.backendNodeId) < 200)
|
||||
await new Promise<void>((resolve) => pending.set(Number(params.backendNodeId), resolve));
|
||||
});
|
||||
const capture = captureObservationFacts(fixture.cdp, 4);
|
||||
await vi.waitFor(() => expect(pending.size).toBe(4));
|
||||
expect(fixture.measured()).toEqual([100, 101, 102, 103]);
|
||||
pending.get(103)!();
|
||||
await vi.waitFor(() => expect(pending.has(104)).toBe(true));
|
||||
pending.get(104)!();
|
||||
await vi.waitFor(() => expect(pending.has(105)).toBe(true));
|
||||
expect(fixture.measured()).not.toContain(200);
|
||||
for (const id of [105, 102, 101, 100]) pending.get(id)!();
|
||||
const captured = await capture;
|
||||
expect(
|
||||
captured.issues
|
||||
.filter((issue) => issue.stage === "geometry")
|
||||
.map((issue) => issue.projectionIssue),
|
||||
).toEqual([]);
|
||||
expect(fixture.peak()).toBe(4);
|
||||
expect(fixture.active()).toBe(0);
|
||||
expect(fixture.measured()).toEqual([100, 101, 102, 103, 104, 105, 200]);
|
||||
expect(captured.documents.map((doc) => doc.frame.frameId)).toEqual(
|
||||
Array.from({ length: 8 }, (_, i) => `frame-${i}`),
|
||||
);
|
||||
expect(
|
||||
captured.documents.find((doc) => doc.frame.frameId === "frame-7")?.domNodes[0].rect,
|
||||
).toEqual({ x: 0, y: 0, w: 200, h: 100 });
|
||||
for (const method of [
|
||||
"DOM.getBoxModel",
|
||||
"DOM.resolveNode",
|
||||
"Runtime.callFunctionOn",
|
||||
"Runtime.releaseObject",
|
||||
])
|
||||
expect(fixture.send.mock.calls.filter(([, name]) => name === method)).toHaveLength(7);
|
||||
});
|
||||
|
||||
it("does not measure descendants of a failed owner and retains sibling geometry and local nodes", async () => {
|
||||
const fixture = siblingCaptureFixture(async (method, params) => {
|
||||
if (method === "DOM.resolveNode" && params.backendNodeId === 100)
|
||||
throw new Error("owner replaced");
|
||||
});
|
||||
const captured = await captureObservationFacts(fixture.cdp, 4);
|
||||
expect(
|
||||
captured.issues
|
||||
.filter((issue) => issue.stage === "geometry")
|
||||
.map((issue) => issue.projectionIssue),
|
||||
).toEqual([
|
||||
{
|
||||
status: "unavailable",
|
||||
source: { target: { tabId: 4 }, frameId: "frame-1" },
|
||||
ownerBackendNodeId: 100,
|
||||
},
|
||||
{
|
||||
status: "blocked",
|
||||
source: { target: { tabId: 4 }, frameId: "frame-7" },
|
||||
cause: captured.issues.find((issue) => issue.frameId === "frame-1")?.projectionIssue,
|
||||
},
|
||||
]);
|
||||
expect(fixture.measured()).not.toContain(200);
|
||||
expect(
|
||||
captured.documents.find((doc) => doc.frame.frameId === "frame-7")?.domNodes[0],
|
||||
).toMatchObject({
|
||||
tag: "body",
|
||||
rect: null,
|
||||
localRect: { x: 0, y: 0, w: 200, h: 100 },
|
||||
});
|
||||
expect(
|
||||
captured.documents.find((doc) => doc.frame.frameId === "frame-2")?.domNodes[0].rect,
|
||||
).toEqual({ x: 0, y: 0, w: 200, h: 100 });
|
||||
expect(captured.documents.map((doc) => doc.frame.frameId)).toEqual(
|
||||
Array.from({ length: 8 }, (_, i) => `frame-${i}`),
|
||||
);
|
||||
});
|
||||
|
||||
it("stops scheduling on cancellation and waits for resolved objects to be released", async () => {
|
||||
const controller = new AbortController();
|
||||
const resolutions = new Map<string, () => void>();
|
||||
const releases = new Map<string, () => void>();
|
||||
const fixture = siblingCaptureFixture(async (method, params) => {
|
||||
if (method === "DOM.resolveNode")
|
||||
await new Promise<void>((resolve) =>
|
||||
resolutions.set(String(params.backendNodeId), resolve),
|
||||
);
|
||||
if (method === "Runtime.releaseObject")
|
||||
await new Promise<void>((resolve) => releases.set(String(params.objectId), resolve));
|
||||
});
|
||||
let settled = false;
|
||||
const capture = captureObservationFacts(fixture.cdp, 4, controller.signal);
|
||||
const rejected = expect(capture).rejects.toMatchObject({ name: "AbortError" });
|
||||
void capture.then(
|
||||
() => {
|
||||
settled = true;
|
||||
},
|
||||
() => {
|
||||
settled = true;
|
||||
},
|
||||
);
|
||||
await vi.waitFor(() => expect(resolutions.size).toBe(4));
|
||||
controller.abort();
|
||||
for (const resolve of resolutions.values()) resolve();
|
||||
await vi.waitFor(() => expect(releases.size).toBe(4));
|
||||
expect(settled).toBe(false);
|
||||
expect(fixture.measured()).toEqual([100, 101, 102, 103]);
|
||||
for (const release of releases.values()) release();
|
||||
await rejected;
|
||||
expect(fixture.active()).toBe(0);
|
||||
expect(fixture.send.mock.calls.some(([, method]) => method === "Runtime.callFunctionOn")).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,7 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
|
||||
import { collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
|
||||
import { captureObservationFacts, semanticCapture } from "../capture-coordinator";
|
||||
import type { CdpRunner } from "../../shared";
|
||||
import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
|
||||
|
||||
// Minimal but format-accurate captureSnapshot reply: a body with one
|
||||
// fixed full-screen overlay div carrying a password input.
|
||||
@@ -27,7 +27,6 @@ function fakeSnapshotReply() {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 2],
|
||||
nodeType: [1, 1, 1, 1],
|
||||
@@ -75,7 +74,6 @@ function hoverTriggerSnapshotReply() {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeType: [1, 1, 1],
|
||||
@@ -120,7 +118,6 @@ function twoHoverTriggerSnapshotReply() {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 1],
|
||||
nodeType: [1, 1, 1, 1],
|
||||
@@ -174,7 +171,6 @@ function nestedHoverTriggerSnapshotReply() {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 2, 3],
|
||||
nodeType: [1, 1, 1, 1, 1],
|
||||
@@ -248,11 +244,13 @@ function makeCdp(snapshot: unknown) {
|
||||
};
|
||||
}
|
||||
|
||||
describe("snapshot capture and hover", () => {
|
||||
describe("captureViewModel", () => {
|
||||
it("parses nodes, attrs, rects, paint order and styles", async () => {
|
||||
const { nodes, viewport } = semanticCapture(
|
||||
await captureObservationFacts(makeCdp(fakeSnapshotReply()), 4),
|
||||
).captured;
|
||||
const { nodes, viewport, iframeNodes } = await captureViewModel(
|
||||
makeCdp(fakeSnapshotReply()),
|
||||
4,
|
||||
);
|
||||
expect(iframeNodes.size).toBe(0);
|
||||
expect(viewport).toEqual({ width: 1000, height: 800 });
|
||||
const div = nodes.find((n) => n.backendNodeId === 12);
|
||||
expect(div).toMatchObject({ tag: "div", position: "fixed", parentBackendNodeId: 11 });
|
||||
@@ -264,11 +262,35 @@ describe("snapshot capture and hover", () => {
|
||||
|
||||
it("never hovers the page while capturing", async () => {
|
||||
const cdp = makeCdp(fakeSnapshotReply());
|
||||
semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
await captureViewModel(cdp, 4);
|
||||
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything());
|
||||
});
|
||||
|
||||
it("batch-enriches form controls without per-node object resolution", async () => {
|
||||
const cdp = makeCdp(fakeSnapshotReply());
|
||||
const result = await captureViewModel(cdp, 4);
|
||||
const input = result.nodes.find((node) => node.backendNodeId === 13);
|
||||
|
||||
expect(input).toMatchObject({
|
||||
formState: "filled",
|
||||
formPlaceholder: "secret",
|
||||
attrs: { type: "password" },
|
||||
});
|
||||
expect(input?.formValue).toBeUndefined();
|
||||
expect(input?.formDefaultValue).toBeUndefined();
|
||||
expect(cdp.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({
|
||||
serializationOptions: expect.objectContaining({ serialization: "deep" }),
|
||||
}),
|
||||
);
|
||||
expect(cdp.send).toHaveBeenCalledWith(4, "Runtime.releaseObjectGroup", expect.anything());
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "DOM.resolveNode", expect.anything());
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "Runtime.callFunctionOn", expect.anything());
|
||||
});
|
||||
|
||||
it("runs hover surface probes when explicitly enabled", async () => {
|
||||
let hoverStateCalls = 0;
|
||||
const cdp = {
|
||||
@@ -308,7 +330,7 @@ describe("snapshot capture and hover", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
const captured = await captureViewModel(cdp, 4);
|
||||
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||
|
||||
expect(surfaceProbes).toEqual([
|
||||
@@ -368,7 +390,7 @@ describe("snapshot capture and hover", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
const captured = await captureViewModel(cdp, 4);
|
||||
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||
|
||||
expect(surfaceProbes).toEqual([
|
||||
@@ -413,7 +435,7 @@ describe("snapshot capture and hover", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
const captured = await captureViewModel(cdp, 4);
|
||||
await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||
|
||||
expect(hoverMoves).toBe(1);
|
||||
@@ -445,7 +467,6 @@ describe("snapshot capture and hover", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 0, 3, 4],
|
||||
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
|
||||
@@ -476,7 +497,7 @@ describe("snapshot capture and hover", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
|
||||
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
const ids = nodes.map((n) => n.backendNodeId);
|
||||
// Real page nodes survive.
|
||||
expect(ids).toContain(12);
|
||||
@@ -507,7 +528,7 @@ describe("snapshot capture and hover", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
frameId: "main",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 0, 3, 4],
|
||||
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
|
||||
@@ -538,10 +559,12 @@ describe("snapshot capture and hover", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const { excludedBackendNodeIds } = semanticCapture(
|
||||
await captureObservationFacts(makeCdp(snapshot), 4),
|
||||
).captured;
|
||||
const { excludedBackendNodeIds, frameExcludedBackendNodeIds } = await captureViewModel(
|
||||
makeCdp(snapshot),
|
||||
4,
|
||||
);
|
||||
expect(excludedBackendNodeIds).toEqual(new Set([13, 14, 15]));
|
||||
expect(frameExcludedBackendNodeIds?.get("main")).toEqual(excludedBackendNodeIds);
|
||||
});
|
||||
|
||||
it("excludes overlay host matched by tag name only", async () => {
|
||||
@@ -561,7 +584,6 @@ describe("snapshot capture and hover", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i(OVERLAY_HOST_NAME)],
|
||||
@@ -584,9 +606,7 @@ describe("snapshot capture and hover", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes, excludedBackendNodeIds } = semanticCapture(
|
||||
await captureObservationFacts(makeCdp(snapshot), 4),
|
||||
).captured;
|
||||
const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
expect(nodes.map((n) => n.backendNodeId)).not.toContain(13);
|
||||
expect(excludedBackendNodeIds).toEqual(new Set([13]));
|
||||
});
|
||||
@@ -609,7 +629,6 @@ describe("snapshot capture and hover", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
@@ -632,9 +651,7 @@ describe("snapshot capture and hover", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes, excludedBackendNodeIds } = semanticCapture(
|
||||
await captureObservationFacts(makeCdp(snapshot), 4),
|
||||
).captured;
|
||||
const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
expect(nodes.map((n) => n.backendNodeId)).not.toContain(13);
|
||||
expect(excludedBackendNodeIds).toEqual(new Set([13]));
|
||||
});
|
||||
@@ -657,7 +674,6 @@ describe("snapshot capture and hover", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
@@ -680,7 +696,7 @@ describe("snapshot capture and hover", () => {
|
||||
},
|
||||
],
|
||||
};
|
||||
const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
|
||||
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.cursor).toBe("pointer");
|
||||
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
|
||||
});
|
||||
@@ -692,7 +708,6 @@ describe("snapshot capture and hover", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 1, 1],
|
||||
nodeName: [i("html"), i("body"), i("button"), i("button"), i("button")],
|
||||
@@ -719,7 +734,7 @@ describe("snapshot capture and hover", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
|
||||
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
|
||||
|
||||
expect(nodes.find((node) => node.backendNodeId === 12)?.rendered).toBe(true);
|
||||
expect(nodes.find((node) => node.backendNodeId === 13)?.rendered).toBe(false);
|
||||
@@ -739,7 +754,7 @@ describe("snapshot capture and hover", () => {
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
const div = nodes.find((n) => n.backendNodeId === 12);
|
||||
expect(div?.localRect?.y).toBe(-200);
|
||||
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
|
||||
@@ -752,7 +767,6 @@ describe("snapshot capture and hover", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
@@ -787,7 +801,7 @@ describe("snapshot capture and hover", () => {
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.rect).toEqual({
|
||||
x: 0,
|
||||
y: 0,
|
||||
@@ -823,7 +837,6 @@ describe("snapshot capture and hover", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
@@ -862,7 +875,7 @@ describe("snapshot capture and hover", () => {
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
const btn = nodes.find((n) => n.backendNodeId === 12);
|
||||
expect(btn?.textContent).toBe("登录");
|
||||
const link = nodes.find((n) => n.backendNodeId === 14);
|
||||
@@ -872,7 +885,7 @@ describe("snapshot capture and hover", () => {
|
||||
expect(body?.textContent).toBeUndefined();
|
||||
});
|
||||
|
||||
it("retains explicit iframe document ownership and projected nodes", async () => {
|
||||
it("parses iframe sub-documents and returns iframeNodes keyed by iframe backendNodeId", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
@@ -958,21 +971,30 @@ describe("snapshot capture and hover", () => {
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const facts = await captureObservationFacts(cdp, 4);
|
||||
const root = facts.documents.find((doc) => doc.frame.frameId === "main-frame")!;
|
||||
const child = facts.documents.find((doc) => doc.frame.frameId === "child-frame")!;
|
||||
expect(facts.documents).toHaveLength(2);
|
||||
expect(root.domNodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
|
||||
const subNodes = child.domNodes;
|
||||
const {
|
||||
nodes,
|
||||
iframeNodes,
|
||||
frameNodes,
|
||||
frameOwnerBackendNodeIds,
|
||||
frameParentIds,
|
||||
rootFrameId,
|
||||
} = await captureViewModel(cdp, 4);
|
||||
// Main frame has 4 nodes; iframe is included
|
||||
expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
|
||||
// iframeNodes keyed by the <iframe> element's backendNodeId=13
|
||||
expect(iframeNodes.size).toBe(1);
|
||||
expect(iframeNodes.has(13)).toBe(true);
|
||||
const subNodes = iframeNodes.get(13)!;
|
||||
const input = subNodes.find((n) => n.backendNodeId === 101);
|
||||
expect(input?.tag).toBe("input");
|
||||
expect(input?.attrs.type).toBe("text");
|
||||
expect(input?.ownerFrameBackendNodeId).toBe(13);
|
||||
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
|
||||
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
|
||||
expect(facts.rootFrameId).toBe("main-frame");
|
||||
expect(child.frame.ownerBackendNodeId).toBe(13);
|
||||
expect(child.frame.parentFrameId).toBe("main-frame");
|
||||
expect(rootFrameId).toBe("main-frame");
|
||||
expect(frameNodes?.get("child-frame")).toBe(subNodes);
|
||||
expect(frameOwnerBackendNodeIds?.get("child-frame")).toBe(13);
|
||||
expect(frameParentIds?.get("child-frame")).toBe("main-frame");
|
||||
});
|
||||
|
||||
it("projects iframe documents independently of owner snapshot geometry", async () => {
|
||||
@@ -993,7 +1015,6 @@ describe("snapshot capture and hover", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
@@ -1011,7 +1032,6 @@ describe("snapshot capture and hover", () => {
|
||||
},
|
||||
},
|
||||
{
|
||||
frameId: "child",
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
@@ -1048,10 +1068,8 @@ describe("snapshot capture and hover", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const { documents } = await captureObservationFacts(cdp, 4);
|
||||
const input = documents
|
||||
.find((doc) => doc.frame.frameId === "child")
|
||||
?.domNodes?.find((n) => n.backendNodeId === 101);
|
||||
const { iframeNodes } = await captureViewModel(cdp, 4);
|
||||
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
|
||||
|
||||
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
|
||||
expect(input?.rect).toEqual({ x: 120, y: 320, w: 120, h: 60 });
|
||||
@@ -1075,7 +1093,6 @@ describe("snapshot capture and hover", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("iframe")],
|
||||
@@ -1097,7 +1114,6 @@ describe("snapshot capture and hover", () => {
|
||||
},
|
||||
},
|
||||
{
|
||||
frameId: "child",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("iframe")],
|
||||
@@ -1119,7 +1135,6 @@ describe("snapshot capture and hover", () => {
|
||||
},
|
||||
},
|
||||
{
|
||||
frameId: "nested",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("input")],
|
||||
@@ -1175,13 +1190,9 @@ describe("snapshot capture and hover", () => {
|
||||
) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const { documents } = await captureObservationFacts(cdp, 4);
|
||||
const nestedIframe = documents
|
||||
.find((doc) => doc.frame.frameId === "child")
|
||||
?.domNodes?.find((n) => n.backendNodeId === 23);
|
||||
const input = documents
|
||||
.find((doc) => doc.frame.frameId === "nested")
|
||||
?.domNodes?.find((n) => n.backendNodeId === 31);
|
||||
const { iframeNodes } = await captureViewModel(cdp, 4);
|
||||
const nestedIframe = iframeNodes.get(13)?.find((n) => n.backendNodeId === 23);
|
||||
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31);
|
||||
|
||||
expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 });
|
||||
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
|
||||
@@ -1197,7 +1208,6 @@ describe("snapshot capture and hover", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("div")],
|
||||
@@ -1232,7 +1242,7 @@ describe("snapshot capture and hover", () => {
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 - 100);
|
||||
});
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type CdpRunner, cdpRunnerForTarget } from "../../shared";
|
||||
import { captureObservationFacts } from "../capture-coordinator";
|
||||
import { enrichFormControlStates } from "../form-capture";
|
||||
import { decodeDocument } from "../snapshot";
|
||||
|
||||
interface Control {
|
||||
id: number;
|
||||
@@ -77,21 +78,32 @@ function reply(entries: Array<[number, unknown]>) {
|
||||
};
|
||||
}
|
||||
|
||||
function fakeCdp(groups: Control[][], evaluate: (params: Record<string, unknown>) => unknown) {
|
||||
async function fakeCdp(
|
||||
groups: Control[][],
|
||||
evaluate: (params: Record<string, unknown>) => unknown,
|
||||
) {
|
||||
const snap = snapshot(groups);
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snap;
|
||||
if (method === "Runtime.evaluate") return evaluate(params as Record<string, unknown>);
|
||||
return {};
|
||||
});
|
||||
return { send, cdp: { send: send as CdpRunner["send"] } };
|
||||
const nodes = await decodeControls(snap);
|
||||
return { send, cdp: { send: send as CdpRunner["send"] }, nodes };
|
||||
}
|
||||
|
||||
async function decodeControls(snap: ReturnType<typeof snapshot>) {
|
||||
return Promise.all(
|
||||
snap.documents.map(async (doc) =>
|
||||
(await decodeDocument(doc, snap.strings)).nodes.map((node) => ({ ...node, rect: null })),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
describe("form state identity", () => {
|
||||
it("matches reordered controls by identity and preserves controls missing from the batch", async () => {
|
||||
// The snapshot includes a shadow input. The runtime query skips it,
|
||||
// encounters a new input, and sees the remaining inputs in a new order.
|
||||
const { cdp } = fakeCdp(
|
||||
const { cdp, nodes } = await fakeCdp(
|
||||
[
|
||||
[
|
||||
{ id: 10, value: "" },
|
||||
@@ -106,23 +118,21 @@ describe("form state identity", () => {
|
||||
[11, state("alice@example.com")],
|
||||
]),
|
||||
);
|
||||
const captured = await captureObservationFacts(cdp, 7);
|
||||
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 10)).toMatchObject({
|
||||
await enrichFormControlStates(cdp, 7, nodes);
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 10)).toMatchObject({
|
||||
formValue: "",
|
||||
formState: "empty",
|
||||
formPlaceholder: "snapshot placeholder",
|
||||
});
|
||||
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 11)).toMatchObject({
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 11)).toMatchObject({
|
||||
formValue: "alice@example.com",
|
||||
});
|
||||
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 12)).toMatchObject({
|
||||
formValue: "上海",
|
||||
});
|
||||
expect(captured.documents[0].domNodes.some((n) => n.backendNodeId === 99)).toBe(false);
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 12)).toMatchObject({ formValue: "上海" });
|
||||
expect(nodes[0].some((n) => n.backendNodeId === 99)).toBe(false);
|
||||
});
|
||||
|
||||
it("does not shift state between frames when one frame is absent from the runtime batch", async () => {
|
||||
const { cdp } = fakeCdp(
|
||||
const { cdp, nodes } = await fakeCdp(
|
||||
[
|
||||
[],
|
||||
[{ id: 11, value: "inaccessible frame" }],
|
||||
@@ -135,10 +145,8 @@ describe("form state identity", () => {
|
||||
[21, state("second frame")],
|
||||
]),
|
||||
);
|
||||
const captured = await captureObservationFacts(cdp, 7);
|
||||
const controls = captured.documents
|
||||
.filter((doc) => doc.frame.parentFrameId)
|
||||
.flatMap((doc) => doc.domNodes);
|
||||
await enrichFormControlStates(cdp, 7, nodes);
|
||||
const controls = nodes.slice(1).flat();
|
||||
expect(controls.find((n) => n.backendNodeId === 11)?.formValue).toBe("inaccessible frame");
|
||||
expect(controls.find((n) => n.backendNodeId === 21)?.formValue).toBe("second frame");
|
||||
expect(controls.find((n) => n.backendNodeId === 31)?.formValue).toBe("third frame");
|
||||
@@ -146,7 +154,6 @@ describe("form state identity", () => {
|
||||
|
||||
it("keeps identical backend ids in different OOPIF targets separate", async () => {
|
||||
const sendToTarget = vi.fn(async (target, method) => {
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot([[{ id: 11, value: "old" }]]);
|
||||
if (method === "Runtime.evaluate") return reply([[11, state(target.sessionId)]]);
|
||||
return {};
|
||||
});
|
||||
@@ -155,13 +162,9 @@ describe("form state identity", () => {
|
||||
});
|
||||
const cdp = { send, sendToTarget } as unknown as CdpRunner;
|
||||
for (const sessionId of ["left-frame", "right-frame"]) {
|
||||
const captured = await captureObservationFacts(
|
||||
cdpRunnerForTarget(cdp, { tabId: 7, sessionId }),
|
||||
7,
|
||||
);
|
||||
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 11)?.formValue).toBe(
|
||||
sessionId,
|
||||
);
|
||||
const nodes = await decodeControls(snapshot([[{ id: 11, value: "old" }]]));
|
||||
await enrichFormControlStates(cdpRunnerForTarget(cdp, { tabId: 7, sessionId }), 7, nodes);
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 11)?.formValue).toBe(sessionId);
|
||||
expect(sendToTarget).toHaveBeenCalledWith(
|
||||
{ tabId: 7, sessionId },
|
||||
"Runtime.releaseObjectGroup",
|
||||
@@ -177,7 +180,7 @@ describe("form state identity", () => {
|
||||
"malformed",
|
||||
"invalid-state",
|
||||
])("preserves snapshot data when enrichment is %s", async (mode) => {
|
||||
const { cdp, send } = fakeCdp([[{ id: 11, value: "snapshot value" }]], () => {
|
||||
const { cdp, send, nodes } = await fakeCdp([[{ id: 11, value: "snapshot value" }]], () => {
|
||||
if (mode === "unsupported") throw new Error("serialization unsupported");
|
||||
if (mode === "legacy") return { result: { value: { controls: [state("wrong value")] } } };
|
||||
const result = reply([
|
||||
@@ -187,8 +190,8 @@ describe("form state identity", () => {
|
||||
result.result.deepSerializedValue.value[0].value[1].value = "not json";
|
||||
return result;
|
||||
});
|
||||
const captured = await captureObservationFacts(cdp, 7);
|
||||
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 11)).toMatchObject({
|
||||
expect(await enrichFormControlStates(cdp, 7, nodes)).toBe(mode !== "unsupported");
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 11)).toMatchObject({
|
||||
formValue: "snapshot value",
|
||||
formPlaceholder: "snapshot placeholder",
|
||||
});
|
||||
@@ -197,18 +200,34 @@ describe("form state identity", () => {
|
||||
|
||||
it("releases the batch's remote objects when capture is cancelled", async () => {
|
||||
const controller = new AbortController();
|
||||
const { cdp, send } = fakeCdp([[{ id: 11, value: "old" }]], () => {
|
||||
const { cdp, send, nodes } = await fakeCdp([[{ id: 11, value: "old" }]], () => {
|
||||
controller.abort();
|
||||
return reply([[11, state("new")]]);
|
||||
});
|
||||
await expect(captureObservationFacts(cdp, 7, controller.signal)).rejects.toMatchObject({
|
||||
await expect(enrichFormControlStates(cdp, 7, nodes, controller.signal)).rejects.toMatchObject({
|
||||
name: "AbortError",
|
||||
});
|
||||
expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
|
||||
});
|
||||
|
||||
it("does not report success when cancelled during object cleanup", async () => {
|
||||
const controller = new AbortController();
|
||||
const nodes = await decodeControls(snapshot([[{ id: 11, value: "old" }]]));
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "Runtime.evaluate") return reply([[11, state("new")]]);
|
||||
if (method === "Runtime.releaseObjectGroup") controller.abort();
|
||||
return {};
|
||||
}) as CdpRunner["send"],
|
||||
};
|
||||
await expect(enrichFormControlStates(cdp, 7, nodes, controller.signal)).rejects.toMatchObject({
|
||||
name: "AbortError",
|
||||
});
|
||||
expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
|
||||
});
|
||||
|
||||
it("skips entries without node identity while still applying valid entries", async () => {
|
||||
const { cdp } = fakeCdp(
|
||||
const { cdp, nodes } = await fakeCdp(
|
||||
[
|
||||
[
|
||||
{ id: 11, value: "snapshot value" },
|
||||
@@ -225,24 +244,20 @@ describe("form state identity", () => {
|
||||
return result;
|
||||
},
|
||||
);
|
||||
const captured = await captureObservationFacts(cdp, 7);
|
||||
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 11)?.formValue).toBe(
|
||||
"snapshot value",
|
||||
);
|
||||
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 12)?.formValue).toBe(
|
||||
"live value",
|
||||
);
|
||||
await enrichFormControlStates(cdp, 7, nodes);
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 11)?.formValue).toBe("snapshot value");
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 12)?.formValue).toBe("live value");
|
||||
});
|
||||
|
||||
it.each([
|
||||
"password",
|
||||
"text",
|
||||
])("redacts a sensitive %s control after identity matching", async (type) => {
|
||||
const { cdp } = fakeCdp([[{ id: 11, value: "secret", type }]], () =>
|
||||
const { cdp, nodes } = await fakeCdp([[{ id: 11, value: "secret", type }]], () =>
|
||||
reply([[11, { ...state("secret"), sensitive: true }]]),
|
||||
);
|
||||
const captured = await captureObservationFacts(cdp, 7);
|
||||
const control = captured.documents[0].domNodes.find((n) => n.backendNodeId === 11);
|
||||
await enrichFormControlStates(cdp, 7, nodes);
|
||||
const control = nodes[0].find((n) => n.backendNodeId === 11);
|
||||
expect(control?.formState).toBe("filled");
|
||||
expect(control?.formValue).toBeUndefined();
|
||||
expect(control?.formDefaultValue).toBeUndefined();
|
||||
@@ -259,7 +274,7 @@ describe("form state identity", () => {
|
||||
defaultValue: "",
|
||||
placeholder: "",
|
||||
}));
|
||||
const { cdp, send } = fakeCdp([controls], (params) => {
|
||||
const { cdp, send, nodes } = await fakeCdp([controls], (params) => {
|
||||
const doc = {
|
||||
querySelectorAll: (selector: string) =>
|
||||
selector === "input,textarea,select" ? elements : [],
|
||||
@@ -270,13 +285,9 @@ describe("form state identity", () => {
|
||||
expect(entries).toHaveLength(250);
|
||||
return reply(entries.map(([element, json]) => [element.backendNodeId, JSON.parse(json)]));
|
||||
});
|
||||
const captured = await captureObservationFacts(cdp, 7);
|
||||
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 250)?.formValue).toBe(
|
||||
"live 250",
|
||||
);
|
||||
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 251)?.formValue).toBe(
|
||||
"old",
|
||||
);
|
||||
await enrichFormControlStates(cdp, 7, nodes);
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 250)?.formValue).toBe("live 250");
|
||||
expect(nodes[0].find((n) => n.backendNodeId === 251)?.formValue).toBe("old");
|
||||
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1);
|
||||
expect(
|
||||
send.mock.calls.filter(([, method]) => method === "Runtime.releaseObjectGroup"),
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
|
||||
import type { CdpRunner } from "../../shared";
|
||||
import type { CapturedNode, CapturedViewModel } from "../capture";
|
||||
import { captureFrameData } from "../frame-capture";
|
||||
|
||||
function ownerNode(backendNodeId: number, x: number): CapturedNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
parentBackendNodeId: 1,
|
||||
frameId: "main",
|
||||
ownerFrameBackendNodeId: null,
|
||||
tag: "iframe",
|
||||
attrs: {},
|
||||
rect: { x, y: 100, w: 300, h: 200 },
|
||||
localRect: { x, y: 100, w: 300, h: 200 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
}
|
||||
|
||||
function childSnapshot(frameId: string, backendNodeId: number) {
|
||||
const strings = [frameId, "body", "button", "static", "auto", "pointer"];
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
frameId,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [1, 2],
|
||||
backendNodeId: [backendNodeId - 1, backendNodeId],
|
||||
attributes: [[], []] as number[][],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[3, 4, 4],
|
||||
[3, 4, 5],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 300, 200],
|
||||
[10, 20, 100, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe("captureFrameData", () => {
|
||||
it("retains nested owner failures when merging a captured OOPIF", async () => {
|
||||
const owner = ownerNode(10, 50);
|
||||
const captured: CapturedViewModel = {
|
||||
nodes: [owner],
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([["main", [owner]]]),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
const child = childSnapshot("child", 101);
|
||||
const nested = childSnapshot("nested", 201);
|
||||
const document = child.documents[0];
|
||||
const snapshot = {
|
||||
strings: child.strings,
|
||||
documents: [
|
||||
{
|
||||
...document,
|
||||
nodes: { ...document.nodes, contentDocumentIndex: { index: [1], value: [1] } },
|
||||
},
|
||||
nested.documents[0],
|
||||
],
|
||||
};
|
||||
const reply = async (_target: unknown, method: string) => {
|
||||
if (method === "Page.getLayoutMetrics")
|
||||
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200 } };
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "DOM.getBoxModel")
|
||||
return { model: { content: [50, 100, 350, 100, 350, 300, 50, 300] } };
|
||||
if (method === "DOM.resolveNode") throw new Error("nested owner replaced");
|
||||
return {};
|
||||
};
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(reply) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(reply) as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
await captureFrameData(cdp, 4, captured);
|
||||
expect(captured.frameGeometryIssues).toEqual([
|
||||
{
|
||||
status: "unavailable",
|
||||
source: { target: { tabId: 4, sessionId: "child-session" }, frameId: "nested" },
|
||||
ownerBackendNodeId: 101,
|
||||
},
|
||||
]);
|
||||
expect(captured.frameNodes?.get("nested")?.[1]).toMatchObject({
|
||||
rect: null,
|
||||
localRect: { x: 10, y: 20, w: 100, h: 40 },
|
||||
});
|
||||
});
|
||||
|
||||
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
|
||||
const leftOwner = ownerNode(10, 50);
|
||||
const rightOwner = ownerNode(20, 500);
|
||||
const captured: CapturedViewModel = {
|
||||
nodes: [leftOwner, rightOwner],
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([["main", [leftOwner, rightOwner]]]),
|
||||
frameOwnerBackendNodeIds: new Map(),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
const sendToTarget = vi.fn(async (target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
if (target.sessionId !== "left-session") return childSnapshot("right", 201);
|
||||
const snapshot = childSnapshot("left", 101);
|
||||
const marker = snapshot.strings.push(OVERLAY_HOST_MARKER_ATTR) - 1;
|
||||
const nodes = snapshot.documents[0].nodes;
|
||||
nodes.backendNodeId.push(201); // Same id as a real button in the other target.
|
||||
nodes.parentIndex.push(0);
|
||||
nodes.nodeName.push(2);
|
||||
nodes.attributes.push([marker, -1]);
|
||||
return snapshot;
|
||||
}
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
throw new Error(`unexpected ${method}`);
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method, params) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
if (method === "DOM.getBoxModel") {
|
||||
const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId;
|
||||
const x = backendNodeId === 10 ? 50 : 500;
|
||||
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
|
||||
}
|
||||
throw new Error(`unexpected root ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "left",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "left-session" },
|
||||
},
|
||||
{
|
||||
frameId: "right",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const trees = await captureFrameData(cdp, 4, captured);
|
||||
|
||||
expect(trees.find((tree) => tree.frameId === "left")?.excludedBackendNodeIds).toEqual(
|
||||
new Set([201]),
|
||||
);
|
||||
expect(trees.find((tree) => tree.frameId === "right")?.excludedBackendNodeIds?.size).toBe(0);
|
||||
expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]);
|
||||
expect(
|
||||
captured.frameNodes?.get("left")?.find((node) => node.backendNodeId === 101)?.rect,
|
||||
).toEqual({ x: 60, y: 120, w: 100, h: 40 });
|
||||
expect(
|
||||
captured.frameNodes?.get("right")?.find((node) => node.backendNodeId === 201)?.rect,
|
||||
).toEqual({ x: 510, y: 120, w: 100, h: 40 });
|
||||
expect(captured.frameOwnerBackendNodeIds).toEqual(
|
||||
new Map([
|
||||
["left", 10],
|
||||
["right", 20],
|
||||
]),
|
||||
);
|
||||
expect(captured.frameParentIds).toEqual(
|
||||
new Map([
|
||||
["left", "main"],
|
||||
["right", "main"],
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps OOPIF semantics when viewport projection is unavailable", async () => {
|
||||
const owner = { ...ownerNode(10, 50), rect: null, localRect: null };
|
||||
const captured: CapturedViewModel = {
|
||||
nodes: [owner],
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([["main", [owner]]]),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable");
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
|
||||
}
|
||||
throw new Error(`unexpected root ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") throw new Error("viewport unavailable");
|
||||
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("child", 101);
|
||||
if (method === "Accessibility.getFullAXTree") {
|
||||
return {
|
||||
nodes: [
|
||||
{
|
||||
nodeId: "button",
|
||||
backendDOMNodeId: 101,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Continue" },
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const documents = await captureFrameData(cdp, 4, captured);
|
||||
const childDocument = documents.find((document) => document.frameId === "child");
|
||||
|
||||
expect(childDocument?.axNodes).toEqual([
|
||||
expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }),
|
||||
]);
|
||||
expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual(
|
||||
expect.objectContaining({ rect: null, localRect: { x: 10, y: 20, w: 100, h: 40 } }),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,12 +1,7 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import type { CapturedNode } from "../capture";
|
||||
import {
|
||||
buildFrameDocuments,
|
||||
type FrameAxBatch,
|
||||
type FrameDomInput,
|
||||
type FrameOwnedAxNode,
|
||||
} from "../frame-document";
|
||||
import type { CapturedNode, CapturedViewModel } from "../capture";
|
||||
import { buildFrameDocuments, type FrameAxBatch, type FrameOwnedAxNode } from "../frame-document";
|
||||
|
||||
function domNode(frameId: string, backendNodeId: number): CapturedNode {
|
||||
return {
|
||||
@@ -22,11 +17,14 @@ function domNode(frameId: string, backendNodeId: number): CapturedNode {
|
||||
};
|
||||
}
|
||||
|
||||
function captured(frameNodes: Map<string, CapturedNode[]>): FrameDomInput {
|
||||
function captured(frameNodes: Map<string, CapturedNode[]>): CapturedViewModel {
|
||||
return {
|
||||
nodes: frameNodes.get("main") ?? [],
|
||||
viewport: { width: 800, height: 600 },
|
||||
iframeNodes: new Map(),
|
||||
frameNodes,
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -159,12 +157,10 @@ describe("buildFrameDocuments", () => {
|
||||
["child", [domNode("child", 9)]],
|
||||
]);
|
||||
const capture = captured(frameNodes);
|
||||
capture.frameOwnerBackendNodeIds = new Map([["child", 7]]);
|
||||
capture.frameParentIds = new Map([["child", "left"]]);
|
||||
const frames = ["main", "left", "right", "child"].map((frameId) => ({
|
||||
frame: {
|
||||
frameId,
|
||||
target: { tabId: 4 },
|
||||
...(frameId === "child" ? { parentFrameId: "left", ownerBackendNodeId: 7 } : {}),
|
||||
},
|
||||
frame: { frameId, target: { tabId: 4 } },
|
||||
nodes: [],
|
||||
}));
|
||||
|
||||
@@ -175,23 +171,6 @@ describe("buildFrameDocuments", () => {
|
||||
ownerBackendNodeId: 7,
|
||||
});
|
||||
});
|
||||
it("rejects explicit AX frame ownership in a different target", async () => {
|
||||
const frames = [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{ frameId: "remote", target: { tabId: 4, sessionId: "remote" } },
|
||||
];
|
||||
const unresolved = vi.fn();
|
||||
const documents = await buildFrameDocuments(
|
||||
{ rootFrameId: "main", frames },
|
||||
[{ frame: frames[0], nodes: [{ nodeId: "wrong", frameId: "remote" }] }],
|
||||
captured(new Map()),
|
||||
undefined,
|
||||
unresolved,
|
||||
);
|
||||
expect(documents.every((doc) => doc.axNodes.length === 0)).toBe(true);
|
||||
expect(unresolved).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("resolves a reverse-ordered deep AX parent chain without recursion", async () => {
|
||||
const frame = { frameId: "main", target: { tabId: 4 } };
|
||||
const nodes: FrameOwnedAxNode[] = Array.from({ length: 10000 }, (_, i) => ({
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
function captureAbortError(): Error {
|
||||
const error = new Error("observation aborted");
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
export function isAbortError(error: unknown): boolean {
|
||||
return (
|
||||
typeof error === "object" &&
|
||||
error !== null &&
|
||||
(error as { name?: string }).name === "AbortError"
|
||||
);
|
||||
}
|
||||
|
||||
export function throwIfAborted(signal: AbortSignal | undefined): void {
|
||||
if (signal?.aborted) throw captureAbortError();
|
||||
}
|
||||
|
||||
/** Called only at bounded block boundaries, not once per node. */
|
||||
export async function captureCheckpoint(signal?: AbortSignal): Promise<void> {
|
||||
throwIfAborted(signal);
|
||||
await new Promise<void>((resolve) => {
|
||||
const channel = new MessageChannel();
|
||||
channel.port1.onmessage = () => {
|
||||
channel.port1.close();
|
||||
channel.port2.close();
|
||||
resolve();
|
||||
};
|
||||
channel.port2.postMessage(null);
|
||||
});
|
||||
throwIfAborted(signal);
|
||||
}
|
||||
@@ -1,334 +0,0 @@
|
||||
import {
|
||||
type CdpFrame,
|
||||
type CdpFrameGraph,
|
||||
type CdpTarget,
|
||||
cdpTargetKey,
|
||||
} from "@/browser-driver/frame-graph";
|
||||
import { cssViewport, GeometryContext } from "../geometry/frame-context";
|
||||
import { type CdpRunner, cdpRunnerForTarget } from "../shared";
|
||||
import { collectOverlayExcludedBackendIds } from "./capture";
|
||||
import type { CapturedSceneInput } from "./facts";
|
||||
import {
|
||||
buildDocumentIndex,
|
||||
type CaptureIssue,
|
||||
isCaptureAbort,
|
||||
type ObservationFacts,
|
||||
throwCaptureAborted,
|
||||
} from "./facts";
|
||||
import { enrichFormControlStates } from "./form-capture";
|
||||
import {
|
||||
buildFrameDocuments,
|
||||
type FrameAxBatch,
|
||||
type FrameDocument,
|
||||
type FrameOwnedAxNode,
|
||||
} from "./frame-document";
|
||||
import { type NormalizedDocument, normalizeSnapshot } from "./normalize";
|
||||
import { describeSnapshotFrames, REQUESTED_STYLES, type SnapshotReply } from "./snapshot";
|
||||
|
||||
interface TargetBatch<T extends FrameOwnedAxNode> {
|
||||
target: CdpTarget;
|
||||
frames: CdpFrame[];
|
||||
documents: NormalizedDocument[];
|
||||
ax: FrameAxBatch<T>[];
|
||||
fallbackExcluded: Set<number>;
|
||||
snapshotFrameIds?: Set<string>;
|
||||
rootFrameId?: string;
|
||||
}
|
||||
|
||||
/** A fixed worker pool scoped to one capture; each worker issues at most one
|
||||
* collection request at a time. Geometry has its own measurement-phase bound. */
|
||||
async function collectTargets<T>(
|
||||
items: readonly T[],
|
||||
collect: (item: T) => Promise<void>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
let cursor = 0;
|
||||
let failed = false;
|
||||
let failure: unknown;
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(4, items.length) }, async () => {
|
||||
try {
|
||||
while (cursor < items.length && !failed && !signal?.aborted) {
|
||||
const item = items[cursor++];
|
||||
await collect(item);
|
||||
}
|
||||
} catch (error) {
|
||||
if (!failed) failure = error;
|
||||
failed = true;
|
||||
}
|
||||
}),
|
||||
);
|
||||
// A worker's rejection must not let the capture outlive its own task scope.
|
||||
// In-flight reads and their finally blocks settle before cancellation/failure.
|
||||
throwCaptureAborted(signal);
|
||||
if (failed) throw failure;
|
||||
}
|
||||
|
||||
export async function captureObservationFacts<T extends FrameOwnedAxNode>(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
signal?: AbortSignal,
|
||||
pageUrl?: string,
|
||||
): Promise<ObservationFacts<T>> {
|
||||
const startedAt = Date.now();
|
||||
throwCaptureAborted(signal);
|
||||
let graph: CdpFrameGraph | undefined;
|
||||
try {
|
||||
graph = await cdp.getFrameGraph?.(tabId);
|
||||
} catch (error) {
|
||||
if (isCaptureAbort(error)) throw error;
|
||||
}
|
||||
throwCaptureAborted(signal);
|
||||
const geometry = new GeometryContext(cdp, tabId, graph, signal);
|
||||
const issues: CaptureIssue[] = [];
|
||||
const graphFrames = new Map(graph?.frames.map((frame) => [frame.frameId, frame]) ?? []);
|
||||
const groups = new Map<string, TargetBatch<T>>();
|
||||
for (const frame of graph?.frames ?? []) {
|
||||
const key = cdpTargetKey(frame.target);
|
||||
let group = groups.get(key);
|
||||
if (!group) {
|
||||
group = {
|
||||
target: frame.target,
|
||||
frames: [],
|
||||
documents: [],
|
||||
ax: [],
|
||||
fallbackExcluded: new Set(),
|
||||
};
|
||||
groups.set(key, group);
|
||||
}
|
||||
group.frames.push(
|
||||
!frame.target.sessionId && frame.frameId === graph?.rootFrameId && !frame.url && pageUrl
|
||||
? { ...frame, url: pageUrl }
|
||||
: frame,
|
||||
);
|
||||
}
|
||||
if (!groups.has(cdpTargetKey({ tabId })))
|
||||
groups.set(cdpTargetKey({ tabId }), {
|
||||
target: { tabId },
|
||||
frames: [],
|
||||
documents: [],
|
||||
ax: [],
|
||||
fallbackExcluded: new Set(),
|
||||
});
|
||||
const batches = [...groups.values()];
|
||||
const unavailable = (batch: TargetBatch<T>, stage: CaptureIssue["stage"], frameId?: string) =>
|
||||
issues.push({ target: batch.target, frameId, stage, reason: "capture-unavailable" });
|
||||
let firstFailure: unknown;
|
||||
await collectTargets(
|
||||
batches,
|
||||
async (batch) => {
|
||||
const scoped = cdpRunnerForTarget(cdp, batch.target);
|
||||
try {
|
||||
throwCaptureAborted(signal);
|
||||
await scoped.send(tabId, "DOMSnapshot.enable", {});
|
||||
throwCaptureAborted(signal);
|
||||
const snapshot = await scoped.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
|
||||
computedStyles: REQUESTED_STYLES,
|
||||
includePaintOrder: true,
|
||||
includeDOMRects: true,
|
||||
});
|
||||
throwCaptureAborted(signal);
|
||||
const observed = describeSnapshotFrames(snapshot, batch.target, graphFrames, pageUrl);
|
||||
batch.snapshotFrameIds = observed.ids;
|
||||
batch.rootFrameId = observed.rootFrameId;
|
||||
// Keep graph-only frames for AX fallback, never in preference to observed membership.
|
||||
batch.frames = [
|
||||
...observed.frames,
|
||||
...batch.frames.filter((frame) => !observed.ids.has(frame.frameId)),
|
||||
];
|
||||
batch.documents = await normalizeSnapshot(
|
||||
snapshot,
|
||||
batch.target,
|
||||
batch.frames.filter((frame) => observed.ids.has(frame.frameId)),
|
||||
geometry,
|
||||
issues,
|
||||
signal,
|
||||
observed.rootFrameId,
|
||||
);
|
||||
const formsAvailable = await enrichFormControlStates(
|
||||
scoped,
|
||||
tabId,
|
||||
batch.documents.map((doc) => doc.domNodes),
|
||||
signal,
|
||||
);
|
||||
throwCaptureAborted(signal);
|
||||
if (!formsAvailable) unavailable(batch, "forms");
|
||||
const present = new Set(batch.documents.map((doc) => doc.frame.frameId));
|
||||
for (const frame of batch.frames)
|
||||
if (!present.has(frame.frameId)) unavailable(batch, "dom", frame.frameId);
|
||||
} catch (error) {
|
||||
throwCaptureAborted(signal);
|
||||
if (isCaptureAbort(error)) throw error;
|
||||
firstFailure ??= error;
|
||||
unavailable(batch, "dom");
|
||||
batch.fallbackExcluded = await collectOverlayExcludedBackendIds(scoped, tabId, signal);
|
||||
}
|
||||
if (!batch.frames.length && !graph)
|
||||
batch.frames = [
|
||||
{
|
||||
frameId: "root",
|
||||
target: batch.target,
|
||||
...(!batch.target.sessionId && pageUrl ? { url: pageUrl } : {}),
|
||||
},
|
||||
];
|
||||
try {
|
||||
throwCaptureAborted(signal);
|
||||
await scoped.send(tabId, "Accessibility.enable", {});
|
||||
} catch (error) {
|
||||
throwCaptureAborted(signal);
|
||||
if (isCaptureAbort(error)) throw error;
|
||||
firstFailure ??= error;
|
||||
unavailable(batch, "ax");
|
||||
return;
|
||||
}
|
||||
for (const frame of batch.frames) {
|
||||
try {
|
||||
throwCaptureAborted(signal);
|
||||
const result = await scoped.send<{ nodes?: T[] }>(
|
||||
tabId,
|
||||
"Accessibility.getFullAXTree",
|
||||
frame.frameId === "root" ? {} : { frameId: frame.frameId },
|
||||
);
|
||||
throwCaptureAborted(signal);
|
||||
batch.ax.push({ frame, nodes: result.nodes ?? [] });
|
||||
} catch (error) {
|
||||
throwCaptureAborted(signal);
|
||||
if (isCaptureAbort(error)) throw error;
|
||||
firstFailure ??= error;
|
||||
unavailable(batch, "ax", frame.frameId);
|
||||
}
|
||||
}
|
||||
},
|
||||
signal,
|
||||
);
|
||||
|
||||
// A snapshot claim outranks an old graph hint. Conflicting actual claims
|
||||
// cannot be resolved by completion order or by overwriting a frameId map.
|
||||
const claims = new Map<string, string>();
|
||||
const invalid = new Set<string>();
|
||||
for (const batch of batches) {
|
||||
const targetKey = cdpTargetKey(batch.target);
|
||||
for (const id of batch.snapshotFrameIds ?? []) {
|
||||
const previous = claims.get(id);
|
||||
if (previous !== undefined && previous !== targetKey) invalid.add(id);
|
||||
claims.set(id, targetKey);
|
||||
}
|
||||
const accepted = new Set(batch.frames.map((frame) => frame.frameId));
|
||||
for (const id of batch.snapshotFrameIds ?? []) if (!accepted.has(id)) invalid.add(id);
|
||||
}
|
||||
const currentFrames = batches.flatMap((batch) =>
|
||||
batch.frames.filter(
|
||||
(frame) =>
|
||||
!claims.has(frame.frameId) || claims.get(frame.frameId) === cdpTargetKey(batch.target),
|
||||
),
|
||||
);
|
||||
const children = new Map<string, string[]>();
|
||||
for (const frame of currentFrames) {
|
||||
if (!frame.parentFrameId) continue;
|
||||
const siblings = children.get(frame.parentFrameId);
|
||||
if (siblings) siblings.push(frame.frameId);
|
||||
else children.set(frame.parentFrameId, [frame.frameId]);
|
||||
}
|
||||
const blocked = [...invalid];
|
||||
for (let i = 0; i < blocked.length; i++) {
|
||||
for (const child of children.get(blocked[i]) ?? []) {
|
||||
if (!invalid.has(child)) {
|
||||
invalid.add(child);
|
||||
blocked.push(child);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const frame of currentFrames) {
|
||||
if (invalid.has(frame.frameId))
|
||||
issues.push({
|
||||
target: frame.target,
|
||||
frameId: frame.frameId,
|
||||
stage: "ownership",
|
||||
reason: "frame-ownership-unresolved",
|
||||
});
|
||||
}
|
||||
const rootBatch = batches.find((batch) => !batch.target.sessionId);
|
||||
const rootFrameId =
|
||||
rootBatch?.rootFrameId ?? graph?.rootFrameId ?? rootBatch?.frames[0]?.frameId ?? "root";
|
||||
if (invalid.has(rootFrameId)) throw new Error("observation root document ownership is ambiguous");
|
||||
const frames = currentFrames.filter((frame) => !invalid.has(frame.frameId));
|
||||
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
|
||||
const belongs = (frame: CdpFrame) => {
|
||||
const current = frameById.get(frame.frameId);
|
||||
return current && cdpTargetKey(current.target) === cdpTargetKey(frame.target);
|
||||
};
|
||||
const decoded = new Map(
|
||||
batches
|
||||
.flatMap((batch) => batch.documents)
|
||||
.filter((doc) => belongs(doc.frame))
|
||||
.map((doc) => [doc.frame.frameId, doc]),
|
||||
);
|
||||
const ax = batches.flatMap((batch) => batch.ax).filter((batch) => belongs(batch.frame));
|
||||
const documents = await buildFrameDocuments(
|
||||
{ rootFrameId, frames },
|
||||
ax,
|
||||
{
|
||||
nodes: [],
|
||||
rootFrameId,
|
||||
frameNodes: new Map([...decoded].map(([id, doc]) => [id, doc.domNodes])),
|
||||
},
|
||||
signal,
|
||||
(frame) =>
|
||||
issues.push({
|
||||
target: frame.target,
|
||||
frameId: frame.frameId,
|
||||
stage: "ownership",
|
||||
reason: "frame-ownership-unresolved",
|
||||
}),
|
||||
);
|
||||
let viewport = { width: 0, height: 0 };
|
||||
try {
|
||||
const measured = cssViewport(await geometry.layoutMetrics({ tabId }));
|
||||
viewport = { width: measured.width, height: measured.height };
|
||||
} catch (error) {
|
||||
if (isCaptureAbort(error)) throw error;
|
||||
}
|
||||
const facts: ObservationFacts<T>["documents"][number][] = [];
|
||||
for (const document of documents) {
|
||||
const doc = decoded.get(document.frameId);
|
||||
const index = doc?.index ?? (await buildDocumentIndex([], signal));
|
||||
const fallback = groups.get(cdpTargetKey(document.target))?.fallbackExcluded;
|
||||
const { domNodes, axNodes, contextScopeId: _scope, ...frame } = document;
|
||||
facts.push({
|
||||
frame,
|
||||
index: fallback?.size ? { ...index, excludedBackendNodeIds: fallback } : index,
|
||||
domNodes,
|
||||
axNodes,
|
||||
});
|
||||
}
|
||||
throwCaptureAborted(signal);
|
||||
if (firstFailure && facts.every((doc) => !doc.domNodes.length && !doc.axNodes.length))
|
||||
throw firstFailure;
|
||||
return { rootFrameId, viewport, documents: facts, issues, startedAt, finishedAt: Date.now() };
|
||||
}
|
||||
|
||||
/** Existing semantic consumers get a narrow view; no raw snapshot
|
||||
* data can leak through recording's allowlisted output DTO. */
|
||||
export function semanticCapture<T extends FrameOwnedAxNode>(
|
||||
facts: ObservationFacts<T>,
|
||||
): { captured: CapturedSceneInput; documents: FrameDocument<T>[] } {
|
||||
const documents = facts.documents.map((doc) => ({
|
||||
...doc.frame,
|
||||
contextScopeId: doc.frame.frameId,
|
||||
domNodes: doc.domNodes.filter(
|
||||
(node) => !doc.index.excludedBackendNodeIds.has(node.backendNodeId),
|
||||
),
|
||||
axNodes: doc.axNodes,
|
||||
excludedBackendNodeIds: doc.index.excludedBackendNodeIds,
|
||||
}));
|
||||
const root = documents.find((doc) => doc.frameId === facts.rootFrameId);
|
||||
return {
|
||||
documents,
|
||||
captured: {
|
||||
nodes: root?.domNodes ?? [],
|
||||
viewport: facts.viewport,
|
||||
rootFrameId: facts.rootFrameId,
|
||||
excludedBackendNodeIds: root?.excludedBackendNodeIds ?? new Set(),
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import type { Rect } from "@browser-skill/vom";
|
||||
|
||||
export interface CapturedNode {
|
||||
backendNodeId: number;
|
||||
parentBackendNodeId: number | null;
|
||||
frameId?: string;
|
||||
/** Owning iframe backend node id; `null` for the top-level document. */
|
||||
ownerFrameBackendNodeId?: number | null;
|
||||
tag: string;
|
||||
attrs: Record<string, string>;
|
||||
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
|
||||
rect: Rect | null;
|
||||
/** Frame-local viewport-relative CSS px before top-level projection. */
|
||||
localRect?: Rect | null;
|
||||
paintOrder: number;
|
||||
position: string;
|
||||
pointerEvents: string;
|
||||
/**
|
||||
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
|
||||
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
|
||||
* is actually an interactive control — used by the adapter to surface
|
||||
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
|
||||
* `textContent`: the live parser always sets it, hand-built fixtures may not.
|
||||
*/
|
||||
cursor?: string;
|
||||
/**
|
||||
* Whether the live DOM snapshot provides a painted, non-hidden box for this
|
||||
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
|
||||
* nodes remain authoritative even when they are outside the viewport.
|
||||
*/
|
||||
rendered?: boolean;
|
||||
textContent?: string;
|
||||
formState?: "empty" | "filled" | "default";
|
||||
formValue?: string;
|
||||
formDefaultValue?: string;
|
||||
formPlaceholder?: string;
|
||||
}
|
||||
@@ -1,17 +1,59 @@
|
||||
// Browser-side hover probes and overlay exclusion fallback. Static collection
|
||||
// lives in capture-coordinator; snapshot decoding has no browser dependency.
|
||||
// Coordinate DOMSnapshot acquisition, frame geometry and hover probes.
|
||||
// Snapshot decoding and form enrichment live in their own modules.
|
||||
|
||||
import type { Viewport } from "@browser-skill/vom";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
||||
import { OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||
import { type FrameProjectionIssue, type FrameProjectionState } from "../geometry/coordinate-types";
|
||||
import { cssViewport, GeometryContext, type LayoutMetrics } from "../geometry/frame-context";
|
||||
import type { CdpRunner } from "../shared";
|
||||
import type { CapturedNode, CapturedSurfaceProbe } from "./facts";
|
||||
|
||||
export type {
|
||||
CapturedNode,
|
||||
CapturedSurfaceProbe,
|
||||
} from "./facts";
|
||||
|
||||
import { isCaptureAbort as isAbortError, throwCaptureAborted as throwIfAborted } from "./facts";
|
||||
import { isAbortError, throwIfAborted } from "./capture-abort";
|
||||
import type { CapturedNode } from "./capture-types";
|
||||
import { enrichFormControlStates } from "./form-capture";
|
||||
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
|
||||
import { type FrameContext, normalizeDocument } from "./normalize";
|
||||
import {
|
||||
REQUESTED_STYLES,
|
||||
type SnapshotDocument,
|
||||
type SnapshotReply,
|
||||
snapshotFrameId,
|
||||
sparseIndexMap,
|
||||
} from "./snapshot";
|
||||
|
||||
export type { CapturedNode } from "./capture-types";
|
||||
|
||||
export type CapturedIframeNodes = Map<number, CapturedNode[]>;
|
||||
|
||||
export interface CapturedSurfaceProbe {
|
||||
triggerBackendNodeId: number;
|
||||
triggerPoint?: { x: number; y: number };
|
||||
triggerAction: "hover" | "focus" | string;
|
||||
subItems: string[];
|
||||
confidence?: "high" | "medium" | "low";
|
||||
}
|
||||
|
||||
export interface CapturedViewModel {
|
||||
nodes: CapturedNode[];
|
||||
viewport: Viewport;
|
||||
iframeNodes: CapturedIframeNodes;
|
||||
frameNodes?: Map<string, CapturedNode[]>;
|
||||
frameExcludedBackendNodeIds?: Map<string, ReadonlySet<number>>;
|
||||
frameOwnerBackendNodeIds?: Map<string, number>;
|
||||
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
|
||||
frameParentIds?: Map<string, string>;
|
||||
rootFrameId?: string;
|
||||
/** Failed boundaries and descendants blocked by them; clipping is not a failure. */
|
||||
frameGeometryIssues?: FrameProjectionIssue[];
|
||||
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
export interface CaptureViewModelOptions {
|
||||
signal?: AbortSignal;
|
||||
geometry?: GeometryContext;
|
||||
target?: CdpTarget;
|
||||
}
|
||||
|
||||
interface RuntimeEvaluateReply {
|
||||
result?: {
|
||||
@@ -443,194 +485,12 @@ export async function collectOverlayExcludedBackendIds(
|
||||
return excluded;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse one DOMSnapshot document into CapturedNode[].
|
||||
*
|
||||
* @param doc - the raw DOMSnapshot document object
|
||||
* @param strings - the shared string table for the whole snapshot
|
||||
* @param context - frame coordinate context; output rects are top viewport-relative
|
||||
*/
|
||||
function parseDocumentNodes(
|
||||
doc: SnapshotDocument,
|
||||
strings: string[],
|
||||
context: FrameContext,
|
||||
): ParseDocumentResult {
|
||||
const dn = doc.nodes;
|
||||
const dl = doc.layout;
|
||||
if (!dn?.backendNodeId) {
|
||||
return { nodes: [], excludedBackendNodeIds: new Set() };
|
||||
}
|
||||
|
||||
const count = dn.backendNodeId.length;
|
||||
const layoutByNode = new Map<number, number>();
|
||||
(dl?.nodeIndex ?? []).forEach((nodeIdx, layoutIdx) => layoutByNode.set(nodeIdx, layoutIdx));
|
||||
|
||||
// The agent's own overlay (WXT shadow host, marked with OVERLAY_HOST_MARKER_ATTR)
|
||||
// is injected into the page and its open shadow root
|
||||
// is inlined by DOMSnapshot as descendants of the host. Its fixed
|
||||
// full-viewport click-blocker would otherwise dominate occlusion detection
|
||||
// and hide the real page, so mark the host + its whole subtree excluded.
|
||||
// parentIndex always references an earlier node, so one forward pass suffices.
|
||||
const excluded = new Array<boolean>(count).fill(false);
|
||||
const excludedBackendNodeIds = new Set<number>();
|
||||
for (let n = 0; n < count; n++) {
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
const inherited = parentIdx >= 0 && excluded[parentIdx];
|
||||
let isOverlayHost = inherited;
|
||||
if (!isOverlayHost) {
|
||||
const pairs = dn.attributes?.[n] ?? [];
|
||||
const attrNames: string[] = [];
|
||||
for (let a = 0; a + 1 < pairs.length; a += 2) {
|
||||
attrNames.push(str(strings, pairs[a]));
|
||||
}
|
||||
isOverlayHost = isOverlayHostNode(str(strings, dn.nodeName?.[n]), attrNames);
|
||||
}
|
||||
if (!isOverlayHost) continue;
|
||||
excluded[n] = true;
|
||||
const backendNodeId = dn.backendNodeId[n];
|
||||
if (backendNodeId !== undefined) {
|
||||
excludedBackendNodeIds.add(backendNodeId);
|
||||
}
|
||||
}
|
||||
|
||||
const posCol = STYLE_COL.position;
|
||||
const peCol = STYLE_COL["pointer-events"];
|
||||
const cursorCol = STYLE_COL.cursor;
|
||||
const visibilityCol = STYLE_COL.visibility;
|
||||
const opacityCol = STYLE_COL.opacity;
|
||||
const inputValues = sparseIndexMap(dn.inputValue);
|
||||
const textValues = sparseIndexMap(dn.textValue);
|
||||
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
|
||||
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
|
||||
|
||||
// Collect visible text from #text child nodes so element CapturedNodes
|
||||
// carry a textContent value usable as a button/link label fallback.
|
||||
// nodeValue is a parallel array: string index for text nodes, -1 otherwise.
|
||||
const nodeTextContent = new Map<number, string>();
|
||||
if (dn.nodeValue) {
|
||||
for (let n = 0; n < count; n++) {
|
||||
const nvIdx = dn.nodeValue[n] ?? -1;
|
||||
if (nvIdx < 0) continue;
|
||||
const text = str(strings, nvIdx).trim();
|
||||
if (!text) continue;
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
if (parentIdx >= 0) {
|
||||
const existing = nodeTextContent.get(parentIdx);
|
||||
nodeTextContent.set(parentIdx, existing ? `${existing} ${text}` : text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build CapturedNodes.
|
||||
const nodes: CapturedNode[] = [];
|
||||
for (let n = 0; n < count; n++) {
|
||||
if (excluded[n]) continue;
|
||||
const backendNodeId = dn.backendNodeId[n];
|
||||
const parentIdx = dn.parentIndex?.[n] ?? -1;
|
||||
const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null;
|
||||
const tag = str(strings, dn.nodeName?.[n]).toLowerCase();
|
||||
|
||||
const attrs: Record<string, string> = {};
|
||||
const pairs = dn.attributes?.[n] ?? [];
|
||||
for (let a = 0; a + 1 < pairs.length; a += 2) {
|
||||
attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]);
|
||||
}
|
||||
|
||||
let rect: Rect | null = null;
|
||||
let localRect: Rect | null = null;
|
||||
let paintOrder = 0;
|
||||
let position = "static";
|
||||
let pointerEvents = "auto";
|
||||
let cursor = "auto";
|
||||
let rendered = false;
|
||||
const li = layoutByNode.get(n);
|
||||
if (li !== undefined) {
|
||||
const b = dl?.bounds?.[li];
|
||||
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
|
||||
const input = snapshotViewportRect(
|
||||
b,
|
||||
{ target: context.target, frameId: context.frameId },
|
||||
{ x: context.scrollX, y: context.scrollY },
|
||||
);
|
||||
if (input) {
|
||||
const { x, y, width: w, height: h } = input.rect;
|
||||
localRect = { x, y, w, h };
|
||||
}
|
||||
const bounds =
|
||||
input && context.projection.status === "available"
|
||||
? projectSnapshotRect(input, context.projection.projection)
|
||||
: null;
|
||||
rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||
}
|
||||
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
||||
const styleRow = dl?.styles?.[li] ?? [];
|
||||
position = str(strings, styleRow[posCol]) || "static";
|
||||
pointerEvents = str(strings, styleRow[peCol]) || "auto";
|
||||
cursor = str(strings, styleRow[cursorCol]) || "auto";
|
||||
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
|
||||
const opacity = str(strings, styleRow[opacityCol]) || "1";
|
||||
rendered =
|
||||
localRect !== null &&
|
||||
visibility !== "hidden" &&
|
||||
visibility !== "collapse" &&
|
||||
(Number.parseFloat(opacity) || 0) > 0;
|
||||
}
|
||||
|
||||
// Skip non-element nodes (#text, #cdata-section, etc.) — they carry no
|
||||
// geometry and are never queried by callers.
|
||||
if (tag.startsWith("#")) continue;
|
||||
|
||||
const textContent = nodeTextContent.get(n);
|
||||
|
||||
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
|
||||
const rawFormValue =
|
||||
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
|
||||
const sensitive = isSensitiveFormControl({ tag, attrs });
|
||||
const formDefaultValue = attrs.value ?? "";
|
||||
const formValue = sensitive ? undefined : rawFormValue;
|
||||
const formState = snapshotFormState(
|
||||
rawFormValue,
|
||||
formDefaultValue,
|
||||
Object.prototype.hasOwnProperty.call(attrs, "value"),
|
||||
sensitive,
|
||||
);
|
||||
if (sensitive) delete attrs.value;
|
||||
|
||||
nodes.push({
|
||||
backendNodeId,
|
||||
parentBackendNodeId,
|
||||
...(context.frameId ? { frameId: context.frameId } : {}),
|
||||
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
|
||||
tag,
|
||||
attrs,
|
||||
rect,
|
||||
localRect,
|
||||
paintOrder,
|
||||
position,
|
||||
pointerEvents,
|
||||
cursor,
|
||||
rendered,
|
||||
textContent,
|
||||
...(formValue !== undefined ? { formValue } : {}),
|
||||
...(tag === "input" || tag === "textarea"
|
||||
? {
|
||||
formPlaceholder: attrs.placeholder ?? "",
|
||||
...(!sensitive ? { formDefaultValue } : {}),
|
||||
...(formState ? { formState } : {}),
|
||||
}
|
||||
: {}),
|
||||
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
|
||||
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
|
||||
});
|
||||
}
|
||||
return { nodes, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
interface ParseFrameDocumentsResult {
|
||||
frameGeometryIssues: FrameProjectionIssue[];
|
||||
iframeNodes: CapturedIframeNodes;
|
||||
frameOwnerBackendNodeIds: Map<string, number>;
|
||||
frameParentIds: Map<string, string>;
|
||||
frameExcludedBackendNodeIds: Map<string, ReadonlySet<number>>;
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
@@ -647,6 +507,7 @@ async function parseChildFrameDocuments(
|
||||
const iframeNodes: CapturedIframeNodes = new Map();
|
||||
const frameOwnerBackendNodeIds = new Map<string, number>();
|
||||
const frameParentIds = new Map<string, string>();
|
||||
const frameExcludedBackendNodeIds = new Map<string, ReadonlySet<number>>();
|
||||
const excludedBackendNodeIds = new Set<number>();
|
||||
const parentDoc = documents[parentDocIndex];
|
||||
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
|
||||
@@ -655,6 +516,7 @@ async function parseChildFrameDocuments(
|
||||
iframeNodes,
|
||||
frameOwnerBackendNodeIds,
|
||||
frameParentIds,
|
||||
frameExcludedBackendNodeIds,
|
||||
excludedBackendNodeIds,
|
||||
frameGeometryIssues,
|
||||
};
|
||||
@@ -732,13 +594,14 @@ async function parseChildFrameDocuments(
|
||||
};
|
||||
const nextVisited = new Set(visited);
|
||||
nextVisited.add(childDocIndex);
|
||||
const parsed = parseDocumentNodes(childDoc, strings, childContext);
|
||||
const parsed = await normalizeDocument(childDoc, strings, childContext, signal);
|
||||
iframeNodes.set(iframeBackendId, parsed.nodes);
|
||||
if (childContext.frameId) {
|
||||
frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId);
|
||||
frameExcludedBackendNodeIds.set(childContext.frameId, parsed.index.excludedBackendNodeIds);
|
||||
if (parentContext.frameId) frameParentIds.set(childContext.frameId, parentContext.frameId);
|
||||
}
|
||||
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
for (const id of parsed.index.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
|
||||
const nested = await parseChildFrameDocuments(
|
||||
documents,
|
||||
@@ -759,6 +622,8 @@ async function parseChildFrameDocuments(
|
||||
for (const [frameId, parentFrameId] of nested.frameParentIds) {
|
||||
frameParentIds.set(frameId, parentFrameId);
|
||||
}
|
||||
for (const [frameId, excluded] of nested.frameExcludedBackendNodeIds)
|
||||
frameExcludedBackendNodeIds.set(frameId, excluded);
|
||||
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
}
|
||||
|
||||
@@ -766,6 +631,7 @@ async function parseChildFrameDocuments(
|
||||
iframeNodes,
|
||||
frameOwnerBackendNodeIds,
|
||||
frameParentIds,
|
||||
frameExcludedBackendNodeIds,
|
||||
excludedBackendNodeIds,
|
||||
frameGeometryIssues,
|
||||
};
|
||||
@@ -827,9 +693,9 @@ export async function captureViewModel(
|
||||
scrollX,
|
||||
scrollY,
|
||||
};
|
||||
const mainParsed = parseDocumentNodes(doc0, strings, topContext);
|
||||
const mainParsed = await normalizeDocument(doc0, strings, topContext, options.signal);
|
||||
const nodes = mainParsed.nodes;
|
||||
const excludedBackendNodeIds = new Set(mainParsed.excludedBackendNodeIds);
|
||||
const excludedBackendNodeIds = new Set(mainParsed.index.excludedBackendNodeIds);
|
||||
|
||||
const ownerIds = new Set<number>();
|
||||
for (const document of documents) {
|
||||
@@ -848,6 +714,11 @@ export async function captureViewModel(
|
||||
options.signal,
|
||||
);
|
||||
const iframeNodes = frameParsed.iframeNodes;
|
||||
if (topContext.frameId)
|
||||
frameParsed.frameExcludedBackendNodeIds.set(
|
||||
topContext.frameId,
|
||||
mainParsed.index.excludedBackendNodeIds,
|
||||
);
|
||||
for (const id of frameParsed.excludedBackendNodeIds) {
|
||||
excludedBackendNodeIds.add(id);
|
||||
}
|
||||
@@ -874,6 +745,7 @@ export async function captureViewModel(
|
||||
frameNodes,
|
||||
frameGeometryIssues: frameParsed.frameGeometryIssues,
|
||||
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
|
||||
frameExcludedBackendNodeIds: frameParsed.frameExcludedBackendNodeIds,
|
||||
frameParentIds: frameParsed.frameParentIds,
|
||||
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
|
||||
excludedBackendNodeIds,
|
||||
|
||||
@@ -1,60 +1,6 @@
|
||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||
import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import { isOverlayHostNode } from "@/lib/overlay-bridge";
|
||||
import type { FrameProjectionIssue } from "../geometry/coordinate-types";
|
||||
import type { FrameOwnedAxNode } from "./frame-document";
|
||||
|
||||
export interface CapturedNode {
|
||||
backendNodeId: number;
|
||||
parentBackendNodeId: number | null;
|
||||
frameId?: string;
|
||||
/** Owning iframe backend node id; `null` for the top-level document. */
|
||||
ownerFrameBackendNodeId?: number | null;
|
||||
tag: string;
|
||||
attrs: Record<string, string>;
|
||||
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
|
||||
rect: Rect | null;
|
||||
/** Frame-local viewport-relative CSS px before top-level projection. */
|
||||
localRect?: Rect | null;
|
||||
paintOrder: number;
|
||||
position: string;
|
||||
pointerEvents: string;
|
||||
/**
|
||||
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
|
||||
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
|
||||
* is actually an interactive control — used by the adapter to surface
|
||||
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
|
||||
* `textContent`: the live parser always sets it, hand-built fixtures may not.
|
||||
*/
|
||||
cursor?: string;
|
||||
/**
|
||||
* Whether the live DOM snapshot provides a painted, non-hidden box for this
|
||||
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
|
||||
* nodes remain authoritative even when they are outside the viewport.
|
||||
*/
|
||||
rendered?: boolean;
|
||||
textContent?: string;
|
||||
formState?: "empty" | "filled" | "default";
|
||||
formValue?: string;
|
||||
formDefaultValue?: string;
|
||||
formPlaceholder?: string;
|
||||
}
|
||||
|
||||
export interface CapturedSurfaceProbe {
|
||||
triggerBackendNodeId: number;
|
||||
triggerPoint?: { x: number; y: number };
|
||||
triggerAction: "hover" | "focus" | string;
|
||||
subItems: string[];
|
||||
confidence?: "high" | "medium" | "low";
|
||||
}
|
||||
|
||||
/** Narrow input shared by the existing semantic scene/hover consumers. */
|
||||
export interface CapturedSceneInput {
|
||||
nodes: CapturedNode[];
|
||||
viewport: Viewport;
|
||||
rootFrameId?: string;
|
||||
excludedBackendNodeIds: ReadonlySet<number>;
|
||||
}
|
||||
import { captureCheckpoint } from "./capture-abort";
|
||||
import type { CapturedNode } from "./capture-types";
|
||||
|
||||
/** Bounds retain their snapshot document CSS units until normalization. */
|
||||
export interface SnapshotLayout {
|
||||
@@ -82,54 +28,6 @@ export interface DecodedDocument {
|
||||
nodes: DecodedNode[];
|
||||
}
|
||||
|
||||
export interface CaptureIssue {
|
||||
/** Owner failure or inherited blockage; absent for other geometry failures. */
|
||||
projectionIssue?: FrameProjectionIssue;
|
||||
target: CdpTarget;
|
||||
frameId?: string;
|
||||
stage: "dom" | "ax" | "ownership" | "geometry" | "forms";
|
||||
reason: "capture-unavailable" | "frame-ownership-unresolved" | "geometry-unavailable";
|
||||
}
|
||||
|
||||
export interface DocumentFacts<T extends FrameOwnedAxNode> {
|
||||
readonly frame: CdpFrame;
|
||||
readonly index: DocumentIndex;
|
||||
readonly domNodes: CapturedNode[];
|
||||
readonly axNodes: T[];
|
||||
}
|
||||
|
||||
export interface ObservationFacts<T extends FrameOwnedAxNode> {
|
||||
readonly rootFrameId: string;
|
||||
readonly viewport: Viewport;
|
||||
readonly documents: readonly DocumentFacts<T>[];
|
||||
readonly issues: readonly CaptureIssue[];
|
||||
readonly startedAt: number;
|
||||
readonly finishedAt: number;
|
||||
}
|
||||
|
||||
export function throwCaptureAborted(signal?: AbortSignal): void {
|
||||
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
||||
}
|
||||
|
||||
export function isCaptureAbort(error: unknown): boolean {
|
||||
return error instanceof Error && error.name === "AbortError";
|
||||
}
|
||||
|
||||
/** Called only at bounded block boundaries, not once per node. */
|
||||
export async function captureCheckpoint(signal?: AbortSignal): Promise<void> {
|
||||
throwCaptureAborted(signal);
|
||||
await new Promise<void>((resolve) => {
|
||||
const channel = new MessageChannel();
|
||||
channel.port1.onmessage = () => {
|
||||
channel.port1.close();
|
||||
channel.port2.close();
|
||||
resolve();
|
||||
};
|
||||
channel.port2.postMessage(null);
|
||||
});
|
||||
throwCaptureAborted(signal);
|
||||
}
|
||||
|
||||
/** All snapshot nodes participate, including document and shadow roots. This
|
||||
* preserves overlay propagation when the semantic adapter omits those nodes. */
|
||||
export async function buildDocumentIndex<T extends DecodedNode>(
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { CdpRunner } from "../shared";
|
||||
import type { CapturedNode } from "./facts";
|
||||
import { isCaptureAbort as isAbortError, throwCaptureAborted as throwIfAborted } from "./facts";
|
||||
import { isAbortError, throwIfAborted } from "./capture-abort";
|
||||
import type { CapturedNode } from "./capture-types";
|
||||
|
||||
interface RuntimeEvaluateReply {
|
||||
result?: { deepSerializedValue?: DeepSerializedValue };
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import { type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import { GeometryContext } from "../geometry/frame-context";
|
||||
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
|
||||
import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
|
||||
import {
|
||||
buildFrameDocuments,
|
||||
type FrameAxBatch,
|
||||
type FrameDocument,
|
||||
type FrameOwnedAxNode,
|
||||
} from "./frame-document";
|
||||
|
||||
export type FrameAxNode = FrameOwnedAxNode;
|
||||
export type CapturedFrameDocument<T extends FrameAxNode> = FrameDocument<T>;
|
||||
|
||||
function throwIfAborted(signal?: AbortSignal): void {
|
||||
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
||||
}
|
||||
|
||||
function isAbortError(err: unknown): boolean {
|
||||
return err instanceof Error && err.name === "AbortError";
|
||||
}
|
||||
|
||||
function transformFrameNodes(
|
||||
nodes: CapturedNode[],
|
||||
projection: GeometryProjection | null,
|
||||
): CapturedNode[] {
|
||||
return nodes.map((node) => ({
|
||||
...node,
|
||||
rect: projection
|
||||
? (() => {
|
||||
const bounds = projectRectToViewport(node.rect, projection);
|
||||
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||
})()
|
||||
: null,
|
||||
}));
|
||||
}
|
||||
|
||||
async function captureMissingFrameDocuments(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
graph: CdpFrameGraph,
|
||||
captured: CapturedViewModel,
|
||||
geometry: GeometryContext,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
if (!captured.frameNodes) captured.frameNodes = new Map();
|
||||
if (!captured.frameOwnerBackendNodeIds) captured.frameOwnerBackendNodeIds = new Map();
|
||||
if (!captured.frameParentIds) captured.frameParentIds = new Map();
|
||||
|
||||
for (const frame of graph.frames) {
|
||||
if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue;
|
||||
try {
|
||||
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
|
||||
signal,
|
||||
geometry,
|
||||
target: frame.target,
|
||||
});
|
||||
if (child.frameExcludedBackendNodeIds?.size) {
|
||||
captured.frameExcludedBackendNodeIds ??= new Map();
|
||||
for (const [id, excluded] of child.frameExcludedBackendNodeIds)
|
||||
captured.frameExcludedBackendNodeIds.set(id, excluded);
|
||||
}
|
||||
if (child.frameGeometryIssues?.length) {
|
||||
(captured.frameGeometryIssues ??= []).push(...child.frameGeometryIssues);
|
||||
}
|
||||
let projection: GeometryProjection | null = null;
|
||||
try {
|
||||
projection = await geometry.targetProjection(frame.frameId);
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
console.debug("[bsk observation] child frame geometry projection failed", {
|
||||
frameId: frame.frameId,
|
||||
err,
|
||||
});
|
||||
}
|
||||
const rootFrameId = child.rootFrameId ?? frame.frameId;
|
||||
const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
|
||||
if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
|
||||
if (rootFrameId !== frame.frameId && !childFrames.has(frame.frameId)) {
|
||||
childFrames.set(
|
||||
frame.frameId,
|
||||
child.nodes.map((node) => ({ ...node, frameId: frame.frameId })),
|
||||
);
|
||||
}
|
||||
for (const [childFrameId, nodes] of childFrames) {
|
||||
captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
|
||||
}
|
||||
if (frame.ownerBackendNodeId !== undefined) {
|
||||
captured.iframeNodes.set(
|
||||
frame.ownerBackendNodeId,
|
||||
captured.frameNodes.get(frame.frameId) ?? [],
|
||||
);
|
||||
captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
|
||||
}
|
||||
if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId);
|
||||
for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) {
|
||||
captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId);
|
||||
}
|
||||
for (const [childFrameId, parentFrameId] of child.frameParentIds ?? []) {
|
||||
captured.frameParentIds.set(childFrameId, parentFrameId);
|
||||
}
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
console.debug("[bsk observation] child frame DOM capture failed", {
|
||||
frameId: frame.frameId,
|
||||
err,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function captureAxTrees<T extends FrameAxNode>(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
discoveredFrames: CdpFrame[],
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
): Promise<FrameAxBatch<T>[]> {
|
||||
let frames = [...discoveredFrames];
|
||||
const capturedFrameIds = [...(captured.frameNodes?.keys() ?? [])];
|
||||
if (frames.length === 0) {
|
||||
const rootFrameId = captured.rootFrameId ?? capturedFrameIds[0] ?? "root";
|
||||
frames = [{ frameId: rootFrameId, target: { tabId } }];
|
||||
}
|
||||
for (const frameId of capturedFrameIds) {
|
||||
if (!frames.some((frame) => frame.frameId === frameId)) {
|
||||
frames.push({ frameId, target: { tabId } });
|
||||
}
|
||||
}
|
||||
|
||||
let firstFailure: unknown;
|
||||
const trees = await Promise.all(
|
||||
frames.map(async (frame): Promise<FrameAxBatch<T>> => {
|
||||
try {
|
||||
throwIfAborted(signal);
|
||||
await sendToCdpTarget(cdp, frame.target, "Accessibility.enable", {});
|
||||
const result = await sendToCdpTarget<{ nodes?: T[] }>(
|
||||
cdp,
|
||||
frame.target,
|
||||
"Accessibility.getFullAXTree",
|
||||
frame.frameId === "root" ? {} : { frameId: frame.frameId },
|
||||
);
|
||||
throwIfAborted(signal);
|
||||
return {
|
||||
frame,
|
||||
nodes: result.nodes ?? [],
|
||||
};
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
firstFailure ??= err;
|
||||
console.debug("[bsk observation] frame AX capture failed", {
|
||||
frameId: frame.frameId,
|
||||
err,
|
||||
});
|
||||
return { frame, nodes: [] };
|
||||
}
|
||||
}),
|
||||
);
|
||||
const capturedNodeCount = [...captured.iframeNodes.values()].reduce(
|
||||
(count, nodes) => count + nodes.length,
|
||||
captured.nodes.length,
|
||||
);
|
||||
if (firstFailure && capturedNodeCount === 0 && trees.every((tree) => tree.nodes.length === 0)) {
|
||||
throw firstFailure;
|
||||
}
|
||||
return trees;
|
||||
}
|
||||
|
||||
export async function captureFrameData<T extends FrameAxNode>(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
geometry = new GeometryContext(cdp, tabId, undefined, signal),
|
||||
): Promise<CapturedFrameDocument<T>[]> {
|
||||
const graph = await geometry.graph();
|
||||
const frames = graph?.frames ?? [];
|
||||
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, geometry, signal);
|
||||
throwIfAborted(signal);
|
||||
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
|
||||
return buildFrameDocuments(graph, batches, captured, signal);
|
||||
}
|
||||
@@ -4,8 +4,8 @@ import {
|
||||
type CdpTarget,
|
||||
cdpTargetKey,
|
||||
} from "@/browser-driver/frame-graph";
|
||||
import type { CapturedNode } from "./facts";
|
||||
import { captureCheckpoint } from "./facts";
|
||||
import { captureCheckpoint } from "./capture-abort";
|
||||
import type { CapturedNode } from "./capture-types";
|
||||
|
||||
export interface FrameOwnedAxNode {
|
||||
nodeId: string;
|
||||
@@ -32,6 +32,9 @@ export interface FrameDomInput {
|
||||
nodes: CapturedNode[];
|
||||
rootFrameId?: string;
|
||||
frameNodes?: ReadonlyMap<string, CapturedNode[]>;
|
||||
frameOwnerBackendNodeIds?: ReadonlyMap<string, number>;
|
||||
frameParentIds?: ReadonlyMap<string, string>;
|
||||
frameExcludedBackendNodeIds?: ReadonlyMap<string, ReadonlySet<number>>;
|
||||
}
|
||||
|
||||
interface Ownership {
|
||||
@@ -56,13 +59,27 @@ function targetBackendKey(target: CdpTarget, backendNodeId: number): string {
|
||||
function frameList<T extends FrameOwnedAxNode>(
|
||||
graph: CdpFrameGraph | null,
|
||||
batches: FrameAxBatch<T>[],
|
||||
captured: FrameDomInput,
|
||||
): CdpFrame[] {
|
||||
const frames = new Map<string, CdpFrame>();
|
||||
for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame);
|
||||
for (const batch of batches) {
|
||||
if (!frames.has(batch.frame.frameId)) frames.set(batch.frame.frameId, batch.frame);
|
||||
}
|
||||
return [...frames.values()];
|
||||
for (const frameId of captured.frameNodes?.keys() ?? []) {
|
||||
if (!frames.has(frameId))
|
||||
frames.set(frameId, { frameId, target: { tabId: batches[0]?.frame.target.tabId ?? 0 } });
|
||||
}
|
||||
return [...frames.values()].map((frame) => {
|
||||
const ownerBackendNodeId =
|
||||
frame.ownerBackendNodeId ?? captured.frameOwnerBackendNodeIds?.get(frame.frameId);
|
||||
const parentFrameId = frame.parentFrameId ?? captured.frameParentIds?.get(frame.frameId);
|
||||
return {
|
||||
...frame,
|
||||
...(ownerBackendNodeId !== undefined ? { ownerBackendNodeId } : {}),
|
||||
...(parentFrameId ? { parentFrameId } : {}),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
@@ -70,10 +87,9 @@ export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
batches: FrameAxBatch<T>[],
|
||||
captured: FrameDomInput,
|
||||
signal?: AbortSignal,
|
||||
unresolved?: (frame: CdpFrame) => void,
|
||||
): Promise<FrameDocument<T>[]> {
|
||||
let work = 0;
|
||||
const frames = frameList(graph, batches);
|
||||
const frames = frameList(graph, batches, captured);
|
||||
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
|
||||
const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId;
|
||||
const domNodesForFrame = (frameId: string): CapturedNode[] =>
|
||||
@@ -93,7 +109,6 @@ export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
if (work++ % 256 === 0) await captureCheckpoint(signal);
|
||||
nodeById.set(node.nodeId, node);
|
||||
}
|
||||
let reported = false;
|
||||
const ownershipByNodeId = new Map<string, Ownership>();
|
||||
const resolveOwnership = async (node: T): Promise<Ownership> => {
|
||||
const path: T[] = [];
|
||||
@@ -109,18 +124,8 @@ export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
}
|
||||
if (visiting.has(current.nodeId)) break;
|
||||
visiting.add(current.nodeId);
|
||||
if (current.frameId) {
|
||||
const frame = frameById.get(current.frameId);
|
||||
// An explicit contradiction must not fall through to a guessed owner.
|
||||
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(batch.frame.target)) {
|
||||
if (!reported) {
|
||||
unresolved?.(batch.frame);
|
||||
reported = true;
|
||||
}
|
||||
ownership = { frameId: "", strength: 0 };
|
||||
break;
|
||||
}
|
||||
ownership = { frameId: frame.frameId, strength: 4 };
|
||||
if (current.frameId && frameById.has(current.frameId)) {
|
||||
ownership = { frameId: current.frameId, strength: 4 };
|
||||
ownershipByNodeId.set(current.nodeId, ownership);
|
||||
break;
|
||||
}
|
||||
@@ -148,7 +153,6 @@ export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
for (const node of batch.nodes) {
|
||||
if (work++ % 256 === 0) await captureCheckpoint(signal);
|
||||
const ownership = await resolveOwnership(node);
|
||||
if (!ownership.frameId) continue;
|
||||
const key = targetNodeKey(batch.frame.target, node.nodeId);
|
||||
const existing = candidates.get(key);
|
||||
if (!existing || ownership.strength > existing.ownership.strength) {
|
||||
@@ -188,5 +192,6 @@ export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
contextScopeId: frame.frameId,
|
||||
axNodes: axNodesByFrame.get(frame.frameId) ?? [],
|
||||
domNodes: domNodesForFrame(frame.frameId),
|
||||
excludedBackendNodeIds: captured.frameExcludedBackendNodeIds?.get(frame.frameId),
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -1,233 +1,69 @@
|
||||
import type { Viewport } from "@browser-skill/vom";
|
||||
import { type CdpFrame, type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
|
||||
import { type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import {
|
||||
type FrameProjectionState,
|
||||
projectSnapshotRect,
|
||||
snapshotViewportRect,
|
||||
} from "../geometry/coordinate-types";
|
||||
import { cssViewport, GeometryContext } from "../geometry/frame-context";
|
||||
import type { CapturedNode } from "./facts";
|
||||
import {
|
||||
buildDocumentIndex,
|
||||
type CaptureIssue,
|
||||
captureCheckpoint,
|
||||
type DocumentIndex,
|
||||
isCaptureAbort,
|
||||
type NodeFacts,
|
||||
throwCaptureAborted,
|
||||
} from "./facts";
|
||||
import {
|
||||
decodeDocument,
|
||||
type SnapshotDocument,
|
||||
type SnapshotReply,
|
||||
snapshotFrameId,
|
||||
} from "./snapshot";
|
||||
import { captureCheckpoint } from "./capture-abort";
|
||||
import { buildDocumentIndex, type DocumentIndex, type NodeFacts } from "./facts";
|
||||
import { decodeDocument, type SnapshotDocument } from "./snapshot";
|
||||
|
||||
export interface FrameContext {
|
||||
frameId?: string;
|
||||
ownerFrameBackendNodeId: number | null;
|
||||
projection: FrameProjectionState;
|
||||
target: CdpTarget;
|
||||
scrollX: number;
|
||||
scrollY: number;
|
||||
}
|
||||
|
||||
export interface NormalizedDocument {
|
||||
nodes: NodeFacts[];
|
||||
index: DocumentIndex;
|
||||
frame: CdpFrame;
|
||||
domNodes: CapturedNode[];
|
||||
}
|
||||
|
||||
/** One target's snapshot. Source ownership is checked before interpreting any
|
||||
* coordinates. Snapshot and live quad coordinates are deliberately not conflated. */
|
||||
export async function normalizeSnapshot(
|
||||
snapshot: SnapshotReply,
|
||||
target: CdpTarget,
|
||||
frames: readonly CdpFrame[],
|
||||
geometry: GeometryContext,
|
||||
issues: CaptureIssue[],
|
||||
/** Interpret one document using a supplied projection. No live reads or frame scheduling. */
|
||||
export async function normalizeDocument(
|
||||
doc: SnapshotDocument,
|
||||
strings: string[],
|
||||
context: FrameContext,
|
||||
signal?: AbortSignal,
|
||||
rootFrameId?: string,
|
||||
): Promise<NormalizedDocument[]> {
|
||||
const strings = snapshot.strings ?? [];
|
||||
const raw = snapshot.documents ?? [];
|
||||
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
|
||||
const sources = new Map<string, SnapshotDocument>();
|
||||
for (const doc of raw) {
|
||||
const id = snapshotFrameId(doc, strings);
|
||||
if (!id || !frameById.has(id) || sources.has(id)) {
|
||||
issues.push({
|
||||
target,
|
||||
frameId: id,
|
||||
stage: "ownership",
|
||||
reason: "frame-ownership-unresolved",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
sources.set(id, doc);
|
||||
}
|
||||
let viewport: Viewport = { width: 0, height: 0 };
|
||||
let scrollX = 0,
|
||||
scrollY = 0;
|
||||
try {
|
||||
const measured = cssViewport(await geometry.layoutMetrics(target));
|
||||
viewport = { width: measured.width, height: measured.height };
|
||||
scrollX = measured.scrollX;
|
||||
scrollY = measured.scrollY;
|
||||
} catch (error) {
|
||||
if (isCaptureAbort(error)) throw error;
|
||||
}
|
||||
|
||||
const children = new Map<string, CdpFrame[]>();
|
||||
const pending: CdpFrame[] = [];
|
||||
for (const frame of frames) {
|
||||
if (!frame.parentFrameId || !frameById.has(frame.parentFrameId)) pending.push(frame);
|
||||
else {
|
||||
const siblings = children.get(frame.parentFrameId);
|
||||
if (siblings) siblings.push(frame);
|
||||
else children.set(frame.parentFrameId, [frame]);
|
||||
}
|
||||
}
|
||||
const projections = new Map<string, FrameProjectionState | null>();
|
||||
const result: NormalizedDocument[] = [];
|
||||
let targetProjection: GeometryProjection | null = null;
|
||||
const targetRoot = rootFrameId ? frameById.get(rootFrameId) : undefined;
|
||||
if (target.sessionId && targetRoot) {
|
||||
try {
|
||||
const known = await geometry.frame(targetRoot.frameId);
|
||||
if (known && cdpTargetKey(known.target) === cdpTargetKey(target))
|
||||
targetProjection = await geometry.targetProjection(targetRoot.frameId);
|
||||
} catch (error) {
|
||||
if (isCaptureAbort(error)) throw error;
|
||||
}
|
||||
}
|
||||
const visited = new Set<string>();
|
||||
const remaining = frames[Symbol.iterator]();
|
||||
for (let cursor = 0; visited.size < frames.length; cursor++) {
|
||||
if (cursor >= pending.length) {
|
||||
let next = remaining.next();
|
||||
while (!next.done && visited.has(next.value.frameId)) next = remaining.next();
|
||||
if (next.done) break;
|
||||
pending.push(next.value);
|
||||
}
|
||||
if (cursor % 256 === 0) await captureCheckpoint(signal);
|
||||
const frame = pending[cursor];
|
||||
if (visited.has(frame.frameId)) continue;
|
||||
visited.add(frame.frameId);
|
||||
for (const child of children.get(frame.frameId) ?? []) pending.push(child);
|
||||
const doc = sources.get(frame.frameId);
|
||||
if (!doc) continue;
|
||||
const source = { target, frameId: frame.frameId };
|
||||
const state: FrameProjectionState | null =
|
||||
frame.frameId === rootFrameId
|
||||
? {
|
||||
status: "available",
|
||||
projection: { source, geometry: { sourceClips: [], edges: [], topViewport: viewport } },
|
||||
}
|
||||
: (projections.get(frame.frameId) ?? null);
|
||||
if (!state && frame.frameId !== rootFrameId)
|
||||
issues.push({
|
||||
target,
|
||||
frameId: frame.frameId,
|
||||
stage: "ownership",
|
||||
reason: "frame-ownership-unresolved",
|
||||
});
|
||||
projections.set(frame.frameId, state);
|
||||
const projection = state?.status === "available" ? state.projection : null;
|
||||
if (!projection || (target.sessionId && !targetProjection))
|
||||
issues.push({
|
||||
target,
|
||||
frameId: frame.frameId,
|
||||
stage: "geometry",
|
||||
reason: "geometry-unavailable",
|
||||
...(state && state.status !== "available" ? { projectionIssue: state } : {}),
|
||||
});
|
||||
|
||||
// Only direct siblings share this pool. Keep normalization in its existing
|
||||
// breadth-first order, independent of owner read completion order.
|
||||
const siblings = (children.get(frame.frameId) ?? []).filter((child) =>
|
||||
sources.has(child.frameId),
|
||||
): Promise<NormalizedDocument> {
|
||||
const decoded = await decodeDocument(doc, strings, signal);
|
||||
const nodes: NodeFacts[] = [];
|
||||
for (let i = 0; i < decoded.nodes.length; i++) {
|
||||
if (i % 256 === 0) await captureCheckpoint(signal);
|
||||
const node = decoded.nodes[i];
|
||||
const input = snapshotViewportRect(
|
||||
node.layout?.bounds ?? [],
|
||||
{ target: context.target, frameId: context.frameId },
|
||||
{ x: context.scrollX, y: context.scrollY },
|
||||
);
|
||||
const edge = projection?.geometry.edges[0];
|
||||
const clips = edge
|
||||
? [edge.destinationQuad, ...(edge.destinationClips ?? [])]
|
||||
: (projection?.geometry.sourceClips ?? []);
|
||||
let next = 0;
|
||||
let failure: unknown;
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(4, siblings.length) }, async () => {
|
||||
while (next < siblings.length && !signal?.aborted && !failure) {
|
||||
const child = siblings[next++];
|
||||
const childSource = { target, frameId: child.frameId };
|
||||
if (!projection) {
|
||||
projections.set(
|
||||
child.frameId,
|
||||
state && state.status !== "available"
|
||||
? {
|
||||
status: "blocked",
|
||||
source: childSource,
|
||||
cause: state.status === "blocked" ? state.cause : state,
|
||||
}
|
||||
: null,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (child.ownerBackendNodeId === undefined) {
|
||||
projections.set(child.frameId, null);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
projections.set(
|
||||
child.frameId,
|
||||
await geometry.snapshotProjection(
|
||||
childSource,
|
||||
child.ownerBackendNodeId,
|
||||
clips,
|
||||
viewport,
|
||||
),
|
||||
);
|
||||
} catch (error) {
|
||||
failure ??= error;
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
// Join active reads and their cleanup before returning cancellation.
|
||||
if (failure) throw failure;
|
||||
throwCaptureAborted(signal);
|
||||
const decoded = await decodeDocument(doc, strings, signal);
|
||||
const nodes: NodeFacts[] = [];
|
||||
for (let i = 0; i < decoded.nodes.length; i++) {
|
||||
if (i % 256 === 0) await captureCheckpoint(signal);
|
||||
const node = decoded.nodes[i];
|
||||
const input = snapshotViewportRect(node.layout?.bounds ?? [], source, {
|
||||
x: doc.scrollOffsetX ?? (frame.frameId === rootFrameId ? scrollX : 0),
|
||||
y: doc.scrollOffsetY ?? (frame.frameId === rootFrameId ? scrollY : 0),
|
||||
});
|
||||
const local = input?.rect;
|
||||
let rect = input && projection ? projectSnapshotRect(input, projection) : null;
|
||||
if (rect && target.sessionId)
|
||||
rect = targetProjection
|
||||
? projectRectToViewport(
|
||||
{ x: rect.x, y: rect.y, w: rect.width, h: rect.height },
|
||||
targetProjection,
|
||||
)
|
||||
: null;
|
||||
const styles = node.layout?.styles;
|
||||
const visibility = styles?.visibility || "visible";
|
||||
const opacity = styles?.opacity || "1";
|
||||
nodes.push({
|
||||
...node,
|
||||
frameId: frame.frameId,
|
||||
ownerFrameBackendNodeId: frame.ownerBackendNodeId ?? null,
|
||||
localRect: local ? { x: local.x, y: local.y, w: local.width, h: local.height } : null,
|
||||
rect: rect ? { x: rect.x, y: rect.y, w: rect.width, h: rect.height } : null,
|
||||
rendered:
|
||||
!!local &&
|
||||
visibility !== "hidden" &&
|
||||
visibility !== "collapse" &&
|
||||
(Number.parseFloat(opacity) || 0) > 0,
|
||||
});
|
||||
}
|
||||
const index = await buildDocumentIndex(nodes, signal);
|
||||
const domNodes = nodes.filter((node) => !node.tag.startsWith("#"));
|
||||
result.push({
|
||||
index,
|
||||
frame,
|
||||
domNodes,
|
||||
const local = input?.rect;
|
||||
const rect =
|
||||
input && context.projection.status === "available"
|
||||
? projectSnapshotRect(input, context.projection.projection)
|
||||
: null;
|
||||
const visibility = node.layout?.styles.visibility || "visible";
|
||||
const opacity = node.layout?.styles.opacity || "1";
|
||||
nodes.push({
|
||||
...node,
|
||||
...(context.frameId ? { frameId: context.frameId } : {}),
|
||||
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
|
||||
localRect: local ? { x: local.x, y: local.y, w: local.width, h: local.height } : null,
|
||||
rect: rect ? { x: rect.x, y: rect.y, w: rect.width, h: rect.height } : null,
|
||||
rendered:
|
||||
!!local &&
|
||||
visibility !== "hidden" &&
|
||||
visibility !== "collapse" &&
|
||||
(Number.parseFloat(opacity) || 0) > 0,
|
||||
});
|
||||
}
|
||||
return result;
|
||||
const index = await buildDocumentIndex(nodes, signal);
|
||||
return {
|
||||
nodes: nodes.filter(
|
||||
(node) => !node.tag.startsWith("#") && !index.excludedBackendNodeIds.has(node.backendNodeId),
|
||||
),
|
||||
index,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { type CdpFrame, type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
|
||||
import type { CapturedNode } from "./facts";
|
||||
import { captureCheckpoint, type DecodedDocument, type DecodedNode } from "./facts";
|
||||
import { captureCheckpoint } from "./capture-abort";
|
||||
import type { CapturedNode } from "./capture-types";
|
||||
import type { DecodedDocument, DecodedNode } from "./facts";
|
||||
export const REQUESTED_STYLES = [
|
||||
"position",
|
||||
"pointer-events",
|
||||
@@ -23,7 +23,6 @@ interface RareBooleanData {
|
||||
|
||||
export interface SnapshotDocument {
|
||||
frameId?: string | number;
|
||||
documentURL?: number;
|
||||
scrollOffsetX?: number;
|
||||
scrollOffsetY?: number;
|
||||
nodes?: {
|
||||
@@ -87,7 +86,7 @@ function str(strings: string[], idx: number | undefined): string {
|
||||
return strings[idx] ?? "";
|
||||
}
|
||||
|
||||
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
|
||||
export function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
|
||||
const out = new Map<number, number>();
|
||||
if (!sparse?.index || !sparse.value) return out;
|
||||
for (let i = 0; i < sparse.index.length; i++) {
|
||||
@@ -213,68 +212,3 @@ export async function decodeDocument(
|
||||
}
|
||||
return { nodes };
|
||||
}
|
||||
|
||||
/** Snapshot membership and same-target edges are current response evidence.
|
||||
* The earlier graph only supplies compatible metadata and cross-target edges. */
|
||||
export function describeSnapshotFrames(
|
||||
snapshot: SnapshotReply,
|
||||
target: CdpTarget,
|
||||
hintById: ReadonlyMap<string, CdpFrame>,
|
||||
pageUrl?: string,
|
||||
): { frames: CdpFrame[]; ids: Set<string>; rootFrameId?: string } {
|
||||
const raw = snapshot.documents ?? [];
|
||||
const strings = snapshot.strings ?? [];
|
||||
const ids = new Set<string>();
|
||||
const invalid = new Set<string>();
|
||||
const frames = raw.map((doc) => {
|
||||
const frameId = snapshotFrameId(doc, strings);
|
||||
if (!frameId) return undefined;
|
||||
if (ids.has(frameId)) invalid.add(frameId);
|
||||
ids.add(frameId);
|
||||
const hint = hintById.get(frameId);
|
||||
const compatible =
|
||||
hint && cdpTargetKey(hint.target) === cdpTargetKey(target) ? hint : undefined;
|
||||
const url =
|
||||
(doc.documentURL === undefined ? undefined : strings[doc.documentURL]) || compatible?.url;
|
||||
return { frameId, target, ...(url ? { url } : {}) } as CdpFrame;
|
||||
});
|
||||
const root = frames[0];
|
||||
if (root) {
|
||||
const hint = hintById.get(root.frameId);
|
||||
const parent = hint?.parentFrameId ? hintById.get(hint.parentFrameId) : undefined;
|
||||
if (
|
||||
target.sessionId &&
|
||||
hint &&
|
||||
cdpTargetKey(hint.target) === cdpTargetKey(target) &&
|
||||
parent &&
|
||||
cdpTargetKey(parent.target) !== cdpTargetKey(target)
|
||||
) {
|
||||
root.parentFrameId = parent.frameId;
|
||||
root.ownerBackendNodeId = hint.ownerBackendNodeId;
|
||||
}
|
||||
if (!target.sessionId && !root.url && pageUrl) root.url = pageUrl;
|
||||
}
|
||||
const owned = new Set<string>();
|
||||
for (let i = 0; i < raw.length; i++) {
|
||||
const edges = raw[i].nodes?.contentDocumentIndex;
|
||||
const parent = frames[i];
|
||||
if (!parent || !edges) continue;
|
||||
for (let e = 0; e < edges.index.length; e++) {
|
||||
const child = frames[edges.value[e]];
|
||||
const owner = raw[i].nodes?.backendNodeId?.[edges.index[e]];
|
||||
if (!child || owner === undefined) continue;
|
||||
if (child === root || child === parent || owned.has(child.frameId)) {
|
||||
invalid.add(child.frameId);
|
||||
continue;
|
||||
}
|
||||
owned.add(child.frameId);
|
||||
child.parentFrameId = parent.frameId;
|
||||
child.ownerBackendNodeId = owner;
|
||||
}
|
||||
}
|
||||
return {
|
||||
frames: frames.filter((frame): frame is CdpFrame => !!frame && !invalid.has(frame.frameId)),
|
||||
ids,
|
||||
rootFrameId: root?.frameId,
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user