figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(observation): organize capture around document facts

Browse Source
This commit is contained in:
Ljy-0827
2026-09-09 15:54:41 +08:00
committed by drakezhang
parent 383e3128eb
commit 8dfa097e55
18 changed files with 2361 additions and 1496 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/extension/src/tools/__tests__/observation.test.ts
+224 -174
View File
@@ -4,9 +4,9 @@ import { CAPTURE_SUPPRESS, type CaptureSuppressMessage } from "@/lib/capture-sup
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "@/lib/overlay-bridge";
import { SessionManager } from "@/session-manager/manager";
import type { CdpRunner } from "@/tools/shared";
import type { BuildVomSceneOptions, VomFrameDocument } from "../observation";
import {
buildFrameVomScene,
buildVomScene,
type CdpAxNode,
captureVomObservation,
handleGetHtml,
@@ -17,7 +17,35 @@ import {
type ScreenshotDeps,
stripDataUrlPrefix,
} from "../observation";
import type { CapturedNode, CapturedViewModel } from "../vom/capture";
import type { CapturedNode, CapturedSceneInput } from "../vom/facts";
interface SceneFixture extends CapturedSceneInput {
documents?: VomFrameDocument[];
}
// Only supplies the common root fixture; child documents declare their own ownership.
function buildTestScene(
axNodes: CdpAxNode[],
captured: SceneFixture,
options: BuildVomSceneOptions = {},
) {
const rootFrameId = captured.rootFrameId ?? "root";
return buildFrameVomScene(
[
{
frameId: rootFrameId,
contextScopeId: rootFrameId,
target: { tabId: 7 },
url: options.pageUrl,
domNodes: captured.nodes,
axNodes,
},
...(captured.documents ?? []),
],
captured,
options,
);
}
function fakeAgentWindow(ids: number[]) {
let i = 0;
@@ -526,10 +554,10 @@ describe("handleScreenshot overlay suppression", () => {
});
// ---------------------------------------------------------------------------
// buildVomScene
// Semantic scene fixtures
// ---------------------------------------------------------------------------
describe("buildVomScene", () => {
describe("buildFrameVomScene", () => {
it("merges multiple sibling and nested frames without mixing their targets", () => {
const node = (
backendNodeId: number,
@@ -567,25 +595,10 @@ describe("buildVomScene", () => {
node(300, null, "nested", "body"),
node(301, 300, "nested", "input", { placeholder: "Nested field" }),
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1200, height: 800 },
nodes: mainNodes,
iframeNodes: new Map([
[10, firstNodes],
[20, secondNodes],
[210, nestedNodes],
]),
frameNodes: new Map([
["main", mainNodes],
["first", firstNodes],
["second", secondNodes],
["nested", nestedNodes],
]),
frameOwnerBackendNodeIds: new Map([
["first", 10],
["second", 20],
["nested", 210],
]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
@@ -684,11 +697,10 @@ describe("buildVomScene", () => {
position: "static",
pointerEvents: "auto",
};
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 800, height: 600 },
nodes: [],
iframeNodes: new Map(),
frameNodes: new Map([["child", [childNode]]]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
@@ -750,14 +762,10 @@ describe("buildVomScene", () => {
pointerEvents: "auto",
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 800, height: 600 },
nodes: mainNodes,
iframeNodes: new Map(),
frameNodes: new Map([
["main", mainNodes],
["child", []],
]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
@@ -833,9 +841,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 200,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -861,11 +868,11 @@ describe("buildVomScene", () => {
],
};
const scene = buildVomScene(axNodes, captured);
const scene = buildTestScene(axNodes, captured);
expect(scene.viewport).toEqual({ width: 1000, height: 800 });
expect(scene.nodes[0]).toEqual(
expect.objectContaining({
id: 100,
backendNodeId: 100,
parentId: null,
role: "RootWebArea",
name: "Example",
@@ -877,14 +884,14 @@ describe("buildVomScene", () => {
);
expect(scene.nodes[1]).toEqual(
expect.objectContaining({
id: 200,
parentId: 100,
backendNodeId: 200,
parentId: scene.nodes.find((node) => node.backendNodeId === 100)!.id,
role: "button",
name: "Submit",
tag: "button",
rect: { x: 20, y: 20, w: 120, h: 40 },
domParentId: 100,
domAncestorIds: [100],
domParentId: scene.nodes.find((node) => node.backendNodeId === 100)!.id,
domAncestorIds: [scene.nodes.find((node) => node.backendNodeId === 100)!.id],
attrs: {},
}),
);
@@ -913,9 +920,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 300,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -951,10 +957,10 @@ describe("buildVomScene", () => {
],
};
const scene = buildVomScene(axNodes, captured, { pageUrl: "https://app.example.test/home" });
const scene = buildTestScene(axNodes, captured, { pageUrl: "https://app.example.test/home" });
expect(scene.nodes.find((node) => node.id === 200)?.href).toBe("docs.example.org");
expect(scene.nodes.find((node) => node.id === 300)?.href).toBeUndefined();
expect(scene.nodes.find((node) => node.backendNodeId === 200)?.href).toBe("docs.example.org");
expect(scene.nodes.find((node) => node.backendNodeId === 300)?.href).toBeUndefined();
});
it("uses the nearest backend AX ancestor as parentId", () => {
@@ -979,9 +985,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 30,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 400, height: 300 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1007,10 +1012,12 @@ describe("buildVomScene", () => {
],
};
const scene = buildVomScene(axNodes, captured);
const scene = buildTestScene(axNodes, captured);
expect(scene.nodes.map((node) => node.id)).toEqual([10, 30]);
expect(scene.nodes.find((node) => node.id === 30)?.parentId).toBe(10);
expect(scene.nodes.map((node) => node.backendNodeId)).toEqual([10, 30]);
expect(scene.nodes.find((node) => node.backendNodeId === 30)?.parentId).toBe(
scene.nodes.find((node) => node.backendNodeId === 10)!.id,
);
});
it("maps iframe sub-document controls to VomNodes", () => {
@@ -1028,7 +1035,7 @@ describe("buildVomScene", () => {
backendDOMNodeId: 20,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
nodes: [
{
@@ -1052,10 +1059,15 @@ describe("buildVomScene", () => {
pointerEvents: "auto",
},
],
iframeNodes: new Map([
[
20,
[
documents: [
{
frameId: "child",
contextScopeId: "child",
parentFrameId: "root",
ownerBackendNodeId: 20,
target: { tabId: 7 },
axNodes: [],
domNodes: [
{
backendNodeId: 101,
parentBackendNodeId: null,
@@ -1077,26 +1089,25 @@ describe("buildVomScene", () => {
pointerEvents: "auto",
},
],
],
]),
},
],
excludedBackendNodeIds: new Set(),
};
const iframeControls = buildVomScene(axNodes, captured).nodes.filter((node) =>
[101, 102].includes(node.id),
);
const scene = buildTestScene(axNodes, captured);
const iframeControls = scene.nodes.filter((node) => [101, 102].includes(node.backendNodeId!));
expect(iframeControls).toEqual([
expect.objectContaining({
id: 101,
parentId: 20,
backendNodeId: 101,
parentId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
role: "textbox",
name: "请输入手机号",
sensitive: false,
}),
expect.objectContaining({
id: 102,
parentId: 20,
backendNodeId: 102,
parentId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
role: "textbox",
name: "密码",
sensitive: true,
@@ -1117,11 +1128,9 @@ describe("buildVomScene", () => {
parentId: "1",
role: { type: "role", value: "Iframe" },
backendDOMNodeId: 20,
childIds: ["3"],
},
{
nodeId: "3",
parentId: "2",
role: { type: "role", value: "textbox" },
name: { type: "x", value: "输入密码" },
value: { value: "iframe-secret" },
@@ -1129,7 +1138,7 @@ describe("buildVomScene", () => {
properties: [{ name: "inputType", value: { value: "password" } }],
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
nodes: [
{
@@ -1153,10 +1162,15 @@ describe("buildVomScene", () => {
pointerEvents: "auto",
},
],
iframeNodes: new Map([
[
20,
[
documents: [
{
frameId: "child",
contextScopeId: "child",
parentFrameId: "root",
ownerBackendNodeId: 20,
target: { tabId: 7 },
axNodes: [axNodes[2]],
domNodes: [
{
backendNodeId: 102,
parentBackendNodeId: null,
@@ -1168,25 +1182,27 @@ describe("buildVomScene", () => {
pointerEvents: "auto",
},
],
],
]),
},
],
excludedBackendNodeIds: new Set(),
};
const passwordNodes = buildVomScene(axNodes, captured).nodes.filter(
const passwordNodes = buildTestScene(axNodes.slice(0, 2), captured).nodes.filter(
(node) => node.role === "textbox" && node.name === "输入密码",
);
expect(passwordNodes).toHaveLength(1);
expect(passwordNodes[0]).toEqual(expect.objectContaining({ id: 202, sensitive: true }));
expect(passwordNodes[0]).toEqual(
expect.objectContaining({ backendNodeId: 202, sensitive: true }),
);
expect(passwordNodes[0].value).toBeUndefined();
const rendered = renderVom(buildVomScene(axNodes, captured)).text;
const rendered = renderVom(buildTestScene(axNodes.slice(0, 2), captured)).text;
expect(rendered).toContain('textbox "输入密码" [filled] ="•••"');
expect(rendered).not.toContain("iframe-secret");
});
it("keeps unnamed iframe controls but skips unnamed iframe links", () => {
const scene = buildVomScene(
const scene = buildTestScene(
[
{
nodeId: "1",
@@ -1225,10 +1241,15 @@ describe("buildVomScene", () => {
pointerEvents: "auto",
},
],
iframeNodes: new Map([
[
20,
[
documents: [
{
frameId: "child",
contextScopeId: "child",
parentFrameId: "root",
ownerBackendNodeId: 20,
target: { tabId: 7 },
axNodes: [],
domNodes: [
{
backendNodeId: 201,
parentBackendNodeId: null,
@@ -1250,25 +1271,25 @@ describe("buildVomScene", () => {
pointerEvents: "auto",
},
],
],
]),
},
],
excludedBackendNodeIds: new Set(),
},
);
expect(scene.nodes.find((node) => node.id === 201)).toEqual(
expect(scene.nodes.find((node) => node.backendNodeId === 201)).toEqual(
expect.objectContaining({
id: 201,
parentId: 20,
backendNodeId: 201,
parentId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
role: "button",
}),
);
expect(scene.nodes.find((node) => node.id === 201)).not.toHaveProperty("name");
expect(scene.nodes.find((node) => node.id === 202)).toBeUndefined();
expect(scene.nodes.find((node) => node.backendNodeId === 201)).not.toHaveProperty("name");
expect(scene.nodes.find((node) => node.backendNodeId === 202)).toBeUndefined();
});
it("preserves captured-only iframe anchors for iframe sub-document controls", () => {
const scene = buildVomScene(
const scene = buildTestScene(
[
{
nodeId: "1",
@@ -1301,10 +1322,15 @@ describe("buildVomScene", () => {
pointerEvents: "auto",
},
],
iframeNodes: new Map<number, CapturedNode[]>([
[
20,
[
documents: [
{
frameId: "child",
contextScopeId: "child",
parentFrameId: "root",
ownerBackendNodeId: 20,
target: { tabId: 7 },
axNodes: [],
domNodes: [
{
backendNodeId: 201,
parentBackendNodeId: null,
@@ -1326,16 +1352,16 @@ describe("buildVomScene", () => {
pointerEvents: "auto",
},
],
],
]),
},
],
excludedBackendNodeIds: new Set(),
},
);
expect(scene.nodes.find((node) => node.id === 20)).toEqual(
expect(scene.nodes.find((node) => node.backendNodeId === 20)).toEqual(
expect.objectContaining({
id: 20,
parentId: 10,
backendNodeId: 20,
parentId: scene.nodes.find((node) => node.backendNodeId === 10)!.id,
role: "Iframe",
name: "支付验证",
}),
@@ -1356,9 +1382,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 10,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 320, height: 240 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1379,9 +1404,9 @@ describe("buildVomScene", () => {
],
};
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
expect(buildTestScene(axNodes, captured).nodes[0]).toEqual(
expect.objectContaining({
id: 10,
backendNodeId: 10,
sensitive: true,
}),
);
@@ -1396,9 +1421,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 10,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1415,9 +1439,9 @@ describe("buildVomScene", () => {
],
};
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
expect(buildTestScene(axNodes, captured).nodes[0]).toEqual(
expect.objectContaining({
id: 10,
backendNodeId: 10,
role: "textbox",
name: "+86",
placeholder: "输入手机号",
@@ -1435,9 +1459,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 10,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1454,8 +1477,13 @@ describe("buildVomScene", () => {
],
};
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
expect.objectContaining({ id: 10, role: "textbox", name: "Email", value: "a@b.com" }),
expect(buildTestScene(axNodes, captured).nodes[0]).toEqual(
expect.objectContaining({
backendNodeId: 10,
role: "textbox",
name: "Email",
value: "a@b.com",
}),
);
});
@@ -1474,9 +1502,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 20,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1514,9 +1541,9 @@ describe("buildVomScene", () => {
],
};
expect(buildVomScene(axNodes, captured).nodes.find((node) => node.id === 20)).toEqual(
expect.objectContaining({ role: "textbox", name: "验证码" }),
);
expect(
buildTestScene(axNodes, captured).nodes.find((node) => node.backendNodeId === 20),
).toEqual(expect.objectContaining({ role: "textbox", name: "验证码" }));
});
it("uses preceding AX static text as a form control label", () => {
@@ -1540,9 +1567,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 20,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1559,13 +1585,13 @@ describe("buildVomScene", () => {
],
};
expect(buildVomScene(axNodes, captured).nodes.find((node) => node.id === 20)).toEqual(
expect.objectContaining({ role: "textbox", name: "验证码", inputState: "empty" }),
);
expect(
buildTestScene(axNodes, captured).nodes.find((node) => node.backendNodeId === 20),
).toEqual(expect.objectContaining({ role: "textbox", name: "验证码", inputState: "empty" }));
});
it("marks runtime default input values without treating them as ordinary filled input", () => {
const scene = buildVomScene(
const scene = buildTestScene(
[
{
nodeId: "1",
@@ -1575,7 +1601,6 @@ describe("buildVomScene", () => {
],
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1619,9 +1644,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 20,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1671,9 +1695,14 @@ describe("buildVomScene", () => {
],
};
const scene = buildVomScene(axNodes, captured);
expect(scene.nodes.find((n) => n.id === 20)).toEqual(
expect.objectContaining({ id: 20, role: "button", name: "close", cursor: "pointer" }),
const scene = buildTestScene(axNodes, captured);
expect(scene.nodes.find((n) => n.backendNodeId === 20)).toEqual(
expect.objectContaining({
backendNodeId: 20,
role: "button",
name: "close",
cursor: "pointer",
}),
);
const rendered = renderVom(scene);
expect(rendered.text).toContain('@e1 button "close"');
@@ -1707,9 +1736,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 30,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1748,9 +1776,9 @@ describe("buildVomScene", () => {
],
};
const scene = buildVomScene(axNodes, captured);
expect(scene.nodes.find((n) => n.id === 20)).toBeUndefined();
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("link");
const scene = buildTestScene(axNodes, captured);
expect(scene.nodes.find((n) => n.backendNodeId === 20)).toBeUndefined();
expect(scene.nodes.find((n) => n.backendNodeId === 30)?.role).toBe("link");
});
it("promotes only the outermost clickable in a nested pointer chain", () => {
@@ -1775,9 +1803,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 30,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1816,11 +1843,15 @@ describe("buildVomScene", () => {
],
};
const scene = buildVomScene(axNodes, captured);
expect(scene.nodes.find((n) => n.id === 20)).toEqual(
expect.objectContaining({ id: 20, role: "button", attrs: { "aria-label": "收藏" } }),
const scene = buildTestScene(axNodes, captured);
expect(scene.nodes.find((n) => n.backendNodeId === 20)).toEqual(
expect.objectContaining({
backendNodeId: 20,
role: "button",
attrs: { "aria-label": "收藏" },
}),
);
expect(scene.nodes.find((n) => n.id === 30)).toBeUndefined();
expect(scene.nodes.find((n) => n.backendNodeId === 30)).toBeUndefined();
const rendered = renderVom(scene);
expect(rendered.text).toContain('@e1 button "收藏"');
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
@@ -1844,9 +1875,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 20,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1894,10 +1924,10 @@ describe("buildVomScene", () => {
],
};
const scene = buildVomScene(axNodes, captured);
const scene = buildTestScene(axNodes, captured);
expect(scene.activeScopeBlocks).toEqual([
{
triggerId: 20,
triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
label: "Reviews (12)",
lines: ["Jane - ear cups are small", "Bob - great sound"],
},
@@ -1921,11 +1951,10 @@ describe("buildVomScene", () => {
backendDOMNodeId: 20,
},
];
const scene = buildVomScene(
const scene = buildTestScene(
axNodes,
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1958,7 +1987,11 @@ describe("buildVomScene", () => {
);
expect(scene.surfaces).toEqual([
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
{
triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
triggerAction: "hover",
subItems: ["Shoes", "Bags"],
},
]);
expect(renderVom(scene).text).toContain('@e1 button "Products" [hover first: Shoes | Bags]');
});
@@ -1979,11 +2012,10 @@ describe("buildVomScene", () => {
backendDOMNodeId: 21,
},
];
const scene = buildVomScene(
const scene = buildTestScene(
axNodes,
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -2044,11 +2076,10 @@ describe("buildVomScene", () => {
backendDOMNodeId: 21,
},
];
const scene = buildVomScene(
const scene = buildTestScene(
axNodes,
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -2093,7 +2124,11 @@ describe("buildVomScene", () => {
);
expect(scene.surfaces).toEqual([
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile"] },
{
triggerId: scene.nodes.find((node) => node.backendNodeId === 21)!.id,
triggerAction: "hover",
subItems: ["My profile"],
},
]);
});
@@ -2113,11 +2148,10 @@ describe("buildVomScene", () => {
backendDOMNodeId: 21,
},
];
const scene = buildVomScene(
const scene = buildTestScene(
axNodes,
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -2168,7 +2202,11 @@ describe("buildVomScene", () => {
);
expect(scene.surfaces).toEqual([
{ triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
{
triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
triggerAction: "hover",
subItems: ["My profile", "Sign out"],
},
]);
expect(renderVom(scene).text).toContain(
'@e1 button "image" [hover first: My profile | Sign out]',
@@ -2191,11 +2229,10 @@ describe("buildVomScene", () => {
backendDOMNodeId: 42,
},
];
const scene = buildVomScene(
const scene = buildTestScene(
axNodes,
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -2265,9 +2302,8 @@ describe("buildVomScene", () => {
properties: [{ name: "inputType", value: { value: "credit-card" } }],
},
];
const scene = buildVomScene(axNodes, {
const scene = buildTestScene(axNodes, {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -2314,7 +2350,7 @@ describe("buildVomScene", () => {
],
});
expect(scene.nodes.find((node) => node.id === 20)).toEqual(
expect(scene.nodes.find((node) => node.backendNodeId === 20)).toEqual(
expect.objectContaining({
name: "Products [expanded]",
attrs: expect.objectContaining({
@@ -2324,13 +2360,17 @@ describe("buildVomScene", () => {
}),
);
expect(scene.activeScopeBlocks).toEqual([
{ triggerId: 20, label: "Products [expanded]", lines: ["Shoes Bags"] },
{
triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
label: "Products [expanded]",
lines: ["Shoes Bags"],
},
]);
expect(scene.nodes.find((node) => node.id === 30)?.sensitive).toBe(true);
expect(scene.nodes.find((node) => node.backendNodeId === 30)?.sensitive).toBe(true);
});
it("aggregates AX virtual text into unnamed structural nodes", () => {
const scene = buildVomScene(
const scene = buildTestScene(
[
{
nodeId: "1",
@@ -2354,7 +2394,6 @@ describe("buildVomScene", () => {
],
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -2381,13 +2420,12 @@ describe("buildVomScene", () => {
},
);
expect(scene.nodes.find((node) => node.id === 20)?.name).toBe("Inline only text");
expect(scene.nodes.find((node) => node.backendNodeId === 20)?.name).toBe("Inline only text");
});
it("marks captured dialog elements as modal without AX role or aria-modal", () => {
const scene = buildVomScene([], {
const scene = buildTestScene([], {
viewport: { width: 640, height: 480 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -2405,7 +2443,7 @@ describe("buildVomScene", () => {
expect(scene.nodes[0]).toEqual(
expect.objectContaining({
id: 50,
backendNodeId: 50,
modal: true,
}),
);
@@ -2434,9 +2472,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 202,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set([200, 201, 202]),
nodes: [
{
@@ -2452,9 +2489,9 @@ describe("buildVomScene", () => {
],
};
const scene = buildVomScene(axNodes, captured);
expect(scene.nodes.find((n) => n.id === 202)).toBeUndefined();
expect(scene.nodes.find((n) => n.id === 2)).toEqual(
const scene = buildTestScene(axNodes, captured);
expect(scene.nodes.find((n) => n.backendNodeId === 202)).toBeUndefined();
expect(scene.nodes.find((n) => n.backendNodeId === 2)).toEqual(
expect.objectContaining({ role: "button", name: "Real Button" }),
);
});
@@ -2482,13 +2519,12 @@ describe("buildVomScene", () => {
backendDOMNodeId: 203,
},
];
const scene = buildVomScene(axNodes, {
const scene = buildTestScene(axNodes, {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set([200]),
nodes: [],
});
expect(scene.nodes.find((n) => n.id === 203)).toBeUndefined();
expect(scene.nodes.find((n) => n.backendNodeId === 203)).toBeUndefined();
});
});
@@ -2601,6 +2637,7 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0],
nodeName: [i("body"), i("button")],
@@ -2696,6 +2733,7 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0],
nodeName: [i("body"), i("button")],
@@ -2794,6 +2832,7 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0],
nodeName: [i("body"), i("button")],
@@ -3062,6 +3101,7 @@ describe("handleSnapshot", () => {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 2],
nodeName: [i("html"), i("body"), i("div"), i("input")],
@@ -3119,6 +3159,7 @@ describe("handleSnapshot", () => {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 1, 1, 1],
nodeName: [i("html"), i("body"), i("input"), i("input"), i("input"), i("input")],
@@ -3270,12 +3311,17 @@ describe("handleSnapshot", () => {
if (method === "Runtime.releaseObject") return {};
}
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "DOMSnapshot.captureSnapshot")
return { ...snapshot, documents: [snapshot.documents[0]] };
if (method === "Accessibility.getFullAXTree") return { nodes: mainAx };
throw new Error(`unexpected root CDP method ${method}`);
});
const sendToTarget = vi.fn(async (_target, method: string) => {
if (method === "Accessibility.enable") return {};
if (method === "Page.getLayoutMetrics" && ownerGeometry !== "unavailable")
return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } };
if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot")
return { ...snapshot, documents: [snapshot.documents[1]] };
if (method === "Accessibility.getFullAXTree") return { nodes: childAx };
throw new Error(`unexpected child CDP method ${method}`);
});
@@ -3376,6 +3422,7 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 1, 3],
nodeName: [index("html"), index("body"), index("button"), index("div"), index("#text")],
@@ -3700,6 +3747,7 @@ describe("handleSnapshot", () => {
strings: S,
documents: [
{
frameId: "root",
scrollOffsetX: 0,
scrollOffsetY: 0,
nodes: {
@@ -3726,6 +3774,7 @@ describe("handleSnapshot", () => {
},
// documents[1]: cross-origin login form
{
frameId: "child",
scrollOffsetX: 0,
scrollOffsetY: 0,
nodes: {
@@ -3904,6 +3953,7 @@ describe("handleSnapshot", () => {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 0, 3, 4, 5],
nodeName: [
apps/extension/src/tools/observation.ts
+12 -221
View File
@@ -1,3 +1,4 @@
import type { CapturedSceneInput } from "./vom/facts";
// Observation handlers — `tool.snapshot`, `tool.get_html`, `tool.screenshot`,
// and semantic `tool.observe` (design §7). Each handler resolves the target
// tab (defaulting to the Agent Window's active tab when omitted) and
@@ -35,7 +36,6 @@ import { attachDialogs, markDialogCursor } from "./dialogs";
import { rpcError } from "./errors";
import { resolveNodeGeometry } from "./frame-geometry";
import { screenshotPageRect } from "./geometry/coordinate-types";
import { cssViewport, GeometryContext } from "./geometry/frame-context";
import {
type ChromeTabsApi,
enforceToolTargetScope,
@@ -49,15 +49,9 @@ import {
type ToolEffect,
} from "./shared";
import { resolveSnapshotRef } from "./snapshot-ref";
import {
type CapturedNode,
type CapturedSurfaceProbe,
type CapturedViewModel,
captureViewModel,
collectOverlayExcludedBackendIds,
probeHoverSurfaces,
} from "./vom/capture";
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
import { type CapturedNode, type CapturedSurfaceProbe, probeHoverSurfaces } from "./vom/capture";
import { captureObservationFacts, semanticCapture } from "./vom/capture-coordinator";
import type { FrameDocument as CapturedFrameDocument } from "./vom/frame-document";
import { withOverlayBypass } from "./vom/hover-perception";
import { probeTooltipNames } from "./vom/name-enrichment";
import {
@@ -531,7 +525,7 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A
function buildConditionalSurfaces(
nodes: VomNode[],
captured: CapturedViewModel,
captured: CapturedSceneInput,
probes: CapturedSurfaceProbe[],
): CondSurface[] {
if (probes.length === 0) return [];
@@ -661,161 +655,9 @@ export interface BuildVomSceneOptions {
export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>;
function legacyFrameDocuments(
axNodes: CdpAxNode[],
captured: CapturedViewModel,
pageUrl?: string,
rootTarget: VomFrameDocument["target"] = { tabId: 0 },
): VomFrameDocument[] {
const rootFrameId = captured.rootFrameId ?? "root";
const documents: VomFrameDocument[] = [
{
frameId: rootFrameId,
contextScopeId: rootFrameId,
target: rootTarget,
...(pageUrl ? { url: pageUrl } : {}),
axNodes: [],
domNodes: captured.nodes.map((node) => ({ ...node, frameId: node.frameId ?? rootFrameId })),
},
];
const pending = [...captured.iframeNodes.entries()];
let progress = true;
let nextSyntheticFrame = 1;
while (pending.length > 0 && progress) {
progress = false;
for (let index = pending.length - 1; index >= 0; index -= 1) {
const [ownerBackendNodeId, domNodes] = pending[index];
const parent = documents.find((document) =>
document.domNodes.some((node) => node.backendNodeId === ownerBackendNodeId),
);
if (!parent) continue;
const frameId =
domNodes.find((node) => node.frameId)?.frameId ?? `legacy-frame-${nextSyntheticFrame++}`;
documents.push({
frameId,
parentFrameId: parent.frameId,
ownerBackendNodeId,
contextScopeId: frameId,
target: parent.target,
axNodes: [],
domNodes: domNodes.map((node) => ({ ...node, frameId: node.frameId ?? frameId })),
});
pending.splice(index, 1);
progress = true;
}
}
const documentByFrameId = new Map(documents.map((document) => [document.frameId, document]));
const backendFrame = new Map<number, string>();
const childFrameByOwner = new Map<number, string>();
for (const document of documents) {
if (document.ownerBackendNodeId !== undefined) {
childFrameByOwner.set(document.ownerBackendNodeId, document.frameId);
}
for (const node of document.domNodes) backendFrame.set(node.backendNodeId, document.frameId);
}
const axById = new Map(axNodes.map((node) => [node.nodeId, node]));
const ownership = new Map<string, string>();
const resolving = new Set<string>();
const frameForAx = (node: CdpAxNode): string => {
const cached = ownership.get(node.nodeId);
if (cached) return cached;
let frameId: string | undefined;
if (node.frameId && documentByFrameId.has(node.frameId)) frameId = node.frameId;
if (!frameId && typeof node.backendDOMNodeId === "number") {
frameId = backendFrame.get(node.backendDOMNodeId);
}
if (!frameId && node.parentId && !resolving.has(node.nodeId)) {
const parent = axById.get(node.parentId);
if (parent) {
frameId =
typeof parent.backendDOMNodeId === "number"
? childFrameByOwner.get(parent.backendDOMNodeId)
: undefined;
if (!frameId) {
resolving.add(node.nodeId);
frameId = frameForAx(parent);
resolving.delete(node.nodeId);
}
}
}
frameId ??= rootFrameId;
ownership.set(node.nodeId, frameId);
return frameId;
};
for (const node of axNodes) frameForAx(node);
for (const node of axNodes) {
const frameId = ownership.get(node.nodeId) ?? rootFrameId;
const document = documentByFrameId.get(frameId) ?? documents[0];
document.axNodes.push({
...node,
frameId,
...(node.parentId && ownership.get(node.parentId) === frameId
? { parentId: node.parentId }
: { parentId: undefined }),
...(node.childIds
? { childIds: node.childIds.filter((childId) => ownership.get(childId) === frameId) }
: {}),
});
}
return documents;
}
function withLegacyBackendIds(scene: VomScene): VomScene {
const used = new Set<number>();
const idMap = new Map<number, number>();
let nextVirtualId = -1;
for (const node of scene.nodes) {
const preferred = node.backendNodeId;
const id = preferred !== undefined && !used.has(preferred) ? preferred : nextVirtualId--;
used.add(id);
idMap.set(node.id, id);
}
return {
...scene,
nodes: scene.nodes.map((node) => ({
...node,
id: idMap.get(node.id) as number,
parentId: node.parentId === null ? null : (idMap.get(node.parentId) ?? null),
...(node.domParentId !== undefined
? { domParentId: node.domParentId === null ? null : (idMap.get(node.domParentId) ?? null) }
: {}),
...(node.domAncestorIds
? { domAncestorIds: node.domAncestorIds.flatMap((id) => idMap.get(id) ?? []) }
: {}),
})),
...(scene.surfaces
? {
surfaces: scene.surfaces.map((surface) => ({
...surface,
triggerId: idMap.get(surface.triggerId) ?? surface.triggerId,
})),
}
: {}),
...(scene.activeScopeBlocks
? {
activeScopeBlocks: scene.activeScopeBlocks.map((block) => ({
...block,
triggerId: idMap.get(block.triggerId) ?? block.triggerId,
})),
}
: {}),
};
}
export function buildVomScene(
axNodes: CdpAxNode[],
captured: CapturedViewModel,
options: BuildVomSceneOptions = {},
): VomScene {
return withLegacyBackendIds(
buildFrameVomScene(legacyFrameDocuments(axNodes, captured, options.pageUrl), captured, options),
);
}
export function buildFrameVomScene(
documents: VomFrameDocument[],
captured: CapturedViewModel,
captured: CapturedSceneInput,
options: BuildVomSceneOptions = {},
): VomScene {
const scene = buildSemanticVomScene({
@@ -831,7 +673,7 @@ export function buildFrameVomScene(
function attachCapturedSceneAnnotations(
scene: VomScene,
documents: VomFrameDocument[],
captured: CapturedViewModel,
captured: CapturedSceneInput,
surfaceProbes: CapturedSurfaceProbe[],
): VomScene {
const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId);
@@ -981,45 +823,6 @@ export async function handleGetHtml(
}
}
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
}
async function fallbackCapturedViewModel(
cdp: CdpRunner,
tabId: number,
geometry: GeometryContext,
signal?: AbortSignal,
): Promise<CapturedViewModel> {
throwIfAborted(signal, "observation");
let viewport = { width: 0, height: 0 };
try {
const metrics = await geometry.layoutMetrics({ tabId });
throwIfAborted(signal, "observation");
const source = cssViewport(metrics);
viewport = { width: source.width, height: source.height };
} catch (error) {
if (isAbortError(error)) throw error;
}
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal);
throwIfAborted(signal, "observation");
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
}
async function captureForVom(
cdp: CdpRunner,
tabId: number,
options: CaptureVomObservationOptions,
geometry: GeometryContext,
): Promise<CapturedViewModel> {
try {
return await captureViewModel(cdp, tabId, { signal: options.signal, geometry });
} catch (error) {
if (isAbortError(error)) throw error;
return fallbackCapturedViewModel(cdp, tabId, geometry, options.signal);
}
}
export interface HoverProbeOutcome {
/** Whether any active hover was dispatched during this observation. */
performed: boolean;
@@ -1052,7 +855,7 @@ const NO_HOVER_PROBES: HoverProbeOutcome = {
async function runHoverProbes(
cdp: CdpRunner,
tabId: number,
captured: CapturedViewModel,
captured: CapturedSceneInput,
documents: VomFrameDocument[],
staticSemantics: ReturnType<typeof resolveSemanticGraph>,
options: CaptureVomObservationOptions,
@@ -1091,21 +894,9 @@ export async function captureVomObservation(
throwIfAborted(options.signal, "observation");
await cdp.ensureAttachedToUrl?.(tabId, url);
throwIfAborted(options.signal, "observation");
const geometry = new GeometryContext(cdp, tabId, undefined, options.signal);
const captured = await captureForVom(cdp, tabId, options, geometry);
const facts = await captureObservationFacts<CdpAxNode>(cdp, tabId, options.signal);
const { captured, documents: normalizedDocuments } = semanticCapture(facts);
throwIfAborted(options.signal, "observation");
const documents = await captureFrameData<CdpAxNode>(
cdp,
tabId,
captured,
options.signal,
geometry,
);
throwIfAborted(options.signal, "observation");
const normalizedDocuments =
documents.length === 1 && captured.iframeNodes.size > 0
? legacyFrameDocuments(documents[0].axNodes, captured, url, documents[0].target)
: documents;
const semanticGraph = buildSemanticGraph({
documents: normalizedDocuments,
viewport: captured.viewport,
@@ -1135,8 +926,8 @@ export async function captureVomObservation(
);
// Reserve space using the renderer's character-based token estimate. Like VOM
// headers, this integrity notice remains visible even under a tiny token budget.
const geometryNotice = captured.frameGeometryIssues?.length
? "@warning geometry incomplete: some page or frame content has no top-level coordinates.\n"
const geometryNotice = facts.issues.some((issue) => issue.stage === "geometry")
? "@warning iframe geometry incomplete: some frame content has no top-level coordinates.\n"
: "";
const rendered = renderVom(decoratedScene, {
maxDepth: options.maxDepth,
apps/extension/src/tools/vom/__tests__/capture-coordinator.test.ts
+619
View File
@@ -0,0 +1,619 @@
import { describe, expect, it, vi } from "vitest";
import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph";
import { cdpTargetKey } from "@/browser-driver/frame-graph";
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import type { CdpRunner } from "../../shared";
import { captureObservationFacts, semanticCapture } from "../capture-coordinator";
import { buildSemanticGraph } from "../semantic-graph/build";
import { REQUESTED_STYLES } from "../snapshot";
function fixture(
options: { frames?: CdpFrame[]; fail?: string; omitDocument?: string; overlay?: string } = {},
) {
const frames = options.frames ?? [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "same",
parentFrameId: "main",
ownerBackendNodeId: 3,
target: { tabId: 4 },
},
{
frameId: "nested",
parentFrameId: "same",
ownerBackendNodeId: 13,
target: { tabId: 4 },
},
{
frameId: "remote",
parentFrameId: "main",
ownerBackendNodeId: 4,
target: { tabId: 4, sessionId: "remote" },
},
];
const elements = new Map(
frames.map((frame, i) => [frame.frameId, frame.target.sessionId ? 1 : i * 10 + 1]),
);
const logs: Array<{ target: CdpTarget; method: string; params: Record<string, unknown> }> = [];
const send = vi.fn(
async <T>(target: CdpTarget, method: string, params: object = {}): Promise<T> => {
logs.push({ target, method, params: params as Record<string, unknown> });
const args = params as Record<string, string | number>;
if (options.fail === `${target.sessionId ?? "main"}:${method}`)
throw new Error("fixture failure");
let result: unknown = {};
if (method === "Page.getLayoutMetrics")
result = { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
if (method === "DOMSnapshot.captureSnapshot") {
expect((params as { computedStyles: unknown }).computedStyles).toEqual(REQUESTED_STYLES);
result = {
strings: [
"#document",
"html",
"button",
OVERLAY_HOST_MARKER_ATTR,
"",
"visible",
"1",
"static",
"auto",
],
documents: frames
.filter(
(frame) =>
cdpTargetKey(frame.target) === cdpTargetKey(target) &&
frame.frameId !== options.omitDocument,
)
.map((frame) => {
const element = elements.get(frame.frameId)!;
return {
frameId: frame.frameId,
nodes: {
backendNodeId: [element - 1, element + 10000, element, element + 1],
nodeName: [0, 1, 1, 2],
nodeType: [9, 10, 1, 1],
parentIndex: [-1, 0, 0, 2],
attributes: [[], [], [], frame.frameId === options.overlay ? [3, 4] : []],
},
layout: {
nodeIndex: [2, 3],
bounds: [
[0, 0, 1000, 800],
[10, 20, 100, 40],
],
styles: [
[7, 8, 8, 5, 6],
[7, 8, 8, 5, 6],
],
},
};
}),
};
}
if (method === "Accessibility.getFullAXTree") {
const frameId = String(args.frameId);
result = {
nodes: [
{
nodeId: `${frameId}-button`,
frameId,
backendDOMNodeId: elements.get(frameId)! + 1,
role: { value: "button" },
name: { value: frameId },
},
],
};
}
return result as T;
},
);
const cdp: CdpRunner = {
getFrameGraph: async () => ({ rootFrameId: frames[0].frameId, frames }),
send: (tabId, method, params) =>
(send as NonNullable<CdpRunner["sendToTarget"]>)({ tabId }, method, params),
sendToTarget: send as NonNullable<CdpRunner["sendToTarget"]>,
};
return { cdp, logs, elements };
}
describe("captureObservationFacts", () => {
it("collects each target once and scopes equal backend IDs", async () => {
const { cdp, logs } = fixture();
const facts = await captureObservationFacts(cdp, 4);
expect(logs.filter((call) => call.method === "DOMSnapshot.captureSnapshot")).toHaveLength(2);
expect(logs.filter((call) => call.method === "Accessibility.enable")).toHaveLength(2);
expect(logs.filter((call) => call.method === "Accessibility.getFullAXTree")).toHaveLength(4);
const main = facts.documents.find((doc) => doc.frame.frameId === "main")!;
const remote = facts.documents.find((doc) => doc.frame.frameId === "remote")!;
expect(main.index.nodes.get(2)).not.toBe(remote.index.nodes.get(2));
expect(main.index.children.get(1)).toEqual([2]);
expect(main.index.nodes.get(10001)?.nodeType).toBe(10);
expect(remote.axNodes[0].frameId).toBe("remote");
expect(main.index.nodes.get(2)).toBe(main.domNodes.find((node) => node.backendNodeId === 2));
expect(facts.finishedAt).toBeGreaterThanOrEqual(facts.startedAt);
});
it("does not retry failed or missing documents and retains valid AX-only semantics", async () => {
const { cdp, logs } = fixture({
fail: "remote:DOMSnapshot.captureSnapshot",
omitDocument: "same",
});
const facts = await captureObservationFacts(cdp, 4);
const remote = facts.documents.find((doc) => doc.frame.frameId === "remote")!;
expect(remote.domNodes).toHaveLength(0);
expect(remote.axNodes).toHaveLength(1);
expect(logs.filter((call) => call.method === "DOMSnapshot.captureSnapshot")).toHaveLength(2);
expect(facts.issues).toContainEqual(
expect.objectContaining({ frameId: "same", stage: "dom", reason: "capture-unavailable" }),
);
expect(facts.issues).toContainEqual(
expect.objectContaining({ target: { tabId: 4, sessionId: "remote" }, stage: "dom" }),
);
});
it("excludes overlay AX in its own document without excluding equal IDs elsewhere", async () => {
const facts = await captureObservationFacts(fixture({ overlay: "remote" }).cdp, 4);
const { documents } = semanticCapture(facts);
const graph = buildSemanticGraph({
documents,
viewport: facts.viewport,
rootFrameId: facts.rootFrameId,
});
expect(
[...graph.nodes.values()].find(
(node) => node.frameId === "remote" && node.backendNodeId === 2,
)?.excluded,
).toBe(true);
expect(
[...graph.nodes.values()].find((node) => node.frameId === "main" && node.backendNodeId === 2)
?.excluded,
).toBe(false);
});
it("bounds collection across many targets and stops scheduling after cancellation", async () => {
const frames = Array.from({ length: 12 }, (_, i) => ({
frameId: `f${i}`,
target: { tabId: 4, ...(i ? { sessionId: `s${i}` } : {}) },
}));
const { cdp } = fixture({ frames });
let active = 0,
peak = 0;
const original = cdp.sendToTarget!;
const send: NonNullable<CdpRunner["sendToTarget"]> = async (target, method, params) => {
active++;
peak = Math.max(peak, active);
try {
await new Promise((resolve) => setTimeout(resolve, 1));
return await original(target, method, params);
} finally {
active--;
}
};
cdp.sendToTarget = send;
cdp.send = (tabId, method, params) => send({ tabId }, method, params);
await captureObservationFacts(cdp, 4);
// This fixture has no frame edges, so no concurrent owner requests.
expect(peak).toBeLessThanOrEqual(4);
expect(peak).toBeGreaterThan(1);
const controller = new AbortController();
controller.abort();
await expect(captureObservationFacts(cdp, 4, controller.signal)).rejects.toMatchObject({
name: "AbortError",
});
expect(active).toBe(0);
});
});
function childSnapshot(frameId: string, backendNodeId: number) {
const strings = [frameId, "body", "button", "static", "auto", "pointer"];
return {
strings,
documents: [
{
frameId,
nodes: {
parentIndex: [-1, 0],
nodeName: [1, 2],
backendNodeId: [backendNodeId - 1, backendNodeId],
attributes: [[], []],
},
layout: {
nodeIndex: [0, 1],
styles: [
[3, 4, 4],
[3, 4, 5],
],
bounds: [
[0, 0, 300, 200],
[10, 20, 100, 40],
],
paintOrders: [0, 1],
},
},
],
};
}
describe("OOPIF capture", () => {
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
const sendToTarget = vi.fn(async (target, method) => {
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } };
}
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
return target.sessionId === "left-session"
? childSnapshot("left", 101)
: childSnapshot("right", 201);
}
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
throw new Error(`unexpected ${method}`);
});
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method, params) => {
if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("main", 1);
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
if (method === "DOM.getBoxModel") {
const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId;
const x = backendNodeId === 10 ? 50 : 500;
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
}
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "left",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "left-session" },
},
{
frameId: "right",
parentFrameId: "main",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "right-session" },
},
],
})),
};
const { documents: trees } = semanticCapture(await captureObservationFacts(cdp, 4));
expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]);
expect(
trees
.find((doc) => doc.frameId === "left")
?.domNodes?.find((node) => node.backendNodeId === 101)?.rect,
).toEqual({ x: 60, y: 120, w: 100, h: 40 });
expect(
trees
.find((doc) => doc.frameId === "right")
?.domNodes?.find((node) => node.backendNodeId === 201)?.rect,
).toEqual({ x: 510, y: 120, w: 100, h: 40 });
expect(
new Map(
trees
.filter((doc) => doc.ownerBackendNodeId !== undefined)
.map((doc) => [doc.frameId, doc.ownerBackendNodeId]),
),
).toEqual(
new Map([
["left", 10],
["right", 20],
]),
);
expect(
new Map(
trees.filter((doc) => doc.parentFrameId).map((doc) => [doc.frameId, doc.parentFrameId]),
),
).toEqual(
new Map([
["left", "main"],
["right", "main"],
]),
);
});
it("retains nested owner failures when merging a captured OOPIF", async () => {
const child = childSnapshot("child", 101);
const nested = childSnapshot("nested", 201);
const document = child.documents[0];
const snapshot = {
strings: child.strings,
documents: [
{
...document,
nodes: { ...document.nodes, contentDocumentIndex: { index: [1], value: [1] } },
},
nested.documents[0],
],
};
const reply = async (_target: unknown, method: string) => {
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200 } };
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "DOM.getBoxModel")
return { model: { content: [50, 100, 350, 100, 350, 300, 50, 300] } };
if (method === "DOM.resolveNode") throw new Error("nested owner replaced");
return {};
};
const cdp: CdpRunner = {
send: vi.fn(reply) as CdpRunner["send"],
sendToTarget: vi.fn(reply) as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "nested",
parentFrameId: "child",
ownerBackendNodeId: 101,
target: { tabId: 4, sessionId: "child-session" },
},
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
})),
};
const captured = await captureObservationFacts(cdp, 4);
expect(
captured.issues
.filter((issue) => issue.projectionIssue)
.map((issue) => issue.projectionIssue),
).toEqual([
{
status: "unavailable",
source: { target: { tabId: 4, sessionId: "child-session" }, frameId: "nested" },
ownerBackendNodeId: 101,
},
]);
expect(
captured.documents.find((doc) => doc.frame.frameId === "nested")?.domNodes[1],
).toMatchObject({
rect: null,
localRect: { x: 10, y: 20, w: 100, h: 40 },
});
});
it("keeps OOPIF semantics when viewport projection is unavailable", async () => {
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method) => {
if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("main", 1);
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable");
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Page.getLayoutMetrics") throw new Error("viewport unavailable");
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("child", 101);
if (method === "Accessibility.getFullAXTree") {
return {
nodes: [
{
nodeId: "button",
backendDOMNodeId: 101,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Continue" },
},
],
};
}
throw new Error(`unexpected ${method}`);
}) as unknown as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
})),
};
const { documents } = semanticCapture(await captureObservationFacts(cdp, 4));
const childDocument = documents.find((document) => document.frameId === "child");
expect(childDocument?.axNodes).toEqual([
expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }),
]);
expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual(
expect.objectContaining({ rect: null, localRect: { x: 10, y: 20, w: 100, h: 40 } }),
);
});
});
// Six sibling frames and one nested frame exercise scheduling through the real capture entry.
function siblingCaptureFixture(
beforeReply: (method: string, params: Record<string, unknown>) => Promise<void> = async () => {},
) {
const document = (id: number, owners: number[], childIndexes: number[]) => ({
frameId: `frame-${id}`,
nodes: {
parentIndex: [-1, ...owners.map(() => 0)],
nodeName: [0, ...owners.map(() => 1)],
backendNodeId: [1000 + id, ...owners],
attributes: [[], ...owners.map(() => [])],
contentDocumentIndex: { index: owners.map((_, i) => i + 1), value: childIndexes },
},
layout: {
nodeIndex: [0, ...owners.map((_, i) => i + 1)],
bounds: [[0, 0, 200, 100], ...owners.map(() => [0, 0, 200, 100])],
},
});
const snapshot = {
strings: ["body", "iframe"],
documents: [
document(0, [100, 101, 102, 103, 104, 105], [1, 2, 3, 4, 5, 6]),
document(1, [200], [7]),
...Array.from({ length: 6 }, (_, i) => document(i + 2, [], [])),
],
};
let active = 0;
let peak = 0;
const send = vi.fn(async (_tabId: number, method: string, params: object = {}) => {
const args = params as Record<string, unknown>;
active++;
peak = Math.max(peak, active);
try {
await beforeReply(method, args);
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
if (method === "DOM.getBoxModel")
return { model: { content: [0, 0, 200, 0, 200, 100, 0, 100] } };
if (method === "DOM.resolveNode") return { object: { objectId: String(args.backendNodeId) } };
if (method === "Runtime.callFunctionOn")
return { result: { value: { width: 200, height: 100 } } };
return {};
} finally {
active--;
}
});
return {
cdp: { send: send as CdpRunner["send"] },
send,
peak: () => peak,
active: () => active,
measured: () =>
send.mock.calls
.filter(([, method]) => method === "DOM.getBoxModel")
.map(([, , params]) => (params as { backendNodeId: number }).backendNodeId),
};
}
describe("sibling frame measurement scheduling", () => {
it("bounds concurrent reads, fills free slots and preserves breadth-first output despite reordered replies", async () => {
const pending = new Map<number, () => void>();
const fixture = siblingCaptureFixture(async (method, params) => {
if (method === "DOM.getBoxModel" && Number(params.backendNodeId) < 200)
await new Promise<void>((resolve) => pending.set(Number(params.backendNodeId), resolve));
});
const capture = captureObservationFacts(fixture.cdp, 4);
await vi.waitFor(() => expect(pending.size).toBe(4));
expect(fixture.measured()).toEqual([100, 101, 102, 103]);
pending.get(103)!();
await vi.waitFor(() => expect(pending.has(104)).toBe(true));
pending.get(104)!();
await vi.waitFor(() => expect(pending.has(105)).toBe(true));
expect(fixture.measured()).not.toContain(200);
for (const id of [105, 102, 101, 100]) pending.get(id)!();
const captured = await capture;
expect(
captured.issues
.filter((issue) => issue.stage === "geometry")
.map((issue) => issue.projectionIssue),
).toEqual([]);
expect(fixture.peak()).toBe(4);
expect(fixture.active()).toBe(0);
expect(fixture.measured()).toEqual([100, 101, 102, 103, 104, 105, 200]);
expect(captured.documents.map((doc) => doc.frame.frameId)).toEqual(
Array.from({ length: 8 }, (_, i) => `frame-${i}`),
);
expect(
captured.documents.find((doc) => doc.frame.frameId === "frame-7")?.domNodes[0].rect,
).toEqual({ x: 0, y: 0, w: 200, h: 100 });
for (const method of [
"DOM.getBoxModel",
"DOM.resolveNode",
"Runtime.callFunctionOn",
"Runtime.releaseObject",
])
expect(fixture.send.mock.calls.filter(([, name]) => name === method)).toHaveLength(7);
});
it("does not measure descendants of a failed owner and retains sibling geometry and local nodes", async () => {
const fixture = siblingCaptureFixture(async (method, params) => {
if (method === "DOM.resolveNode" && params.backendNodeId === 100)
throw new Error("owner replaced");
});
const captured = await captureObservationFacts(fixture.cdp, 4);
expect(
captured.issues
.filter((issue) => issue.stage === "geometry")
.map((issue) => issue.projectionIssue),
).toEqual([
{
status: "unavailable",
source: { target: { tabId: 4 }, frameId: "frame-1" },
ownerBackendNodeId: 100,
},
{
status: "blocked",
source: { target: { tabId: 4 }, frameId: "frame-7" },
cause: captured.issues.find((issue) => issue.frameId === "frame-1")?.projectionIssue,
},
]);
expect(fixture.measured()).not.toContain(200);
expect(
captured.documents.find((doc) => doc.frame.frameId === "frame-7")?.domNodes[0],
).toMatchObject({
tag: "body",
rect: null,
localRect: { x: 0, y: 0, w: 200, h: 100 },
});
expect(
captured.documents.find((doc) => doc.frame.frameId === "frame-2")?.domNodes[0].rect,
).toEqual({ x: 0, y: 0, w: 200, h: 100 });
expect(captured.documents.map((doc) => doc.frame.frameId)).toEqual(
Array.from({ length: 8 }, (_, i) => `frame-${i}`),
);
});
it("stops scheduling on cancellation and waits for resolved objects to be released", async () => {
const controller = new AbortController();
const resolutions = new Map<string, () => void>();
const releases = new Map<string, () => void>();
const fixture = siblingCaptureFixture(async (method, params) => {
if (method === "DOM.resolveNode")
await new Promise<void>((resolve) =>
resolutions.set(String(params.backendNodeId), resolve),
);
if (method === "Runtime.releaseObject")
await new Promise<void>((resolve) => releases.set(String(params.objectId), resolve));
});
let settled = false;
const capture = captureObservationFacts(fixture.cdp, 4, controller.signal);
const rejected = expect(capture).rejects.toMatchObject({ name: "AbortError" });
void capture.then(
() => {
settled = true;
},
() => {
settled = true;
},
);
await vi.waitFor(() => expect(resolutions.size).toBe(4));
controller.abort();
for (const resolve of resolutions.values()) resolve();
await vi.waitFor(() => expect(releases.size).toBe(4));
expect(settled).toBe(false);
expect(fixture.measured()).toEqual([100, 101, 102, 103]);
for (const release of releases.values()) release();
await rejected;
expect(fixture.active()).toBe(0);
expect(fixture.send.mock.calls.some(([, method]) => method === "Runtime.callFunctionOn")).toBe(
false,
);
});
});
apps/extension/src/tools/vom/__tests__/capture.test.ts
+71 -149
View File
@@ -1,7 +1,7 @@
import { describe, expect, it, vi } from "vitest";
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
import type { CdpRunner } from "../../shared";
import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
import { collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
import { captureObservationFacts, semanticCapture } from "../capture-coordinator";
// Minimal but format-accurate captureSnapshot reply: a body with one
// fixed full-screen overlay div carrying a password input.
@@ -27,6 +27,7 @@ function fakeSnapshotReply() {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 2],
nodeType: [1, 1, 1, 1],
@@ -74,6 +75,7 @@ function hoverTriggerSnapshotReply() {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1],
nodeType: [1, 1, 1],
@@ -118,6 +120,7 @@ function twoHoverTriggerSnapshotReply() {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 1],
nodeType: [1, 1, 1, 1],
@@ -171,6 +174,7 @@ function nestedHoverTriggerSnapshotReply() {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 2, 3],
nodeType: [1, 1, 1, 1, 1],
@@ -211,11 +215,7 @@ function makeCdp(snapshot: unknown) {
if (method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
}
if (method === "Runtime.evaluate") {
return {
@@ -248,13 +248,11 @@ function makeCdp(snapshot: unknown) {
};
}
describe("captureViewModel", () => {
describe("snapshot capture and hover", () => {
it("parses nodes, attrs, rects, paint order and styles", async () => {
const { nodes, viewport, iframeNodes } = await captureViewModel(
makeCdp(fakeSnapshotReply()),
4,
);
expect(iframeNodes.size).toBe(0);
const { nodes, viewport } = semanticCapture(
await captureObservationFacts(makeCdp(fakeSnapshotReply()), 4),
).captured;
expect(viewport).toEqual({ width: 1000, height: 800 });
const div = nodes.find((n) => n.backendNodeId === 12);
expect(div).toMatchObject({ tag: "div", position: "fixed", parentBackendNodeId: 11 });
@@ -266,35 +264,11 @@ describe("captureViewModel", () => {
it("never hovers the page while capturing", async () => {
const cdp = makeCdp(fakeSnapshotReply());
await captureViewModel(cdp, 4);
semanticCapture(await captureObservationFacts(cdp, 4)).captured;
expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything());
});
it("batch-enriches form controls without per-node object resolution", async () => {
const cdp = makeCdp(fakeSnapshotReply());
const result = await captureViewModel(cdp, 4);
const input = result.nodes.find((node) => node.backendNodeId === 13);
expect(input).toMatchObject({
formState: "filled",
formPlaceholder: "secret",
attrs: { type: "password" },
});
expect(input?.formValue).toBeUndefined();
expect(input?.formDefaultValue).toBeUndefined();
expect(cdp.send).toHaveBeenCalledWith(
4,
"Runtime.evaluate",
expect.objectContaining({
serializationOptions: expect.objectContaining({ serialization: "deep" }),
}),
);
expect(cdp.send).toHaveBeenCalledWith(4, "Runtime.releaseObjectGroup", expect.anything());
expect(cdp.send).not.toHaveBeenCalledWith(4, "DOM.resolveNode", expect.anything());
expect(cdp.send).not.toHaveBeenCalledWith(4, "Runtime.callFunctionOn", expect.anything());
});
it("runs hover surface probes when explicitly enabled", async () => {
let hoverStateCalls = 0;
const cdp = {
@@ -303,8 +277,6 @@ describe("captureViewModel", () => {
if (method === "DOMSnapshot.captureSnapshot") return hoverTriggerSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -336,7 +308,7 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const captured = await captureViewModel(cdp, 4);
const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
expect(surfaceProbes).toEqual([
@@ -368,8 +340,6 @@ describe("captureViewModel", () => {
if (method === "DOMSnapshot.captureSnapshot") return twoHoverTriggerSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -398,7 +368,7 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const captured = await captureViewModel(cdp, 4);
const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
expect(surfaceProbes).toEqual([
@@ -418,8 +388,6 @@ describe("captureViewModel", () => {
if (method === "DOMSnapshot.captureSnapshot") return nestedHoverTriggerSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -445,7 +413,7 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const captured = await captureViewModel(cdp, 4);
const captured = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
await probeHoverSurfaces(cdp, 4, captured.nodes);
expect(hoverMoves).toBe(1);
@@ -477,6 +445,7 @@ describe("captureViewModel", () => {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 0, 3, 4],
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
@@ -507,7 +476,7 @@ describe("captureViewModel", () => {
],
};
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
const ids = nodes.map((n) => n.backendNodeId);
// Real page nodes survive.
expect(ids).toContain(12);
@@ -538,6 +507,7 @@ describe("captureViewModel", () => {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 0, 3, 4],
nodeName: [i("html"), i("body"), i("div"), i(OVERLAY_HOST_NAME), i("div"), i("button")],
@@ -568,7 +538,9 @@ describe("captureViewModel", () => {
],
};
const { excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
const { excludedBackendNodeIds } = semanticCapture(
await captureObservationFacts(makeCdp(snapshot), 4),
).captured;
expect(excludedBackendNodeIds).toEqual(new Set([13, 14, 15]));
});
@@ -589,6 +561,7 @@ describe("captureViewModel", () => {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i(OVERLAY_HOST_NAME)],
@@ -611,7 +584,9 @@ describe("captureViewModel", () => {
],
};
const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
const { nodes, excludedBackendNodeIds } = semanticCapture(
await captureObservationFacts(makeCdp(snapshot), 4),
).captured;
expect(nodes.map((n) => n.backendNodeId)).not.toContain(13);
expect(excludedBackendNodeIds).toEqual(new Set([13]));
});
@@ -634,6 +609,7 @@ describe("captureViewModel", () => {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("div")],
@@ -656,7 +632,9 @@ describe("captureViewModel", () => {
],
};
const { nodes, excludedBackendNodeIds } = await captureViewModel(makeCdp(snapshot), 4);
const { nodes, excludedBackendNodeIds } = semanticCapture(
await captureObservationFacts(makeCdp(snapshot), 4),
).captured;
expect(nodes.map((n) => n.backendNodeId)).not.toContain(13);
expect(excludedBackendNodeIds).toEqual(new Set([13]));
});
@@ -679,6 +657,7 @@ describe("captureViewModel", () => {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("div")],
@@ -701,7 +680,7 @@ describe("captureViewModel", () => {
},
],
};
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
expect(nodes.find((n) => n.backendNodeId === 12)?.cursor).toBe("pointer");
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
});
@@ -713,6 +692,7 @@ describe("captureViewModel", () => {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 1, 1],
nodeName: [i("html"), i("body"), i("button"), i("button"), i("button")],
@@ -739,7 +719,7 @@ describe("captureViewModel", () => {
],
};
const { nodes } = await captureViewModel(makeCdp(snapshot), 4);
const { nodes } = semanticCapture(await captureObservationFacts(makeCdp(snapshot), 4)).captured;
expect(nodes.find((node) => node.backendNodeId === 12)?.rendered).toBe(true);
expect(nodes.find((node) => node.backendNodeId === 13)?.rendered).toBe(false);
@@ -753,66 +733,26 @@ describe("captureViewModel", () => {
if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply();
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 200 },
};
}
throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { nodes } = await captureViewModel(cdp, 4);
const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
const div = nodes.find((n) => n.backendNodeId === 12);
expect(div?.localRect?.y).toBe(-200);
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
});
it("normalizes snapshot-owned scroll before using a stale CSS metrics fallback", async () => {
const snapshot = {
strings: ["html", "button"],
documents: [
{
frameId: "root",
scrollOffsetX: 40,
scrollOffsetY: 200,
nodes: {
parentIndex: [-1, 0],
nodeName: [0, 1],
backendNodeId: [10, 11],
attributes: [[], []],
},
layout: { nodeIndex: [1], bounds: [[200, 800, 240, 80]] },
},
],
};
const cdp: CdpRunner = {
send: vi.fn(async (_tab: number, method: string) => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 2000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 999, pageY: 999 },
};
return {};
}) as CdpRunner["send"],
};
const captured = await captureViewModel(cdp, 4);
expect(captured.nodes.find((node) => node.backendNodeId === 11)).toMatchObject({
localRect: { x: 80, y: 300, w: 120, h: 40 },
rect: { x: 80, y: 300, w: 120, h: 40 },
rendered: true,
});
expect(captured.frameGeometryIssues).toEqual([]);
});
it("normalizes raw snapshot bounds before viewport clipping", async () => {
it("keeps CSS snapshot bounds independent of legacy metrics ratio", async () => {
const S = ["html", "body", "div", "position", "fixed", "static", "pointer-events", "auto"];
const i = (s: string) => S.indexOf(s);
const snapshot = {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("div")],
@@ -840,8 +780,6 @@ describe("captureViewModel", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 2000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
layoutViewport: { clientWidth: 2000, clientHeight: 1600 },
};
@@ -849,7 +787,7 @@ describe("captureViewModel", () => {
throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { nodes } = await captureViewModel(cdp, 4);
const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
expect(nodes.find((n) => n.backendNodeId === 12)?.rect).toEqual({
x: 0,
y: 0,
@@ -859,8 +797,8 @@ describe("captureViewModel", () => {
expect(nodes.find((n) => n.backendNodeId === 12)?.localRect).toEqual({
x: 0,
y: 0,
w: 1000,
h: 800,
w: 2000,
h: 1600,
});
});
@@ -885,6 +823,7 @@ describe("captureViewModel", () => {
strings: S,
documents: [
{
frameId: "root",
scrollOffsetX: 0,
scrollOffsetY: 0,
nodes: {
@@ -917,15 +856,13 @@ describe("captureViewModel", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { nodes } = await captureViewModel(cdp, 4);
const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
const btn = nodes.find((n) => n.backendNodeId === 12);
expect(btn?.textContent).toBe("登录");
const link = nodes.find((n) => n.backendNodeId === 14);
@@ -935,7 +872,7 @@ describe("captureViewModel", () => {
expect(body?.textContent).toBeUndefined();
});
it("parses iframe sub-documents and returns iframeNodes keyed by iframe backendNodeId", async () => {
it("retains explicit iframe document ownership and projected nodes", async () => {
const S = [
"html",
"body",
@@ -1009,8 +946,6 @@ describe("captureViewModel", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -1023,30 +958,21 @@ describe("captureViewModel", () => {
throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const {
nodes,
iframeNodes,
frameNodes,
frameOwnerBackendNodeIds,
frameParentIds,
rootFrameId,
} = await captureViewModel(cdp, 4);
// Main frame has 4 nodes; iframe is included
expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
// iframeNodes keyed by the <iframe> element's backendNodeId=13
expect(iframeNodes.size).toBe(1);
expect(iframeNodes.has(13)).toBe(true);
const subNodes = iframeNodes.get(13)!;
const facts = await captureObservationFacts(cdp, 4);
const root = facts.documents.find((doc) => doc.frame.frameId === "main-frame")!;
const child = facts.documents.find((doc) => doc.frame.frameId === "child-frame")!;
expect(facts.documents).toHaveLength(2);
expect(root.domNodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
const subNodes = child.domNodes;
const input = subNodes.find((n) => n.backendNodeId === 101);
expect(input?.tag).toBe("input");
expect(input?.attrs.type).toBe("text");
expect(input?.ownerFrameBackendNodeId).toBe(13);
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
expect(rootFrameId).toBe("main-frame");
expect(frameNodes?.get("child-frame")).toBe(subNodes);
expect(frameOwnerBackendNodeIds?.get("child-frame")).toBe(13);
expect(frameParentIds?.get("child-frame")).toBe("main-frame");
expect(facts.rootFrameId).toBe("main-frame");
expect(child.frame.ownerBackendNodeId).toBe(13);
expect(child.frame.parentFrameId).toBe("main-frame");
});
it("projects iframe documents independently of owner snapshot geometry", async () => {
@@ -1067,6 +993,7 @@ describe("captureViewModel", () => {
strings: S,
documents: [
{
frameId: "root",
scrollOffsetX: 0,
scrollOffsetY: 0,
nodes: {
@@ -1084,6 +1011,7 @@ describe("captureViewModel", () => {
},
},
{
frameId: "child",
scrollOffsetX: 0,
scrollOffsetY: 0,
nodes: {
@@ -1107,8 +1035,6 @@ describe("captureViewModel", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -1122,8 +1048,10 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { iframeNodes } = await captureViewModel(cdp, 4);
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
const { documents } = await captureObservationFacts(cdp, 4);
const input = documents
.find((doc) => doc.frame.frameId === "child")
?.domNodes?.find((n) => n.backendNodeId === 101);
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
expect(input?.rect).toEqual({ x: 120, y: 320, w: 120, h: 60 });
@@ -1147,6 +1075,7 @@ describe("captureViewModel", () => {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("iframe")],
@@ -1168,6 +1097,7 @@ describe("captureViewModel", () => {
},
},
{
frameId: "child",
nodes: {
parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("iframe")],
@@ -1175,8 +1105,6 @@ describe("captureViewModel", () => {
attributes: [[], [], []],
contentDocumentIndex: { index: [2], value: [2] },
},
scrollOffsetX: 0,
scrollOffsetY: 0,
layout: {
nodeIndex: [1, 2],
styles: [
@@ -1191,14 +1119,13 @@ describe("captureViewModel", () => {
},
},
{
frameId: "nested",
nodes: {
parentIndex: [-1, 0],
nodeName: [i("body"), i("input")],
backendNodeId: [30, 31],
attributes: [[], [i("type"), i("text")]],
},
scrollOffsetX: 0,
scrollOffsetY: 0,
layout: {
nodeIndex: [1],
styles: [[i("static"), i("auto")]],
@@ -1219,8 +1146,6 @@ describe("captureViewModel", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
}
@@ -1250,9 +1175,13 @@ describe("captureViewModel", () => {
) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { iframeNodes } = await captureViewModel(cdp, 4);
const nestedIframe = iframeNodes.get(13)?.find((n) => n.backendNodeId === 23);
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31);
const { documents } = await captureObservationFacts(cdp, 4);
const nestedIframe = documents
.find((doc) => doc.frame.frameId === "child")
?.domNodes?.find((n) => n.backendNodeId === 23);
const input = documents
.find((doc) => doc.frame.frameId === "nested")
?.domNodes?.find((n) => n.backendNodeId === 31);
expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 });
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
@@ -1268,6 +1197,7 @@ describe("captureViewModel", () => {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1],
nodeName: [i("html"), i("body"), i("div")],
@@ -1295,8 +1225,6 @@ describe("captureViewModel", () => {
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 2000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 100 },
layoutViewport: { clientWidth: 2000, clientHeight: 1600 },
};
@@ -1304,8 +1232,8 @@ describe("captureViewModel", () => {
throw new Error(method);
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { nodes } = await captureViewModel(cdp, 4);
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 / 2 - 100);
const { nodes } = semanticCapture(await captureObservationFacts(cdp, 4)).captured;
expect(nodes.find((n) => n.backendNodeId === 12)?.rect?.y).toBe(400 - 100);
});
it("collectOverlayExcludedBackendIds walks the pierced overlay host subtree", async () => {
@@ -1336,8 +1264,6 @@ function siblingCaptureFixture(
beforeReply: (method: string, params: Record<string, unknown>) => Promise<void> = async () => {},
) {
const document = (id: number, owners: number[], childIndexes: number[]) => ({
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId: `frame-${id}`,
nodes: {
parentIndex: [-1, ...owners.map(() => 0)],
@@ -1369,11 +1295,7 @@ function siblingCaptureFixture(
await beforeReply(method, args);
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
if (method === "DOM.getBoxModel")
return { model: { content: [0, 0, 200, 0, 200, 100, 0, 100] } };
if (method === "DOM.resolveNode") return { object: { objectId: String(args.backendNodeId) } };
apps/extension/src/tools/vom/__tests__/facts.test.ts
+156
View File
@@ -0,0 +1,156 @@
import { describe, expect, it } from "vitest";
import { OVERLAY_HOST_MARKER_ATTR } from "@/lib/overlay-bridge";
import { buildDocumentIndex, type DecodedNode } from "../facts";
import { decodeDocument, REQUESTED_STYLES } from "../snapshot";
function node(id: number, parent: number | null): DecodedNode {
return {
backendNodeId: id,
parentBackendNodeId: parent,
tag: "div",
attrs: {},
paintOrder: 0,
position: "static",
pointerEvents: "auto",
};
}
describe("document facts", () => {
it.each([
1_000, 10_000, 100_000,
])("indexes deep and wide inputs of %i without copying ancestors", async (count) => {
for (const wide of [false, true]) {
let reads = 0;
const input = Array.from({ length: count }, (_, i) => {
const item = node(i, i ? (wide ? 0 : i - 1) : null);
Object.defineProperty(item, "parentBackendNodeId", {
get: () => {
reads++;
return i ? (wide ? 0 : i - 1) : null;
},
});
return item;
}).reverse(); // parent need not precede child
const index = await buildDocumentIndex(input);
expect(index.nodes.size).toBe(count);
expect(index.ancestry.get(count - 1)).toEqual({ complete: true, overlay: false });
expect(index.children.get(0)?.length).toBe(wide ? count - 1 : 1);
expect(reads).toBeLessThanOrEqual(count * 8);
expect(index.nodes.get(count - 1)).toBe(input[0]);
}
});
it("marks cycles and orphans incomplete and propagates overlay through shadow roots", async () => {
const host = { ...node(1, null), attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } };
const input = [
node(3, 2),
{ ...node(2, 1), tag: "#document-fragment" },
host,
node(4, 99),
node(5, 6),
node(6, 5),
node(7, 6),
];
const index = await buildDocumentIndex(input);
expect([...index.excludedBackendNodeIds].sort()).toEqual([1, 2, 3]);
for (const id of [4, 5, 6, 7]) expect(index.ancestry.get(id)?.complete).toBe(false);
});
it("does not propagate a descendant overlay backwards into a cycle", async () => {
const overlay = { ...node(3, 1), attrs: { [OVERLAY_HOST_MARKER_ATTR]: "" } };
const index = await buildDocumentIndex([overlay, node(1, 2), node(2, 1)]);
expect([...index.excludedBackendNodeIds]).toEqual([3]);
});
it("does not treat a missing snapshot parent as a complete root", async () => {
const { nodes } = await decodeDocument(
{ nodes: { backendNodeId: [1], nodeName: [0], parentIndex: [99] } },
["div"],
);
const index = await buildDocumentIndex(nodes);
expect(index.ancestry.get(1)?.complete).toBe(false);
});
it("retains raw box units and independent style/interaction evidence", async () => {
const strings = [
"html",
"canvas",
"aria-hidden",
"true",
"inert",
"",
"hidden",
"visible",
"0",
"auto",
"scroll",
"matrix(2,0,0,2,0,0)",
];
const styles = REQUESTED_STYLES.map((style) =>
style === "visibility"
? 7
: style === "opacity"
? 8
: style === "overflow-x"
? 10
: style === "transform"
? 11
: 9,
);
const { nodes } = await decodeDocument(
{
nodes: {
backendNodeId: [1, 2],
parentIndex: [-1, 0],
nodeName: [0, 1],
attributes: [[2, 3, 4, 5], []],
},
layout: {
nodeIndex: [0, 1],
bounds: [
[0, 0, 200, 100],
[10, 20, 120, 40],
],
clientRects: [
[5, 5, 90, 40],
[0, 0, 120, 40],
],
styles: [styles, styles],
},
},
strings,
);
expect(nodes[0].attrs).toEqual({ "aria-hidden": "true", inert: "" });
expect(nodes[0].layout).toMatchObject({
boundsSpace: "snapshot-document-css",
clientSpace: "unscaled-client-offset-css",
clientRect: [5, 5, 90, 40],
styles: {
visibility: "visible",
opacity: "0",
"overflow-x": "scroll",
transform: "matrix(2,0,0,2,0,0)",
},
});
expect(nodes[0]).not.toHaveProperty("rect");
expect(nodes[0]).not.toHaveProperty("excluded");
});
it("yields during large inputs so cancellation interrupts indexing", async () => {
const controller = new AbortController();
const input = Array.from({ length: 100_000 }, (_, i) => node(i, i ? i - 1 : null));
let reads = 0;
for (const item of input)
Object.defineProperty(item, "attrs", {
get: () => {
reads++;
if (reads === 100) controller.abort();
return {};
},
});
await expect(buildDocumentIndex(input, controller.signal)).rejects.toMatchObject({
name: "AbortError",
});
expect(reads).toBeLessThanOrEqual(356);
});
});
apps/extension/src/tools/vom/__tests__/form-state.test.ts
+38 -21
View File
@@ -1,6 +1,6 @@
import { describe, expect, it, vi } from "vitest";
import { type CdpRunner, cdpRunnerForTarget } from "../../shared";
import { captureViewModel } from "../capture";
import { captureObservationFacts } from "../capture-coordinator";
interface Control {
id: number;
@@ -106,17 +106,19 @@ describe("form state identity", () => {
[11, state("alice@example.com")],
]),
);
const captured = await captureViewModel(cdp, 7);
expect(captured.nodes.find((n) => n.backendNodeId === 10)).toMatchObject({
const captured = await captureObservationFacts(cdp, 7);
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 10)).toMatchObject({
formValue: "",
formState: "empty",
formPlaceholder: "snapshot placeholder",
});
expect(captured.nodes.find((n) => n.backendNodeId === 11)).toMatchObject({
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 11)).toMatchObject({
formValue: "alice@example.com",
});
expect(captured.nodes.find((n) => n.backendNodeId === 12)).toMatchObject({ formValue: "上海" });
expect(captured.nodes.some((n) => n.backendNodeId === 99)).toBe(false);
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 12)).toMatchObject({
formValue: "上海",
});
expect(captured.documents[0].domNodes.some((n) => n.backendNodeId === 99)).toBe(false);
});
it("does not shift state between frames when one frame is absent from the runtime batch", async () => {
@@ -133,8 +135,10 @@ describe("form state identity", () => {
[21, state("second frame")],
]),
);
const captured = await captureViewModel(cdp, 7);
const controls = [...captured.iframeNodes.values()].flat();
const captured = await captureObservationFacts(cdp, 7);
const controls = captured.documents
.filter((doc) => doc.frame.parentFrameId)
.flatMap((doc) => doc.domNodes);
expect(controls.find((n) => n.backendNodeId === 11)?.formValue).toBe("inaccessible frame");
expect(controls.find((n) => n.backendNodeId === 21)?.formValue).toBe("second frame");
expect(controls.find((n) => n.backendNodeId === 31)?.formValue).toBe("third frame");
@@ -151,8 +155,13 @@ describe("form state identity", () => {
});
const cdp = { send, sendToTarget } as unknown as CdpRunner;
for (const sessionId of ["left-frame", "right-frame"]) {
const captured = await captureViewModel(cdpRunnerForTarget(cdp, { tabId: 7, sessionId }), 7);
expect(captured.nodes.find((n) => n.backendNodeId === 11)?.formValue).toBe(sessionId);
const captured = await captureObservationFacts(
cdpRunnerForTarget(cdp, { tabId: 7, sessionId }),
7,
);
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 11)?.formValue).toBe(
sessionId,
);
expect(sendToTarget).toHaveBeenCalledWith(
{ tabId: 7, sessionId },
"Runtime.releaseObjectGroup",
@@ -178,8 +187,8 @@ describe("form state identity", () => {
result.result.deepSerializedValue.value[0].value[1].value = "not json";
return result;
});
const captured = await captureViewModel(cdp, 7);
expect(captured.nodes.find((n) => n.backendNodeId === 11)).toMatchObject({
const captured = await captureObservationFacts(cdp, 7);
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 11)).toMatchObject({
formValue: "snapshot value",
formPlaceholder: "snapshot placeholder",
});
@@ -192,7 +201,7 @@ describe("form state identity", () => {
controller.abort();
return reply([[11, state("new")]]);
});
await expect(captureViewModel(cdp, 7, { signal: controller.signal })).rejects.toMatchObject({
await expect(captureObservationFacts(cdp, 7, controller.signal)).rejects.toMatchObject({
name: "AbortError",
});
expect(send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", expect.anything());
@@ -216,9 +225,13 @@ describe("form state identity", () => {
return result;
},
);
const captured = await captureViewModel(cdp, 7);
expect(captured.nodes.find((n) => n.backendNodeId === 11)?.formValue).toBe("snapshot value");
expect(captured.nodes.find((n) => n.backendNodeId === 12)?.formValue).toBe("live value");
const captured = await captureObservationFacts(cdp, 7);
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 11)?.formValue).toBe(
"snapshot value",
);
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 12)?.formValue).toBe(
"live value",
);
});
it.each([
@@ -228,8 +241,8 @@ describe("form state identity", () => {
const { cdp } = fakeCdp([[{ id: 11, value: "secret", type }]], () =>
reply([[11, { ...state("secret"), sensitive: true }]]),
);
const captured = await captureViewModel(cdp, 7);
const control = captured.nodes.find((n) => n.backendNodeId === 11);
const captured = await captureObservationFacts(cdp, 7);
const control = captured.documents[0].domNodes.find((n) => n.backendNodeId === 11);
expect(control?.formState).toBe("filled");
expect(control?.formValue).toBeUndefined();
expect(control?.formDefaultValue).toBeUndefined();
@@ -257,9 +270,13 @@ describe("form state identity", () => {
expect(entries).toHaveLength(250);
return reply(entries.map(([element, json]) => [element.backendNodeId, JSON.parse(json)]));
});
const captured = await captureViewModel(cdp, 7);
expect(captured.nodes.find((n) => n.backendNodeId === 250)?.formValue).toBe("live 250");
expect(captured.nodes.find((n) => n.backendNodeId === 251)?.formValue).toBe("old");
const captured = await captureObservationFacts(cdp, 7);
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 250)?.formValue).toBe(
"live 250",
);
expect(captured.documents[0].domNodes.find((n) => n.backendNodeId === 251)?.formValue).toBe(
"old",
);
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1);
expect(
send.mock.calls.filter(([, method]) => method === "Runtime.releaseObjectGroup"),
apps/extension/src/tools/vom/__tests__/frame-capture.test.ts
-283
View File
@@ -1,283 +0,0 @@
import { describe, expect, it, vi } from "vitest";
import type { CdpRunner } from "../../shared";
import type { CapturedNode, CapturedViewModel } from "../capture";
import { captureFrameData } from "../frame-capture";
function ownerNode(backendNodeId: number, x: number): CapturedNode {
return {
backendNodeId,
parentBackendNodeId: 1,
frameId: "main",
ownerFrameBackendNodeId: null,
tag: "iframe",
attrs: {},
rect: { x, y: 100, w: 300, h: 200 },
localRect: { x, y: 100, w: 300, h: 200 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
};
}
function childSnapshot(frameId: string, backendNodeId: number) {
const strings = [frameId, "body", "button", "static", "auto", "pointer"];
return {
strings,
documents: [
{
scrollOffsetX: 0,
scrollOffsetY: 0,
frameId,
nodes: {
parentIndex: [-1, 0],
nodeName: [1, 2],
backendNodeId: [backendNodeId - 1, backendNodeId],
attributes: [[], []],
},
layout: {
nodeIndex: [0, 1],
styles: [
[3, 4, 4],
[3, 4, 5],
],
bounds: [
[0, 0, 300, 200],
[10, 20, 100, 40],
],
paintOrders: [0, 1],
},
},
],
};
}
describe("captureFrameData", () => {
it("retains nested owner failures when merging a captured OOPIF", async () => {
const owner = ownerNode(10, 50);
const captured: CapturedViewModel = {
nodes: [owner],
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
frameNodes: new Map([["main", [owner]]]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const child = childSnapshot("child", 101);
const nested = childSnapshot("nested", 201);
const document = child.documents[0];
const snapshot = {
strings: child.strings,
documents: [
{
...document,
nodes: { ...document.nodes, contentDocumentIndex: { index: [1], value: [1] } },
},
nested.documents[0],
],
};
const reply = async (_target: unknown, method: string) => {
if (method === "Page.getLayoutMetrics")
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 300, clientHeight: 200 },
};
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "DOM.getBoxModel")
return { model: { content: [50, 100, 350, 100, 350, 300, 50, 300] } };
if (method === "DOM.resolveNode") throw new Error("nested owner replaced");
return {};
};
const cdp: CdpRunner = {
send: vi.fn(reply) as CdpRunner["send"],
sendToTarget: vi.fn(reply) as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
})),
};
await captureFrameData(cdp, 4, captured);
expect(captured.frameGeometryIssues).toEqual([
{
status: "unavailable",
source: { target: { tabId: 4, sessionId: "child-session" }, frameId: "nested" },
ownerBackendNodeId: 101,
},
]);
expect(captured.frameNodes?.get("nested")?.[1]).toMatchObject({
rect: null,
localRect: { x: 10, y: 20, w: 100, h: 40 },
});
});
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
const leftOwner = ownerNode(10, 50);
const rightOwner = ownerNode(20, 500);
const captured: CapturedViewModel = {
nodes: [leftOwner, rightOwner],
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
frameNodes: new Map([["main", [leftOwner, rightOwner]]]),
frameOwnerBackendNodeIds: new Map(),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const sendToTarget = vi.fn(async (target, method) => {
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 },
};
}
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") {
return target.sessionId === "left-session"
? childSnapshot("left", 101)
: childSnapshot("right", 201);
}
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
throw new Error(`unexpected ${method}`);
});
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method, params) => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
if (method === "DOM.getBoxModel") {
const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId;
const x = backendNodeId === 10 ? 50 : 500;
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
}
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "left",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "left-session" },
},
{
frameId: "right",
parentFrameId: "main",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "right-session" },
},
],
})),
};
const trees = await captureFrameData(cdp, 4, captured);
expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]);
expect(
captured.frameNodes?.get("left")?.find((node) => node.backendNodeId === 101)?.rect,
).toEqual({ x: 60, y: 120, w: 100, h: 40 });
expect(
captured.frameNodes?.get("right")?.find((node) => node.backendNodeId === 201)?.rect,
).toEqual({ x: 510, y: 120, w: 100, h: 40 });
expect(captured.frameOwnerBackendNodeIds).toEqual(
new Map([
["left", 10],
["right", 20],
]),
);
expect(captured.frameParentIds).toEqual(
new Map([
["left", "main"],
["right", "main"],
]),
);
});
it("keeps OOPIF semantics when viewport projection is unavailable", async () => {
const owner = { ...ownerNode(10, 50), rect: null, localRect: null };
const captured: CapturedViewModel = {
nodes: [owner],
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
frameNodes: new Map([["main", [owner]]]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
const cdp: CdpRunner = {
send: vi.fn(async (_tabId, method) => {
if (method === "Accessibility.enable") return {};
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable");
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
cssVisualViewport: { clientWidth: 1000 },
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 },
};
}
throw new Error(`unexpected root ${method}`);
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Page.getLayoutMetrics") throw new Error("viewport unavailable");
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("child", 101);
if (method === "Accessibility.getFullAXTree") {
return {
nodes: [
{
nodeId: "button",
backendDOMNodeId: 101,
role: { type: "role", value: "button" },
name: { type: "computedString", value: "Continue" },
},
],
};
}
throw new Error(`unexpected ${method}`);
}) as unknown as NonNullable<CdpRunner["sendToTarget"]>,
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child-session" },
},
],
})),
};
const documents = await captureFrameData(cdp, 4, captured);
const childDocument = documents.find((document) => document.frameId === "child");
expect(childDocument?.axNodes).toEqual([
expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }),
]);
expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual(
expect.objectContaining({ rect: null, localRect: null, rendered: true }),
);
expect(captured.frameGeometryIssues).toContainEqual({
status: "unavailable",
source: { target: { tabId: 4, sessionId: "child-session" }, frameId: "child" },
reason: "snapshot-coordinates-unavailable",
});
});
});
apps/extension/src/tools/vom/__tests__/frame-document.test.ts
+48 -15
View File
@@ -1,7 +1,12 @@
import { describe, expect, it } from "vitest";
import { describe, expect, it, vi } from "vitest";
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
import type { CapturedNode, CapturedViewModel } from "../capture";
import { buildFrameDocuments, type FrameAxBatch, type FrameOwnedAxNode } from "../frame-document";
import type { CapturedNode } from "../capture";
import {
buildFrameDocuments,
type FrameAxBatch,
type FrameDomInput,
type FrameOwnedAxNode,
} from "../frame-document";
function domNode(frameId: string, backendNodeId: number): CapturedNode {
return {
@@ -17,19 +22,16 @@ function domNode(frameId: string, backendNodeId: number): CapturedNode {
};
}
function captured(frameNodes: Map<string, CapturedNode[]>): CapturedViewModel {
function captured(frameNodes: Map<string, CapturedNode[]>): FrameDomInput {
return {
nodes: frameNodes.get("main") ?? [],
viewport: { width: 800, height: 600 },
iframeNodes: new Map(),
frameNodes,
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
}
describe("buildFrameDocuments", () => {
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", () => {
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", async () => {
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
@@ -54,7 +56,7 @@ describe("buildFrameDocuments", () => {
},
];
const documents = buildFrameDocuments(
const documents = await buildFrameDocuments(
graph,
batches,
captured(
@@ -72,7 +74,7 @@ describe("buildFrameDocuments", () => {
expect(childNodes.find((node) => node.nodeId === "virtual")?.frameId).toBe("child");
});
it("deduplicates overlapping same-target AX results using the strongest ownership", () => {
it("deduplicates overlapping same-target AX results using the strongest ownership", async () => {
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
@@ -92,7 +94,7 @@ describe("buildFrameDocuments", () => {
},
];
const documents = buildFrameDocuments(
const documents = await buildFrameDocuments(
graph,
batches,
captured(
@@ -107,7 +109,7 @@ describe("buildFrameDocuments", () => {
expect(documents.find((document) => document.frameId === "child")?.axNodes).toHaveLength(1);
});
it("uses target-scoped backend ownership when OOPIF backend ids collide", () => {
it("uses target-scoped backend ownership when OOPIF backend ids collide", async () => {
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
@@ -129,7 +131,7 @@ describe("buildFrameDocuments", () => {
nodes: [{ nodeId: `${frame.frameId}-node`, backendDOMNodeId: 7 }],
}));
const documents = buildFrameDocuments(
const documents = await buildFrameDocuments(
graph,
batches,
captured(
@@ -149,7 +151,7 @@ describe("buildFrameDocuments", () => {
);
});
it("uses captured frame parent identity instead of guessing from backend ids", () => {
it("uses captured frame parent identity instead of guessing from backend ids", async () => {
const frameNodes = new Map([
["main", [domNode("main", 1)]],
["left", [domNode("left", 7)]],
@@ -164,11 +166,42 @@ describe("buildFrameDocuments", () => {
nodes: [],
}));
const documents = buildFrameDocuments(null, frames, capture);
const documents = await buildFrameDocuments(null, frames, capture);
expect(documents.find((document) => document.frameId === "child")).toMatchObject({
parentFrameId: "left",
ownerBackendNodeId: 7,
});
});
it("rejects explicit AX frame ownership in a different target", async () => {
const frames = [
{ frameId: "main", target: { tabId: 4 } },
{ frameId: "remote", target: { tabId: 4, sessionId: "remote" } },
];
const unresolved = vi.fn();
const documents = await buildFrameDocuments(
{ rootFrameId: "main", frames },
[{ frame: frames[0], nodes: [{ nodeId: "wrong", frameId: "remote" }] }],
captured(new Map()),
undefined,
unresolved,
);
expect(documents.every((doc) => doc.axNodes.length === 0)).toBe(true);
expect(unresolved).toHaveBeenCalledOnce();
});
it("resolves a reverse-ordered deep AX parent chain without recursion", async () => {
const frame = { frameId: "main", target: { tabId: 4 } };
const nodes: FrameOwnedAxNode[] = Array.from({ length: 10000 }, (_, i) => ({
nodeId: String(i),
...(i ? { parentId: String(i - 1) } : { frameId: "main" }),
})).reverse();
const documents = await buildFrameDocuments(
{ rootFrameId: "main", frames: [frame] },
[{ frame, nodes }],
captured(new Map()),
);
expect(documents[0].axNodes).toHaveLength(10000);
expect(documents[0].axNodes[0].frameId).toBe("main");
});
});
apps/extension/src/tools/vom/capture-coordinator.ts
+268
View File
@@ -0,0 +1,268 @@
import {
type CdpFrame,
type CdpFrameGraph,
type CdpTarget,
cdpTargetKey,
} from "@/browser-driver/frame-graph";
import { cssViewport, GeometryContext } from "../geometry/frame-context";
import { type CdpRunner, cdpRunnerForTarget } from "../shared";
import { collectOverlayExcludedBackendIds } from "./capture";
import type { CapturedSceneInput } from "./facts";
import {
buildDocumentIndex,
type CaptureIssue,
isCaptureAbort,
type ObservationFacts,
throwCaptureAborted,
} from "./facts";
import { enrichFormControlStates } from "./form-capture";
import {
buildFrameDocuments,
type FrameAxBatch,
type FrameDocument,
type FrameOwnedAxNode,
} from "./frame-document";
import { type NormalizedDocument, normalizeSnapshot } from "./normalize";
import { REQUESTED_STYLES, type SnapshotReply, snapshotFrameId } from "./snapshot";
interface TargetBatch<T extends FrameOwnedAxNode> {
target: CdpTarget;
frames: CdpFrame[];
documents: NormalizedDocument[];
ax: FrameAxBatch<T>[];
fallbackExcluded: Set<number>;
}
/** A fixed worker pool scoped to one capture; each worker issues at most one
* collection request at a time. Geometry has its own measurement-phase bound. */
async function collectTargets<T>(
items: readonly T[],
collect: (item: T) => Promise<void>,
signal?: AbortSignal,
): Promise<void> {
let cursor = 0;
await Promise.all(
Array.from({ length: Math.min(4, items.length) }, async () => {
while (cursor < items.length) {
throwCaptureAborted(signal);
const item = items[cursor++];
await collect(item);
}
}),
);
}
/** When topology discovery is unavailable, snapshot documents still carry
* explicit target/frame provenance. Only explicit contentDocumentIndex edges
* establish ancestry; a foreign frame is never renamed to fit a request. */
function snapshotFrames(snapshot: SnapshotReply, target: CdpTarget): CdpFrame[] {
const raw = snapshot.documents ?? [];
const strings = snapshot.strings ?? [];
const frames = raw.map((doc) => {
const frameId = snapshotFrameId(doc, strings);
return frameId ? ({ frameId, target } as CdpFrame) : undefined;
});
for (let i = 0; i < raw.length; i++) {
const edges = raw[i].nodes?.contentDocumentIndex;
const parent = frames[i];
if (!parent || !edges) continue;
for (let e = 0; e < edges.index.length; e++) {
const child = frames[edges.value[e]];
const owner = raw[i].nodes?.backendNodeId?.[edges.index[e]];
if (child && owner !== undefined && child !== parent) {
child.parentFrameId = parent.frameId;
child.ownerBackendNodeId = owner;
}
}
}
return frames.filter((frame): frame is CdpFrame => !!frame);
}
export async function captureObservationFacts<T extends FrameOwnedAxNode>(
cdp: CdpRunner,
tabId: number,
signal?: AbortSignal,
): Promise<ObservationFacts<T>> {
const startedAt = Date.now();
throwCaptureAborted(signal);
let graph: CdpFrameGraph | undefined;
try {
graph = await cdp.getFrameGraph?.(tabId);
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
const geometry = new GeometryContext(cdp, tabId, graph, signal);
const issues: CaptureIssue[] = [];
const groups = new Map<string, TargetBatch<T>>();
for (const frame of graph?.frames ?? []) {
const key = cdpTargetKey(frame.target);
let group = groups.get(key);
if (!group) {
group = {
target: frame.target,
frames: [],
documents: [],
ax: [],
fallbackExcluded: new Set(),
};
groups.set(key, group);
}
group.frames.push(frame);
}
if (!groups.size)
groups.set(cdpTargetKey({ tabId }), {
target: { tabId },
frames: [],
documents: [],
ax: [],
fallbackExcluded: new Set(),
});
const batches = [...groups.values()];
const unavailable = (batch: TargetBatch<T>, stage: CaptureIssue["stage"], frameId?: string) =>
issues.push({ target: batch.target, frameId, stage, reason: "capture-unavailable" });
let firstFailure: unknown;
await collectTargets(
batches,
async (batch) => {
const scoped = cdpRunnerForTarget(cdp, batch.target);
try {
throwCaptureAborted(signal);
await scoped.send(tabId, "DOMSnapshot.enable", {});
throwCaptureAborted(signal);
const snapshot = await scoped.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
computedStyles: REQUESTED_STYLES,
includePaintOrder: true,
includeDOMRects: true,
});
throwCaptureAborted(signal);
if (!graph) batch.frames = snapshotFrames(snapshot, batch.target);
batch.documents = await normalizeSnapshot(
snapshot,
batch.target,
batch.frames,
geometry,
issues,
signal,
);
if (
!(await enrichFormControlStates(
scoped,
tabId,
batch.documents.map((doc) => doc.domNodes),
signal,
))
)
unavailable(batch, "forms");
const present = new Set(batch.documents.map((doc) => doc.frame.frameId));
for (const frame of batch.frames)
if (!present.has(frame.frameId)) unavailable(batch, "dom", frame.frameId);
} catch (error) {
if (isCaptureAbort(error)) throw error;
firstFailure ??= error;
unavailable(batch, "dom");
batch.fallbackExcluded = await collectOverlayExcludedBackendIds(scoped, tabId, signal);
}
if (!batch.frames.length && !graph)
batch.frames = [{ frameId: "root", target: batch.target }];
try {
throwCaptureAborted(signal);
await scoped.send(tabId, "Accessibility.enable", {});
} catch (error) {
if (isCaptureAbort(error)) throw error;
firstFailure ??= error;
unavailable(batch, "ax");
return;
}
for (const frame of batch.frames) {
try {
throwCaptureAborted(signal);
const result = await scoped.send<{ nodes?: T[] }>(
tabId,
"Accessibility.getFullAXTree",
frame.frameId === "root" ? {} : { frameId: frame.frameId },
);
batch.ax.push({ frame, nodes: result.nodes ?? [] });
} catch (error) {
if (isCaptureAbort(error)) throw error;
firstFailure ??= error;
unavailable(batch, "ax", frame.frameId);
}
}
},
signal,
);
const frames = batches.flatMap((batch) => batch.frames);
const rootFrameId = graph?.rootFrameId ?? frames[0]?.frameId ?? "root";
const decoded = new Map(
batches.flatMap((batch) => batch.documents).map((doc) => [doc.frame.frameId, doc]),
);
const ax = batches.flatMap((batch) => batch.ax);
const documents = await buildFrameDocuments(
{ rootFrameId, frames },
ax,
{
nodes: [],
rootFrameId,
frameNodes: new Map([...decoded].map(([id, doc]) => [id, doc.domNodes])),
},
signal,
(frame) =>
issues.push({
target: frame.target,
frameId: frame.frameId,
stage: "ownership",
reason: "frame-ownership-unresolved",
}),
);
let viewport = { width: 0, height: 0 };
try {
const measured = cssViewport(await geometry.layoutMetrics({ tabId }));
viewport = { width: measured.width, height: measured.height };
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
const facts: ObservationFacts<T>["documents"][number][] = [];
for (const document of documents) {
const doc = decoded.get(document.frameId);
const index = doc?.index ?? (await buildDocumentIndex([], signal));
const fallback = groups.get(cdpTargetKey(document.target))?.fallbackExcluded;
const { domNodes, axNodes, contextScopeId: _scope, ...frame } = document;
facts.push({
frame,
index: fallback?.size ? { ...index, excludedBackendNodeIds: fallback } : index,
domNodes,
axNodes,
});
}
throwCaptureAborted(signal);
if (firstFailure && facts.every((doc) => !doc.domNodes.length && !doc.axNodes.length))
throw firstFailure;
return { rootFrameId, viewport, documents: facts, issues, startedAt, finishedAt: Date.now() };
}
/** Existing semantic consumers get a narrow view; no raw snapshot
* data can leak through recording's allowlisted output DTO. */
export function semanticCapture<T extends FrameOwnedAxNode>(
facts: ObservationFacts<T>,
): { captured: CapturedSceneInput; documents: FrameDocument<T>[] } {
const documents = facts.documents.map((doc) => ({
...doc.frame,
contextScopeId: doc.frame.frameId,
domNodes: doc.domNodes.filter(
(node) => !doc.index.excludedBackendNodeIds.has(node.backendNodeId),
),
axNodes: doc.axNodes,
excludedBackendNodeIds: doc.index.excludedBackendNodeIds,
}));
const root = documents.find((doc) => doc.frameId === facts.rootFrameId);
return {
documents,
captured: {
nodes: root?.domNodes ?? [],
viewport: facts.viewport,
rootFrameId: facts.rootFrameId,
excludedBackendNodeIds: root?.excludedBackendNodeIds ?? new Set(),
},
};
}
apps/extension/src/tools/vom/capture.ts
+31 -408
View File
@@ -1,339 +1,21 @@
// CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics
// → CapturedNode[] + Viewport. This is the ONLY VOM module that touches
// raw CDP. The captureSnapshot reply is columnar (parallel arrays + a
// shared string table); requested computed styles are decoded through
// STYLE_COL so capture and visibility policy share one explicit contract.
import type { Rect, Viewport } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph";
// Browser-side hover probes and overlay exclusion fallback. Static collection
// lives in capture-coordinator; snapshot decoding has no browser dependency.
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
import {
type FrameProjectionIssue,
type FrameProjectionState,
projectSnapshotRect,
type SnapshotCoordinates,
snapshotCoordinates,
snapshotViewportRect,
} from "../geometry/coordinate-types";
import {
GeometryContext,
type LayoutMetrics,
snapshotLayoutScale,
} from "../geometry/frame-context";
import { OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
import type { CdpRunner } from "../shared";
import type { CapturedNode, CapturedSurfaceProbe } from "./facts";
export type {
CapturedNode,
CapturedSurfaceProbe,
} from "./facts";
import { isCaptureAbort as isAbortError, throwCaptureAborted as throwIfAborted } from "./facts";
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
const STYLE_COL = Object.fromEntries(
REQUESTED_STYLES.map((name, index) => [name, index]),
) as Record<(typeof REQUESTED_STYLES)[number], number>;
export interface CapturedNode {
backendNodeId: number;
parentBackendNodeId: number | null;
frameId?: string;
/** Owning iframe backend node id; `null` for the top-level document. */
ownerFrameBackendNodeId?: number | null;
tag: string;
attrs: Record<string, string>;
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
rect: Rect | null;
/** Frame-local viewport-relative CSS px before top-level projection. */
localRect?: Rect | null;
paintOrder: number;
position: string;
pointerEvents: string;
/**
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
* is actually an interactive control — used by the adapter to surface
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
* `textContent`: the live parser always sets it, hand-built fixtures may not.
*/
cursor?: string;
/**
* Whether the live DOM snapshot provides a painted, non-hidden box for this
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
* nodes remain authoritative even when they are outside the viewport.
*/
rendered?: boolean;
textContent?: string;
formState?: "empty" | "filled" | "default";
formValue?: string;
formDefaultValue?: string;
formPlaceholder?: string;
}
export type CapturedIframeNodes = Map<number, CapturedNode[]>;
export interface CapturedSurfaceProbe {
triggerBackendNodeId: number;
triggerPoint?: { x: number; y: number };
triggerAction: "hover" | "focus" | string;
subItems: string[];
confidence?: "high" | "medium" | "low";
}
export interface CapturedViewModel {
nodes: CapturedNode[];
viewport: Viewport;
iframeNodes: CapturedIframeNodes;
frameNodes?: Map<string, CapturedNode[]>;
frameOwnerBackendNodeIds?: Map<string, number>;
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
frameParentIds?: Map<string, string>;
rootFrameId?: string;
/** Failed boundaries and descendants blocked by them; clipping is not a failure. */
frameGeometryIssues?: FrameProjectionIssue[];
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
excludedBackendNodeIds: Set<number>;
}
export interface CaptureViewModelOptions {
signal?: AbortSignal;
geometry?: GeometryContext;
target?: CdpTarget;
}
function captureAbortError(): Error {
const error = new Error("observation aborted");
error.name = "AbortError";
return error;
}
function isAbortError(error: unknown): boolean {
return (
typeof error === "object" &&
error !== null &&
(error as { name?: string }).name === "AbortError"
);
}
function throwIfAborted(signal: AbortSignal | undefined): void {
if (signal?.aborted) throw captureAbortError();
}
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
interface SparseArray {
index: number[];
value: number[];
}
interface RareBooleanData {
index: number[];
}
interface SnapshotDocument {
frameId?: string | number;
scrollOffsetX?: number;
scrollOffsetY?: number;
nodes?: {
parentIndex?: number[];
nodeName?: number[];
backendNodeId?: number[];
attributes?: number[][];
/**
* Per-node text value (index into strings), set for `#text` / CDATA nodes.
* Element nodes carry -1. Same length as `backendNodeId`.
*/
nodeValue?: number[];
/** Maps node array index → index into `documents[]` for frame content. */
contentDocumentIndex?: SparseArray;
inputValue?: SparseArray;
textValue?: SparseArray;
inputChecked?: RareBooleanData;
optionSelected?: RareBooleanData;
};
layout?: {
nodeIndex?: number[];
styles?: number[][];
bounds?: number[][];
paintOrders?: number[];
};
}
function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
if (typeof document.frameId === "string") return document.frameId || undefined;
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
return undefined;
}
interface SnapshotReply {
strings?: string[];
documents?: SnapshotDocument[];
}
function isFormControlTag(tag: string): boolean {
return tag === "input" || tag === "textarea" || tag === "select";
}
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
}
function snapshotFormState(
value: string | undefined,
defaultValue: string,
hasDefaultValue: boolean,
sensitive: boolean,
): CapturedNode["formState"] {
if (sensitive) {
if (value === undefined && !hasDefaultValue) return undefined;
return (value ?? defaultValue) === "" ? "empty" : "filled";
}
if (value === undefined) return undefined;
if (value === "") return "empty";
return value === defaultValue ? "default" : "filled";
}
const MAX_FORM_ENRICH_CONTROLS = 250;
interface CapturedFormState {
value?: string;
defaultValue?: string;
placeholder?: string;
state?: "empty" | "filled" | "default";
sensitive?: boolean;
}
interface DeepSerializedValue {
type: string;
value?: unknown;
}
function formStateBatchExpression(maxControls: number): string {
return `(() => {
const maxControls = ${JSON.stringify(maxControls)};
let remaining = maxControls;
const controlSelector = "input,textarea,select";
const controlState = (el) => {
const tag = el.tagName.toLowerCase();
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
const sensitive = type === "password" || type === "credit-card";
const rawValue = typeof el.value === "string" ? el.value : "";
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
return {
state,
sensitive,
placeholder,
...(sensitive ? {} : { value: rawValue, defaultValue }),
};
};
const controls = [];
const collect = (doc) => {
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
if (remaining <= 0) break;
// Deep serialization supplies the node's backend id. Keep the
// state as JSON so decoding needs no general-purpose V8 deserializer.
controls.push([el, JSON.stringify(controlState(el))]);
remaining -= 1;
}
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
if (remaining <= 0) break;
let childDoc = null;
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
if (childDoc) collect(childDoc);
}
};
collect(document);
return controls;
})()`;
}
function formStatesByBackendId(result: RuntimeEvaluateReply): Map<number, CapturedFormState> {
const states = new Map<number, CapturedFormState>();
const serialized = result.result?.deepSerializedValue;
if (serialized?.type !== "array" || !Array.isArray(serialized.value)) return states;
for (const entry of serialized.value as DeepSerializedValue[]) {
if (entry?.type !== "array" || !Array.isArray(entry.value)) continue;
const [element, json] = entry.value as DeepSerializedValue[];
if (element?.type !== "node" || json?.type !== "string" || typeof json.value !== "string") {
continue;
}
const backendNodeId = (element.value as { backendNodeId?: number } | undefined)?.backendNodeId;
if (typeof backendNodeId !== "number" || !Number.isSafeInteger(backendNodeId)) continue;
try {
const state = JSON.parse(json.value) as CapturedFormState | null;
if (
!state ||
!["empty", "filled", "default"].includes(state.state ?? "") ||
typeof state.sensitive !== "boolean" ||
typeof state.placeholder !== "string" ||
(!state.sensitive &&
(typeof state.value !== "string" || typeof state.defaultValue !== "string"))
) {
continue;
}
states.set(backendNodeId, state);
} catch {
// A malformed entry must not overwrite the snapshot's own state.
}
}
return states;
}
function applyFormStates(nodes: CapturedNode[], states: Map<number, CapturedFormState>): void {
for (const node of nodes) {
if (!isFormControlTag(node.tag)) continue;
const state = states.get(node.backendNodeId);
if (!state) continue;
node.formState = state.state;
node.formPlaceholder = state.placeholder ?? "";
if (state.sensitive || isSensitiveFormControl(node)) {
delete node.formValue;
delete node.formDefaultValue;
delete node.attrs.value;
} else {
node.formDefaultValue = state.defaultValue ?? "";
if (state.value !== undefined) node.formValue = state.value;
}
}
}
async function enrichFormControlStates(
cdp: CdpRunner,
tabId: number,
frameNodeGroups: CapturedNode[][],
signal?: AbortSignal,
): Promise<void> {
const hasControls = frameNodeGroups.some((nodes) =>
nodes.some((node) => isFormControlTag(node.tag)),
);
if (!hasControls) return;
const objectGroup = `bsk-vom-forms-${crypto.randomUUID()}`;
try {
throwIfAborted(signal);
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
objectGroup,
serializationOptions: {
serialization: "deep",
maxDepth: 3,
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
},
});
throwIfAborted(signal);
// Backend ids are scoped to this capture's CDP target. OOPIF targets
// use their own captureViewModel call and never share this lookup.
const states = formStatesByBackendId(result);
for (const nodes of frameNodeGroups) {
applyFormStates(nodes, states);
}
} catch (error) {
if (isAbortError(error)) throw error;
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
} finally {
await cdp.send(tabId, "Runtime.releaseObjectGroup", { objectGroup }).catch(() => undefined);
}
}
interface RuntimeEvaluateReply {
result?: {
value?: unknown;
deepSerializedValue?: DeepSerializedValue;
};
}
@@ -352,36 +34,6 @@ interface CdpDomNode {
shadowRoots?: CdpDomNode[];
}
interface ParseDocumentResult {
nodes: CapturedNode[];
excludedBackendNodeIds: Set<number>;
}
interface FrameContext {
frameId?: string;
ownerFrameBackendNodeId: number | null;
projection: FrameProjectionState;
target: CdpTarget;
coordinates: SnapshotCoordinates | null;
layoutUnitsPerCssPixel: number | null;
}
function str(strings: string[], idx: number | undefined): string {
if (idx === undefined || idx < 0) return "";
return strings[idx] ?? "";
}
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
const out = new Map<number, number>();
if (!sparse?.index || !sparse.value) return out;
for (let i = 0; i < sparse.index.length; i++) {
const nodeIndex = sparse.index[i];
const docIndex = sparse.value[i];
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
}
return out;
}
function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<number>): void {
if (!node) return;
if (typeof node.backendNodeId === "number") {
@@ -898,7 +550,7 @@ function parseDocumentNodes(
const input = snapshotViewportRect(
b,
{ target: context.target, frameId: context.frameId },
context.coordinates,
{ x: context.scrollX, y: context.scrollY },
);
if (input) {
const { x, y, width: w, height: h } = input.rect;
@@ -918,11 +570,7 @@ function parseDocumentNodes(
const visibility = str(strings, styleRow[visibilityCol]) || "visible";
const opacity = str(strings, styleRow[opacityCol]) || "1";
rendered =
!!b &&
b.length >= 4 &&
b.slice(0, 4).every(Number.isFinite) &&
b[2] > 0 &&
b[3] > 0 &&
localRect !== null &&
visibility !== "hidden" &&
visibility !== "collapse" &&
(Number.parseFloat(opacity) || 0) > 0;
@@ -1019,13 +667,10 @@ async function parseChildFrameDocuments(
const iframeBackendId = parentBackendIds[nodeArrayIdx];
if (visited.has(childDocIndex) || !childDoc || iframeBackendId === undefined) return [];
const source = { target: parentContext.target, frameId: snapshotFrameId(childDoc, strings) };
const coordinates = snapshotCoordinates(childDoc, parentContext.layoutUnitsPerCssPixel);
const parent = parentContext.projection;
const projection: FrameProjectionState =
parent.status === "available"
? coordinates
? { status: "unavailable", source, ownerBackendNodeId: iframeBackendId }
: { status: "unavailable", source, reason: "snapshot-coordinates-unavailable" }
? { status: "unavailable", source, ownerBackendNodeId: iframeBackendId }
: { status: "blocked", source, cause: parent.status === "blocked" ? parent.cause : parent };
return [
{
@@ -1033,7 +678,6 @@ async function parseChildFrameDocuments(
childDoc,
iframeBackendId,
source,
coordinates,
projection: projection as FrameProjectionState,
},
];
@@ -1055,7 +699,6 @@ async function parseChildFrameDocuments(
Array.from({ length: Math.min(4, children.length) }, async () => {
while (cursor < children.length && !signal?.aborted && !failure) {
const child = children[cursor++];
if (!child.coordinates) continue;
try {
// Owner quads are target-relative; do not apply the parent transform twice.
child.projection = await geometry.snapshotProjection(
@@ -1076,14 +719,7 @@ async function parseChildFrameDocuments(
throwIfAborted(signal);
// Completion order must not change document traversal or result insertion order.
for (const {
childDocIndex,
childDoc,
iframeBackendId,
source,
projection,
coordinates,
} of children) {
for (const { childDocIndex, childDoc, iframeBackendId, source, projection } of children) {
throwIfAborted(signal);
if (projection.status !== "available") frameGeometryIssues.push(projection);
const childContext: FrameContext = {
@@ -1091,8 +727,8 @@ async function parseChildFrameDocuments(
target: source.target,
ownerFrameBackendNodeId: iframeBackendId,
projection,
coordinates,
layoutUnitsPerCssPixel: parentContext.layoutUnitsPerCssPixel,
scrollX: childDoc.scrollOffsetX ?? 0,
scrollY: childDoc.scrollOffsetY ?? 0,
};
const nextVisited = new Set(visited);
nextVisited.add(childDocIndex);
@@ -1151,11 +787,10 @@ export async function captureViewModel(
console.debug("[bsk capture] layout metrics unavailable", error);
}
throwIfAborted(options.signal);
const viewport: Viewport = {
width: metrics.cssLayoutViewport?.clientWidth ?? 0,
height: metrics.cssLayoutViewport?.clientHeight ?? 0,
};
const layoutUnitsPerCssPixel = snapshotLayoutScale(metrics);
const measured = cssViewport(metrics);
const viewport: Viewport = { width: measured.width, height: measured.height };
const scrollX = measured.scrollX;
const scrollY = measured.scrollY;
await cdp.send(tabId, "DOMSnapshot.enable", {});
throwIfAborted(options.signal);
@@ -1178,28 +813,19 @@ export async function captureViewModel(
};
}
const source = { target, frameId: snapshotFrameId(doc0, strings) };
const coordinates = snapshotCoordinates(doc0, layoutUnitsPerCssPixel, {
x: metrics.cssLayoutViewport?.pageX,
y: metrics.cssLayoutViewport?.pageY,
});
const topContext: FrameContext = {
frameId: snapshotFrameId(doc0, strings),
ownerFrameBackendNodeId: null,
target,
projection:
coordinates &&
[viewport.width, viewport.height].every((size) => Number.isFinite(size) && size > 0)
? {
status: "available",
projection: {
source,
geometry: { sourceClips: [], edges: [], topViewport: viewport },
},
}
: { status: "unavailable", source, reason: "snapshot-coordinates-unavailable" },
coordinates,
layoutUnitsPerCssPixel,
projection: {
status: "available",
projection: {
source: { target, frameId: snapshotFrameId(doc0, strings) },
geometry: { sourceClips: [], edges: [], topViewport: viewport },
},
},
scrollX,
scrollY,
};
const mainParsed = parseDocumentNodes(doc0, strings, topContext);
const nodes = mainParsed.nodes;
@@ -1246,10 +872,7 @@ export async function captureViewModel(
viewport,
iframeNodes,
frameNodes,
frameGeometryIssues: [
...(topContext.projection.status === "available" ? [] : [topContext.projection]),
...frameParsed.frameGeometryIssues,
],
frameGeometryIssues: frameParsed.frameGeometryIssues,
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
frameParentIds: frameParsed.frameParentIds,
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
apps/extension/src/tools/vom/facts.ts
+210
View File
@@ -0,0 +1,210 @@
import type { Rect, Viewport } from "@browser-skill/vom";
import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph";
import { isOverlayHostNode } from "@/lib/overlay-bridge";
import type { FrameProjectionIssue } from "../geometry/coordinate-types";
import type { FrameOwnedAxNode } from "./frame-document";
export interface CapturedNode {
backendNodeId: number;
parentBackendNodeId: number | null;
frameId?: string;
/** Owning iframe backend node id; `null` for the top-level document. */
ownerFrameBackendNodeId?: number | null;
tag: string;
attrs: Record<string, string>;
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
rect: Rect | null;
/** Frame-local viewport-relative CSS px before top-level projection. */
localRect?: Rect | null;
paintOrder: number;
position: string;
pointerEvents: string;
/**
* computed `cursor`. `cursor: pointer` is the strongest CDP-free signal
* that a non-semantic element (a `<div>`/`<span>` with a click handler)
* is actually an interactive control — used by the adapter to surface
* custom buttons/checkboxes the AX tree drops as `generic`. Optional like
* `textContent`: the live parser always sets it, hand-built fixtures may not.
*/
cursor?: string;
/**
* Whether the live DOM snapshot provides a painted, non-hidden box for this
* node. Semantic resolution uses this only for DOM fallback nodes; AX-backed
* nodes remain authoritative even when they are outside the viewport.
*/
rendered?: boolean;
textContent?: string;
formState?: "empty" | "filled" | "default";
formValue?: string;
formDefaultValue?: string;
formPlaceholder?: string;
}
export interface CapturedSurfaceProbe {
triggerBackendNodeId: number;
triggerPoint?: { x: number; y: number };
triggerAction: "hover" | "focus" | string;
subItems: string[];
confidence?: "high" | "medium" | "low";
}
/** Narrow input shared by the existing semantic scene/hover consumers. */
export interface CapturedSceneInput {
nodes: CapturedNode[];
viewport: Viewport;
rootFrameId?: string;
excludedBackendNodeIds: ReadonlySet<number>;
}
/** Snapshot client/offset rectangles retain their protocol units. In particular,
* transformed bounds cannot be combined with unscaled client offsets. */
export interface SnapshotLayout {
readonly boundsSpace: "snapshot-document-css";
readonly clientSpace: "unscaled-client-offset-css";
bounds?: number[];
clientRect?: number[];
offsetRect?: number[];
scrollRect?: number[];
styles: Readonly<Record<string, string>>;
}
export interface DecodedNode extends Omit<CapturedNode, "rect" | "localRect" | "rendered"> {
nodeType?: number;
parentMissing?: boolean;
layout?: SnapshotLayout;
}
export interface AncestorState {
complete: boolean;
overlay: boolean;
}
export interface NodeFacts extends CapturedNode {
nodeType?: number;
layout?: SnapshotLayout;
}
export interface DocumentIndex<T extends DecodedNode = NodeFacts> {
readonly nodes: ReadonlyMap<number, T>;
readonly children: ReadonlyMap<number, readonly number[]>;
readonly ancestry: ReadonlyMap<number, AncestorState>;
readonly excludedBackendNodeIds: ReadonlySet<number>;
}
export interface DecodedDocument {
nodes: DecodedNode[];
}
export interface CaptureIssue {
/** Owner failure or inherited blockage; absent for other geometry failures. */
projectionIssue?: FrameProjectionIssue;
target: CdpTarget;
frameId?: string;
stage: "dom" | "ax" | "ownership" | "geometry" | "forms";
reason: "capture-unavailable" | "frame-ownership-unresolved" | "geometry-unavailable";
}
export interface DocumentFacts<T extends FrameOwnedAxNode> {
readonly frame: CdpFrame;
readonly index: DocumentIndex;
readonly domNodes: CapturedNode[];
readonly axNodes: T[];
}
export interface ObservationFacts<T extends FrameOwnedAxNode> {
readonly rootFrameId: string;
readonly viewport: Viewport;
readonly documents: readonly DocumentFacts<T>[];
readonly issues: readonly CaptureIssue[];
readonly startedAt: number;
readonly finishedAt: number;
}
export function throwCaptureAborted(signal?: AbortSignal): void {
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
}
export function isCaptureAbort(error: unknown): boolean {
return error instanceof Error && error.name === "AbortError";
}
/** Called only at bounded block boundaries, not once per node. */
export async function captureCheckpoint(signal?: AbortSignal): Promise<void> {
throwCaptureAborted(signal);
await new Promise<void>((resolve) => {
const channel = new MessageChannel();
channel.port1.onmessage = () => {
channel.port1.close();
channel.port2.close();
resolve();
};
channel.port2.postMessage(null);
});
throwCaptureAborted(signal);
}
/** All snapshot nodes participate, including document and shadow roots. This
* preserves ancestry evidence even when the semantic adapter omits those nodes. */
export async function buildDocumentIndex<T extends DecodedNode>(
input: readonly T[],
signal?: AbortSignal,
): Promise<DocumentIndex<T>> {
const nodes = new Map<number, T>();
const children = new Map<number, number[]>();
const ancestry = new Map<number, AncestorState>();
const excludedBackendNodeIds = new Set<number>();
for (let i = 0; i < input.length; i++) {
if (i % 256 === 0) await captureCheckpoint(signal);
const node = input[i];
nodes.set(node.backendNodeId, node);
if (node.parentBackendNodeId !== null) {
const siblings = children.get(node.parentBackendNodeId);
if (siblings) siblings.push(node.backendNodeId);
else children.set(node.parentBackendNodeId, [node.backendNodeId]);
}
}
let work = 0;
for (const node of input) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
if (ancestry.has(node.backendNodeId)) continue;
const path: DecodedNode[] = [];
const visiting = new Set<number>();
let current: DecodedNode | undefined = node;
let state: AncestorState = { complete: true, overlay: false };
while (current && !ancestry.has(current.backendNodeId)) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
if (visiting.has(current.backendNodeId)) {
// No member of a malformed cycle supplies complete ancestry evidence.
state = {
complete: false,
overlay: path
.slice(path.findIndex((item) => item.backendNodeId === current!.backendNodeId))
.some((item) => isOverlayHostNode(item.tag, Object.keys(item.attrs))),
};
break;
}
visiting.add(current.backendNodeId);
path.push(current);
if (current.parentMissing) state = { complete: false, overlay: false };
if (current.parentBackendNodeId === null) {
current = undefined;
break;
}
current = nodes.get(current.parentBackendNodeId);
if (!current) state = { complete: false, overlay: false };
}
if (current && ancestry.has(current.backendNodeId))
state = ancestry.get(current.backendNodeId)!;
for (let i = path.length - 1; i >= 0; i--) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
const item = path[i];
state = {
complete: state.complete,
overlay: state.overlay || isOverlayHostNode(item.tag, Object.keys(item.attrs)),
};
ancestry.set(item.backendNodeId, state);
if (state.overlay) excludedBackendNodeIds.add(item.backendNodeId);
}
}
return { nodes, children, ancestry, excludedBackendNodeIds };
}
apps/extension/src/tools/vom/form-capture.ts
+154
View File
@@ -0,0 +1,154 @@
import type { CdpRunner } from "../shared";
import type { CapturedNode } from "./facts";
import { isCaptureAbort as isAbortError, throwCaptureAborted as throwIfAborted } from "./facts";
interface RuntimeEvaluateReply {
result?: { deepSerializedValue?: DeepSerializedValue };
}
const MAX_FORM_ENRICH_CONTROLS = 250;
interface CapturedFormState {
value?: string;
defaultValue?: string;
placeholder?: string;
state?: "empty" | "filled" | "default";
sensitive?: boolean;
}
interface DeepSerializedValue {
type: string;
value?: unknown;
}
function formStateBatchExpression(maxControls: number): string {
return `(() => {
const maxControls = ${JSON.stringify(maxControls)};
let remaining = maxControls;
const controlSelector = "input,textarea,select";
const controlState = (el) => {
const tag = el.tagName.toLowerCase();
const type = tag === "input" ? String(el.type || "text").toLowerCase() : tag;
const sensitive = type === "password" || type === "credit-card";
const rawValue = typeof el.value === "string" ? el.value : "";
const defaultValue = typeof el.defaultValue === "string" ? el.defaultValue : "";
const placeholder = typeof el.placeholder === "string" ? el.placeholder : "";
const state = rawValue === "" ? "empty" : rawValue === defaultValue ? "default" : "filled";
return {
state,
sensitive,
placeholder,
...(sensitive ? {} : { value: rawValue, defaultValue }),
};
};
const controls = [];
const collect = (doc) => {
for (const el of Array.from(doc.querySelectorAll(controlSelector))) {
if (remaining <= 0) break;
// Deep serialization supplies the node's backend id. Keep the
// state as JSON so decoding needs no general-purpose V8 deserializer.
controls.push([el, JSON.stringify(controlState(el))]);
remaining -= 1;
}
for (const frame of Array.from(doc.querySelectorAll("iframe"))) {
if (remaining <= 0) break;
let childDoc = null;
try { childDoc = frame.contentDocument; } catch { childDoc = null; }
if (childDoc) collect(childDoc);
}
};
collect(document);
return controls;
})()`;
}
function formStatesByBackendId(result: RuntimeEvaluateReply): Map<number, CapturedFormState> {
const states = new Map<number, CapturedFormState>();
const serialized = result.result?.deepSerializedValue;
if (serialized?.type !== "array" || !Array.isArray(serialized.value)) return states;
for (const entry of serialized.value as DeepSerializedValue[]) {
if (entry?.type !== "array" || !Array.isArray(entry.value)) continue;
const [element, json] = entry.value as DeepSerializedValue[];
if (element?.type !== "node" || json?.type !== "string" || typeof json.value !== "string") {
continue;
}
const backendNodeId = (element.value as { backendNodeId?: number } | undefined)?.backendNodeId;
if (typeof backendNodeId !== "number" || !Number.isSafeInteger(backendNodeId)) continue;
try {
const state = JSON.parse(json.value) as CapturedFormState | null;
if (
!state ||
!["empty", "filled", "default"].includes(state.state ?? "") ||
typeof state.sensitive !== "boolean" ||
typeof state.placeholder !== "string" ||
(!state.sensitive &&
(typeof state.value !== "string" || typeof state.defaultValue !== "string"))
) {
continue;
}
states.set(backendNodeId, state);
} catch {
// A malformed entry must not overwrite the snapshot's own state.
}
}
return states;
}
function applyFormStates(nodes: CapturedNode[], states: Map<number, CapturedFormState>): void {
for (const node of nodes) {
if (!["input", "textarea", "select"].includes(node.tag)) continue;
const state = states.get(node.backendNodeId);
if (!state) continue;
node.formState = state.state;
node.formPlaceholder = state.placeholder ?? "";
if (
state.sensitive ||
(node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password")
) {
delete node.formValue;
delete node.formDefaultValue;
delete node.attrs.value;
} else {
node.formDefaultValue = state.defaultValue ?? "";
if (state.value !== undefined) node.formValue = state.value;
}
}
}
export async function enrichFormControlStates(
cdp: CdpRunner,
tabId: number,
frameNodeGroups: CapturedNode[][],
signal?: AbortSignal,
): Promise<boolean> {
const hasControls = frameNodeGroups.some((nodes) =>
nodes.some((node) => ["input", "textarea", "select"].includes(node.tag)),
);
if (!hasControls) return true;
const objectGroup = `bsk-vom-forms-${crypto.randomUUID()}`;
try {
throwIfAborted(signal);
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
objectGroup,
serializationOptions: {
serialization: "deep",
maxDepth: 3,
additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" },
},
});
throwIfAborted(signal);
// Backend ids are scoped to this capture's CDP target. OOPIF targets
// have separate enrichment calls and never share this lookup.
const states = formStatesByBackendId(result);
for (const nodes of frameNodeGroups) {
applyFormStates(nodes, states);
}
return true;
} catch (error) {
if (isAbortError(error)) throw error;
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
return false;
} finally {
await cdp.send(tabId, "Runtime.releaseObjectGroup", { objectGroup }).catch(() => undefined);
}
}
apps/extension/src/tools/vom/frame-capture.ts
-178
View File
@@ -1,178 +0,0 @@
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
import { type GeometryProjection, projectRectToViewport } from "../geometry";
import { GeometryContext } from "../geometry/frame-context";
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
import {
buildFrameDocuments,
type FrameAxBatch,
type FrameDocument,
type FrameOwnedAxNode,
} from "./frame-document";
export type FrameAxNode = FrameOwnedAxNode;
export type CapturedFrameDocument<T extends FrameAxNode> = FrameDocument<T>;
function throwIfAborted(signal?: AbortSignal): void {
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
}
function isAbortError(err: unknown): boolean {
return err instanceof Error && err.name === "AbortError";
}
function transformFrameNodes(
nodes: CapturedNode[],
projection: GeometryProjection | null,
): CapturedNode[] {
return nodes.map((node) => ({
...node,
rect: projection
? (() => {
const bounds = projectRectToViewport(node.rect, projection);
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
})()
: null,
}));
}
async function captureMissingFrameDocuments(
cdp: CdpRunner,
tabId: number,
graph: CdpFrameGraph,
captured: CapturedViewModel,
geometry: GeometryContext,
signal?: AbortSignal,
): Promise<void> {
if (!captured.frameNodes) captured.frameNodes = new Map();
if (!captured.frameOwnerBackendNodeIds) captured.frameOwnerBackendNodeIds = new Map();
if (!captured.frameParentIds) captured.frameParentIds = new Map();
for (const frame of graph.frames) {
if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue;
try {
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
signal,
geometry,
target: frame.target,
});
if (child.frameGeometryIssues?.length) {
(captured.frameGeometryIssues ??= []).push(...child.frameGeometryIssues);
}
let projection: GeometryProjection | null = null;
try {
projection = await geometry.targetProjection(frame.frameId);
} catch (err) {
if (isAbortError(err)) throw err;
console.debug("[bsk observation] child frame geometry projection failed", {
frameId: frame.frameId,
err,
});
}
const rootFrameId = child.rootFrameId ?? frame.frameId;
const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
if (rootFrameId !== frame.frameId && !childFrames.has(frame.frameId)) {
childFrames.set(
frame.frameId,
child.nodes.map((node) => ({ ...node, frameId: frame.frameId })),
);
}
for (const [childFrameId, nodes] of childFrames) {
captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
}
if (frame.ownerBackendNodeId !== undefined) {
captured.iframeNodes.set(
frame.ownerBackendNodeId,
captured.frameNodes.get(frame.frameId) ?? [],
);
captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
}
if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId);
for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) {
captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId);
}
for (const [childFrameId, parentFrameId] of child.frameParentIds ?? []) {
captured.frameParentIds.set(childFrameId, parentFrameId);
}
} catch (err) {
if (isAbortError(err)) throw err;
console.debug("[bsk observation] child frame DOM capture failed", {
frameId: frame.frameId,
err,
});
}
}
}
async function captureAxTrees<T extends FrameAxNode>(
cdp: CdpRunner,
tabId: number,
discoveredFrames: CdpFrame[],
captured: CapturedViewModel,
signal?: AbortSignal,
): Promise<FrameAxBatch<T>[]> {
let frames = [...discoveredFrames];
const capturedFrameIds = [...(captured.frameNodes?.keys() ?? [])];
if (frames.length === 0) {
const rootFrameId = captured.rootFrameId ?? capturedFrameIds[0] ?? "root";
frames = [{ frameId: rootFrameId, target: { tabId } }];
}
for (const frameId of capturedFrameIds) {
if (!frames.some((frame) => frame.frameId === frameId)) {
frames.push({ frameId, target: { tabId } });
}
}
let firstFailure: unknown;
const trees = await Promise.all(
frames.map(async (frame): Promise<FrameAxBatch<T>> => {
try {
throwIfAborted(signal);
await sendToCdpTarget(cdp, frame.target, "Accessibility.enable", {});
const result = await sendToCdpTarget<{ nodes?: T[] }>(
cdp,
frame.target,
"Accessibility.getFullAXTree",
frame.frameId === "root" ? {} : { frameId: frame.frameId },
);
throwIfAborted(signal);
return {
frame,
nodes: result.nodes ?? [],
};
} catch (err) {
if (isAbortError(err)) throw err;
firstFailure ??= err;
console.debug("[bsk observation] frame AX capture failed", {
frameId: frame.frameId,
err,
});
return { frame, nodes: [] };
}
}),
);
const capturedNodeCount = [...captured.iframeNodes.values()].reduce(
(count, nodes) => count + nodes.length,
captured.nodes.length,
);
if (firstFailure && capturedNodeCount === 0 && trees.every((tree) => tree.nodes.length === 0)) {
throw firstFailure;
}
return trees;
}
export async function captureFrameData<T extends FrameAxNode>(
cdp: CdpRunner,
tabId: number,
captured: CapturedViewModel,
signal?: AbortSignal,
geometry = new GeometryContext(cdp, tabId, undefined, signal),
): Promise<CapturedFrameDocument<T>[]> {
const graph = await geometry.graph();
const frames = graph?.frames ?? [];
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, geometry, signal);
throwIfAborted(signal);
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
return buildFrameDocuments(graph, batches, captured);
}
apps/extension/src/tools/vom/frame-document.ts
+73 -42
View File
@@ -4,7 +4,8 @@ import {
type CdpTarget,
cdpTargetKey,
} from "@/browser-driver/frame-graph";
import type { CapturedNode, CapturedViewModel } from "./capture";
import type { CapturedNode } from "./facts";
import { captureCheckpoint } from "./facts";
export interface FrameOwnedAxNode {
nodeId: string;
@@ -23,6 +24,16 @@ export interface FrameDocument<T extends FrameOwnedAxNode> extends CdpFrame {
contextScopeId: string;
axNodes: T[];
domNodes: CapturedNode[];
excludedBackendNodeIds?: ReadonlySet<number>;
}
/** Explicit snapshot document data used to join DOM and AX ownership. */
export interface FrameDomInput {
nodes: CapturedNode[];
rootFrameId?: string;
frameNodes?: ReadonlyMap<string, CapturedNode[]>;
frameOwnerBackendNodeIds?: ReadonlyMap<string, number>;
frameParentIds?: ReadonlyMap<string, string>;
}
interface Ownership {
@@ -47,17 +58,13 @@ function targetBackendKey(target: CdpTarget, backendNodeId: number): string {
function frameList<T extends FrameOwnedAxNode>(
graph: CdpFrameGraph | null,
batches: FrameAxBatch<T>[],
captured: CapturedViewModel,
captured: FrameDomInput,
): CdpFrame[] {
const frames = new Map<string, CdpFrame>();
for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame);
for (const batch of batches) {
if (!frames.has(batch.frame.frameId)) frames.set(batch.frame.frameId, batch.frame);
}
for (const frameId of captured.frameNodes?.keys() ?? []) {
if (!frames.has(frameId))
frames.set(frameId, { frameId, target: { tabId: batches[0]?.frame.target.tabId ?? 0 } });
}
return [...frames.values()].map((frame) => {
const ownerBackendNodeId =
frame.ownerBackendNodeId ?? captured.frameOwnerBackendNodeIds?.get(frame.frameId);
@@ -70,11 +77,14 @@ function frameList<T extends FrameOwnedAxNode>(
});
}
export function buildFrameDocuments<T extends FrameOwnedAxNode>(
export async function buildFrameDocuments<T extends FrameOwnedAxNode>(
graph: CdpFrameGraph | null,
batches: FrameAxBatch<T>[],
captured: CapturedViewModel,
): FrameDocument<T>[] {
captured: FrameDomInput,
signal?: AbortSignal,
unresolved?: (frame: CdpFrame) => void,
): Promise<FrameDocument<T>[]> {
let work = 0;
const frames = frameList(graph, batches, captured);
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId;
@@ -83,54 +93,74 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
const backendOwner = new Map<string, string>();
for (const frame of frames) {
for (const node of domNodesForFrame(frame.frameId)) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId);
}
}
const candidates = new Map<string, OwnedCandidate<T>>();
for (const batch of batches) {
const nodeById = new Map(batch.nodes.map((node) => [node.nodeId, node]));
const nodeById = new Map<string, T>();
for (const node of batch.nodes) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
nodeById.set(node.nodeId, node);
}
let reported = false;
const ownershipByNodeId = new Map<string, Ownership>();
const resolving = new Set<string>();
const resolveOwnership = (node: T): Ownership => {
const cached = ownershipByNodeId.get(node.nodeId);
if (cached) return cached;
if (node.frameId && frameById.has(node.frameId)) {
const ownership = { frameId: node.frameId, strength: 4 };
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
}
if (typeof node.backendDOMNodeId === "number") {
const frameId = backendOwner.get(
targetBackendKey(batch.frame.target, node.backendDOMNodeId),
);
const resolveOwnership = async (node: T): Promise<Ownership> => {
const path: T[] = [];
const visiting = new Set<string>();
let current: T | undefined = node;
let ownership: Ownership = { frameId: batch.frame.frameId, strength: 1 };
while (current) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
const cached = ownershipByNodeId.get(current.nodeId);
if (cached) {
ownership = cached;
break;
}
if (visiting.has(current.nodeId)) break;
visiting.add(current.nodeId);
if (current.frameId) {
const frame = frameById.get(current.frameId);
// An explicit contradiction must not fall through to a guessed owner.
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(batch.frame.target)) {
if (!reported) {
unresolved?.(batch.frame);
reported = true;
}
ownership = { frameId: "", strength: 0 };
break;
}
ownership = { frameId: frame.frameId, strength: 4 };
ownershipByNodeId.set(current.nodeId, ownership);
break;
}
const frameId =
typeof current.backendDOMNodeId === "number"
? backendOwner.get(targetBackendKey(batch.frame.target, current.backendDOMNodeId))
: undefined;
if (frameId) {
const ownership = { frameId, strength: 3 };
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
ownership = { frameId, strength: 3 };
ownershipByNodeId.set(current.nodeId, ownership);
break;
}
path.push(current);
current = current.parentId ? nodeById.get(current.parentId) : undefined;
}
if (node.parentId && !resolving.has(node.nodeId)) {
const parent = nodeById.get(node.parentId);
if (parent) {
resolving.add(node.nodeId);
const inherited = resolveOwnership(parent);
resolving.delete(node.nodeId);
const ownership = {
frameId: inherited.frameId,
strength: Math.min(2, inherited.strength),
};
ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
}
for (let i = path.length - 1; i >= 0; i--) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
ownership = { frameId: ownership.frameId, strength: Math.min(2, ownership.strength) };
ownershipByNodeId.set(path[i].nodeId, ownership);
}
const ownership = { frameId: batch.frame.frameId, strength: 1 };
ownershipByNodeId.set(node.nodeId, ownership);
if (!ownershipByNodeId.has(node.nodeId)) ownershipByNodeId.set(node.nodeId, ownership);
return ownership;
};
for (const node of batch.nodes) {
const ownership = resolveOwnership(node);
if (work++ % 256 === 0) await captureCheckpoint(signal);
const ownership = await resolveOwnership(node);
if (!ownership.frameId) continue;
const key = targetNodeKey(batch.frame.target, node.nodeId);
const existing = candidates.get(key);
if (!existing || ownership.strength > existing.ownership.strength) {
@@ -144,6 +174,7 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
);
const axNodesByFrame = new Map<string, T[]>();
for (const candidate of candidates.values()) {
if (work++ % 256 === 0) await captureCheckpoint(signal);
const { node, target, ownership } = candidate;
const parentFrameId = node.parentId
? ownershipByTargetNode.get(targetNodeKey(target, node.parentId))
apps/extension/src/tools/vom/normalize.ts
+217
View File
@@ -0,0 +1,217 @@
import type { Viewport } from "@browser-skill/vom";
import type { CdpFrame, CdpTarget } from "@/browser-driver/frame-graph";
import { type GeometryProjection, projectRectToViewport } from "../geometry";
import {
type FrameProjectionState,
projectSnapshotRect,
snapshotViewportRect,
} from "../geometry/coordinate-types";
import { cssViewport, GeometryContext } from "../geometry/frame-context";
import type { CapturedNode } from "./facts";
import {
buildDocumentIndex,
type CaptureIssue,
captureCheckpoint,
type DocumentIndex,
isCaptureAbort,
type NodeFacts,
throwCaptureAborted,
} from "./facts";
import {
decodeDocument,
type SnapshotDocument,
type SnapshotReply,
snapshotFrameId,
} from "./snapshot";
export interface NormalizedDocument {
index: DocumentIndex;
frame: CdpFrame;
domNodes: CapturedNode[];
}
/** One target's snapshot. Source ownership is checked before interpreting any
* coordinates. Snapshot and live quad coordinates are deliberately not conflated. */
export async function normalizeSnapshot(
snapshot: SnapshotReply,
target: CdpTarget,
frames: readonly CdpFrame[],
geometry: GeometryContext,
issues: CaptureIssue[],
signal?: AbortSignal,
): Promise<NormalizedDocument[]> {
const strings = snapshot.strings ?? [];
const raw = snapshot.documents ?? [];
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
const sources = new Map<string, SnapshotDocument>();
for (const doc of raw) {
const id = snapshotFrameId(doc, strings);
if (!id || !frameById.has(id) || sources.has(id)) {
issues.push({
target,
frameId: id,
stage: "ownership",
reason: "frame-ownership-unresolved",
});
continue;
}
sources.set(id, doc);
}
let viewport: Viewport = { width: 0, height: 0 };
let scrollX = 0,
scrollY = 0;
try {
const measured = cssViewport(await geometry.layoutMetrics(target));
viewport = { width: measured.width, height: measured.height };
scrollX = measured.scrollX;
scrollY = measured.scrollY;
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
const children = new Map<string, CdpFrame[]>();
const pending: CdpFrame[] = [];
for (const frame of frames) {
if (!frame.parentFrameId || !frameById.has(frame.parentFrameId)) pending.push(frame);
else {
const siblings = children.get(frame.parentFrameId);
if (siblings) siblings.push(frame);
else children.set(frame.parentFrameId, [frame]);
}
}
const projections = new Map<string, FrameProjectionState | null>();
const result: NormalizedDocument[] = [];
let targetProjection: GeometryProjection | null = null;
const targetRoot = pending[0];
if (target.sessionId && targetRoot) {
try {
targetProjection = await geometry.targetProjection(targetRoot.frameId);
} catch (error) {
if (isCaptureAbort(error)) throw error;
}
}
for (let cursor = 0; cursor < pending.length; cursor++) {
if (cursor % 256 === 0) await captureCheckpoint(signal);
const frame = pending[cursor];
for (const child of children.get(frame.frameId) ?? []) pending.push(child);
const doc = sources.get(frame.frameId);
if (!doc) continue;
const source = { target, frameId: frame.frameId };
const parent = frame.parentFrameId ? frameById.get(frame.parentFrameId) : undefined;
const state: FrameProjectionState | null = !parent
? {
status: "available",
projection: { source, geometry: { sourceClips: [], edges: [], topViewport: viewport } },
}
: (projections.get(frame.frameId) ?? null);
projections.set(frame.frameId, state);
const projection = state?.status === "available" ? state.projection : null;
if (!projection || (target.sessionId && !targetProjection))
issues.push({
target,
frameId: frame.frameId,
stage: "geometry",
reason: "geometry-unavailable",
...(state && state.status !== "available" ? { projectionIssue: state } : {}),
});
// Only direct siblings share this pool. Keep normalization in its existing
// breadth-first order, independent of owner read completion order.
const siblings = (children.get(frame.frameId) ?? []).filter((child) =>
sources.has(child.frameId),
);
const edge = projection?.geometry.edges[0];
const clips = edge
? [edge.destinationQuad, ...(edge.destinationClips ?? [])]
: (projection?.geometry.sourceClips ?? []);
let next = 0;
let failure: unknown;
await Promise.all(
Array.from({ length: Math.min(4, siblings.length) }, async () => {
while (next < siblings.length && !signal?.aborted && !failure) {
const child = siblings[next++];
const childSource = { target, frameId: child.frameId };
if (!projection) {
projections.set(
child.frameId,
state && state.status !== "available"
? {
status: "blocked",
source: childSource,
cause: state.status === "blocked" ? state.cause : state,
}
: null,
);
continue;
}
if (child.ownerBackendNodeId === undefined) {
projections.set(child.frameId, null);
continue;
}
try {
projections.set(
child.frameId,
await geometry.snapshotProjection(
childSource,
child.ownerBackendNodeId,
clips,
viewport,
),
);
} catch (error) {
failure ??= error;
}
}
}),
);
// Join active reads and their cleanup before returning cancellation.
if (failure) throw failure;
throwCaptureAborted(signal);
const decoded = await decodeDocument(doc, strings, signal);
const nodes: NodeFacts[] = [];
for (let i = 0; i < decoded.nodes.length; i++) {
if (i % 256 === 0) await captureCheckpoint(signal);
const node = decoded.nodes[i];
const input = snapshotViewportRect(node.layout?.bounds ?? [], source, {
x: doc.scrollOffsetX ?? (parent ? 0 : scrollX),
y: doc.scrollOffsetY ?? (parent ? 0 : scrollY),
});
const local = input?.rect;
let rect = input && projection ? projectSnapshotRect(input, projection) : null;
if (rect && target.sessionId)
rect = targetProjection
? projectRectToViewport(
{ x: rect.x, y: rect.y, w: rect.width, h: rect.height },
targetProjection,
)
: null;
const styles = node.layout?.styles;
const visibility = styles?.visibility || "visible";
const opacity = styles?.opacity || "1";
nodes.push({
...node,
frameId: frame.frameId,
ownerFrameBackendNodeId: frame.ownerBackendNodeId ?? null,
localRect: local ? { x: local.x, y: local.y, w: local.width, h: local.height } : null,
rect: rect ? { x: rect.x, y: rect.y, w: rect.width, h: rect.height } : null,
rendered:
!!local &&
visibility !== "hidden" &&
visibility !== "collapse" &&
(Number.parseFloat(opacity) || 0) > 0,
});
}
const index = await buildDocumentIndex(nodes, signal);
const domNodes = nodes.filter((node) => !node.tag.startsWith("#"));
result.push({
index,
frame,
domNodes,
});
}
for (const frameId of sources.keys()) {
if (!projections.has(frameId))
issues.push({ target, frameId, stage: "ownership", reason: "frame-ownership-unresolved" });
}
return result;
}
apps/extension/src/tools/vom/record-safe-observation.ts
+4 -1
View File
@@ -2,7 +2,10 @@ import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { readRecordingDocumentIdentity } from "@/shared/recording-document-identity";
import type { CapturedSurfaceProbe } from "./capture";
import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture";
import type {
FrameDocument as CapturedFrameDocument,
FrameOwnedAxNode as FrameAxNode,
} from "./frame-document";
/** Frame identity needed to resolve an `@eN` that lives in an iframe. */
export interface CaptureVomFrame {
apps/extension/src/tools/vom/semantic-graph/build.ts
+6 -4
View File
@@ -61,8 +61,9 @@ export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGrap
dom,
sourceOrder: index,
excluded:
document.frameId === rootFrameId &&
input.excludedBackendNodeIds?.has(dom.backendNodeId) === true,
document.excludedBackendNodeIds?.has(dom.backendNodeId) === true ||
(document.frameId === rootFrameId &&
input.excludedBackendNodeIds?.has(dom.backendNodeId) === true),
};
nodes.set(id, node);
frameNodeIds.push(id);
@@ -90,9 +91,10 @@ export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGrap
ax,
sourceOrder: document.domNodes.length + index,
excluded:
document.frameId === rootFrameId &&
typeof ax.backendDOMNodeId === "number" &&
input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true,
(document.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true ||
(document.frameId === rootFrameId &&
input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true)),
});
frameNodeIds.push(id);
}
apps/extension/src/tools/vom/snapshot.ts
+230
View File
@@ -0,0 +1,230 @@
import type { CapturedNode } from "./facts";
import { captureCheckpoint, type DecodedDocument, type DecodedNode } from "./facts";
export const REQUESTED_STYLES = [
"position",
"pointer-events",
"cursor",
"visibility",
"opacity",
"display",
"overflow-x",
"overflow-y",
"transform",
"zoom",
"clip-path",
"mask-image",
] as const;
const STYLE_COL = Object.fromEntries(
REQUESTED_STYLES.map((name, index) => [name, index]),
) as Record<(typeof REQUESTED_STYLES)[number], number>;
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
interface SparseArray {
index: number[];
value: number[];
}
interface RareBooleanData {
index: number[];
}
export interface SnapshotDocument {
frameId?: string | number;
scrollOffsetX?: number;
scrollOffsetY?: number;
nodes?: {
parentIndex?: number[];
nodeName?: number[];
nodeType?: number[];
backendNodeId?: number[];
attributes?: number[][];
/**
* Per-node text value (index into strings), set for `#text` / CDATA nodes.
* Element nodes carry -1. Same length as `backendNodeId`.
*/
nodeValue?: number[];
/** Maps node array index → index into `documents[]` for frame content. */
contentDocumentIndex?: SparseArray;
inputValue?: SparseArray;
textValue?: SparseArray;
inputChecked?: RareBooleanData;
optionSelected?: RareBooleanData;
};
layout?: {
nodeIndex?: number[];
styles?: number[][];
bounds?: number[][];
paintOrders?: number[];
clientRects?: number[][];
offsetRects?: number[][];
scrollRects?: number[][];
};
}
export function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
if (typeof document.frameId === "string") return document.frameId || undefined;
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
return undefined;
}
export interface SnapshotReply {
strings?: string[];
documents?: SnapshotDocument[];
}
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
}
function snapshotFormState(
value: string | undefined,
defaultValue: string,
hasDefaultValue: boolean,
sensitive: boolean,
): CapturedNode["formState"] {
if (sensitive) {
if (value === undefined && !hasDefaultValue) return undefined;
return (value ?? defaultValue) === "" ? "empty" : "filled";
}
if (value === undefined) return undefined;
if (value === "") return "empty";
return value === defaultValue ? "default" : "filled";
}
function str(strings: string[], idx: number | undefined): string {
if (idx === undefined || idx < 0) return "";
return strings[idx] ?? "";
}
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
const out = new Map<number, number>();
if (!sparse?.index || !sparse.value) return out;
for (let i = 0; i < sparse.index.length; i++) {
const nodeIndex = sparse.index[i];
const docIndex = sparse.value[i];
if (nodeIndex !== undefined && docIndex !== undefined) out.set(nodeIndex, docIndex);
}
return out;
}
export async function decodeDocument(
doc: SnapshotDocument,
strings: string[],
signal?: AbortSignal,
): Promise<DecodedDocument> {
const dn = doc.nodes;
const dl = doc.layout;
if (!dn?.backendNodeId) {
return { nodes: [] };
}
const count = dn.backendNodeId.length;
const layoutByNode = new Map<number, number>();
for (let i = 0; i < (dl?.nodeIndex?.length ?? 0); i++) {
if (i % 256 === 0) await captureCheckpoint(signal);
layoutByNode.set(dl!.nodeIndex![i], i);
}
const inputValues = sparseIndexMap(dn.inputValue);
const textValues = sparseIndexMap(dn.textValue);
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
// Collect visible text from #text child nodes so element CapturedNodes
// carry a textContent value usable as a button/link label fallback.
// nodeValue is a parallel array: string index for text nodes, -1 otherwise.
const nodeTextContent = new Map<number, string[]>();
if (dn.nodeValue) {
for (let n = 0; n < count; n++) {
if (n % 256 === 0) await captureCheckpoint(signal);
const nvIdx = dn.nodeValue[n] ?? -1;
if (nvIdx < 0) continue;
const text = str(strings, nvIdx).trim();
if (!text) continue;
const parentIdx = dn.parentIndex?.[n] ?? -1;
if (parentIdx >= 0) {
const existing = nodeTextContent.get(parentIdx);
if (existing) existing.push(text);
else nodeTextContent.set(parentIdx, [text]);
}
}
}
// Decode fields without assigning a coordinate projection or semantic policy.
const nodes: DecodedNode[] = [];
for (let n = 0; n < count; n++) {
if (n % 256 === 0) await captureCheckpoint(signal);
const backendNodeId = dn.backendNodeId[n];
const parentIdx = dn.parentIndex?.[n] ?? -1;
const parentBackendNodeId = parentIdx >= 0 ? (dn.backendNodeId[parentIdx] ?? null) : null;
const tag = str(strings, dn.nodeName?.[n]).toLowerCase();
const attrs: Record<string, string> = {};
const pairs = dn.attributes?.[n] ?? [];
for (let a = 0; a + 1 < pairs.length; a += 2) {
attrs[str(strings, pairs[a]).toLowerCase()] = str(strings, pairs[a + 1]);
}
const li = layoutByNode.get(n);
const styleRow = li === undefined ? [] : (dl?.styles?.[li] ?? []);
const styles: Record<string, string> = {};
for (const name of REQUESTED_STYLES) styles[name] = str(strings, styleRow[STYLE_COL[name]]);
const layout =
li === undefined
? undefined
: {
boundsSpace: "snapshot-document-css" as const,
clientSpace: "unscaled-client-offset-css" as const,
bounds: dl?.bounds?.[li],
clientRect: dl?.clientRects?.[li],
offsetRect: dl?.offsetRects?.[li],
scrollRect: dl?.scrollRects?.[li],
styles,
};
const textContent = nodeTextContent.get(n)?.join(" ");
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
const rawFormValue =
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
const sensitive = isSensitiveFormControl({ tag, attrs });
const formDefaultValue = attrs.value ?? "";
const formValue = sensitive ? undefined : rawFormValue;
const formState = snapshotFormState(
rawFormValue,
formDefaultValue,
Object.prototype.hasOwnProperty.call(attrs, "value"),
sensitive,
);
if (sensitive) delete attrs.value;
nodes.push({
backendNodeId,
nodeType: dn.nodeType?.[n],
parentBackendNodeId,
...(parentIdx >= 0 && dn.backendNodeId[parentIdx] === undefined
? { parentMissing: true }
: {}),
tag,
attrs,
layout,
paintOrder: li === undefined ? 0 : (dl?.paintOrders?.[li] ?? 0),
position: styles.position || "static",
pointerEvents: styles["pointer-events"] || "auto",
cursor: styles.cursor || "auto",
textContent,
...(formValue !== undefined ? { formValue } : {}),
...(tag === "input" || tag === "textarea"
? {
formPlaceholder: attrs.placeholder ?? "",
...(!sensitive ? { formDefaultValue } : {}),
...(formState ? { formState } : {}),
}
: {}),
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
});
}
return { nodes };
}