refactor(observation): organize capture around document facts

This commit is contained in:
Ljy-0827
2026-09-09 13:20:22 +08:00
committed by GitHub
parent 2d2636030d
commit cda20d979b
18 changed files with 2334 additions and 1338 deletions
@@ -4,9 +4,9 @@ import { CAPTURE_SUPPRESS, type CaptureSuppressMessage } from "@/lib/capture-sup
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "@/lib/overlay-bridge";
import { SessionManager } from "@/session-manager/manager";
import type { CdpRunner } from "@/tools/shared";
import type { BuildVomSceneOptions, VomFrameDocument } from "../observation";
import {
buildFrameVomScene,
buildVomScene,
type CdpAxNode,
captureVomObservation,
handleGetHtml,
@@ -17,7 +17,35 @@ import {
type ScreenshotDeps,
stripDataUrlPrefix,
} from "../observation";
import type { CapturedNode, CapturedViewModel } from "../vom/capture";
import type { CapturedNode, CapturedSceneInput } from "../vom/facts";
interface SceneFixture extends CapturedSceneInput {
documents?: VomFrameDocument[];
}
// Only supplies the common root fixture; child documents declare their own ownership.
function buildTestScene(
axNodes: CdpAxNode[],
captured: SceneFixture,
options: BuildVomSceneOptions = {},
) {
const rootFrameId = captured.rootFrameId ?? "root";
return buildFrameVomScene(
[
{
frameId: rootFrameId,
contextScopeId: rootFrameId,
target: { tabId: 7 },
url: options.pageUrl,
domNodes: captured.nodes,
axNodes,
},
...(captured.documents ?? []),
],
captured,
options,
);
}
function fakeAgentWindow(ids: number[]) {
let i = 0;
@@ -522,10 +550,10 @@ describe("handleScreenshot overlay suppression", () => {
});
// ---------------------------------------------------------------------------
// buildVomScene
// Semantic scene fixtures
// ---------------------------------------------------------------------------
describe("buildVomScene", () => {
describe("buildFrameVomScene", () => {
it("merges multiple sibling and nested frames without mixing their targets", () => {
const node = (
backendNodeId: number,
@@ -563,25 +591,10 @@ describe("buildVomScene", () => {
node(300, null, "nested", "body"),
node(301, 300, "nested", "input", { placeholder: "Nested field" }),
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1200, height: 800 },
nodes: mainNodes,
iframeNodes: new Map([
[10, firstNodes],
[20, secondNodes],
[210, nestedNodes],
]),
frameNodes: new Map([
["main", mainNodes],
["first", firstNodes],
["second", secondNodes],
["nested", nestedNodes],
]),
frameOwnerBackendNodeIds: new Map([
["first", 10],
["second", 20],
["nested", 210],
]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
@@ -680,11 +693,10 @@ describe("buildVomScene", () => {
position: "static",
pointerEvents: "auto",
};
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 800, height: 600 },
nodes: [],
iframeNodes: new Map(),
frameNodes: new Map([["child", [childNode]]]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
@@ -746,14 +758,10 @@ describe("buildVomScene", () => {
pointerEvents: "auto",
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 800, height: 600 },
nodes: mainNodes,
iframeNodes: new Map(),
frameNodes: new Map([
["main", mainNodes],
["child", []],
]),
rootFrameId: "main",
excludedBackendNodeIds: new Set(),
};
@@ -829,9 +837,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 200,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -857,11 +864,11 @@ describe("buildVomScene", () => {
],
};
const scene = buildVomScene(axNodes, captured);
const scene = buildTestScene(axNodes, captured);
expect(scene.viewport).toEqual({ width: 1000, height: 800 });
expect(scene.nodes[0]).toEqual(
expect.objectContaining({
id: 100,
backendNodeId: 100,
parentId: null,
role: "RootWebArea",
name: "Example",
@@ -873,14 +880,14 @@ describe("buildVomScene", () => {
);
expect(scene.nodes[1]).toEqual(
expect.objectContaining({
id: 200,
parentId: 100,
backendNodeId: 200,
parentId: scene.nodes.find((node) => node.backendNodeId === 100)!.id,
role: "button",
name: "Submit",
tag: "button",
rect: { x: 20, y: 20, w: 120, h: 40 },
domParentId: 100,
domAncestorIds: [100],
domParentId: scene.nodes.find((node) => node.backendNodeId === 100)!.id,
domAncestorIds: [scene.nodes.find((node) => node.backendNodeId === 100)!.id],
attrs: {},
}),
);
@@ -909,9 +916,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 300,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -947,10 +953,10 @@ describe("buildVomScene", () => {
],
};
const scene = buildVomScene(axNodes, captured, { pageUrl: "https://app.example.test/home" });
const scene = buildTestScene(axNodes, captured, { pageUrl: "https://app.example.test/home" });
expect(scene.nodes.find((node) => node.id === 200)?.href).toBe("docs.example.org");
expect(scene.nodes.find((node) => node.id === 300)?.href).toBeUndefined();
expect(scene.nodes.find((node) => node.backendNodeId === 200)?.href).toBe("docs.example.org");
expect(scene.nodes.find((node) => node.backendNodeId === 300)?.href).toBeUndefined();
});
it("uses the nearest backend AX ancestor as parentId", () => {
@@ -975,9 +981,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 30,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 400, height: 300 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1003,10 +1008,12 @@ describe("buildVomScene", () => {
],
};
const scene = buildVomScene(axNodes, captured);
const scene = buildTestScene(axNodes, captured);
expect(scene.nodes.map((node) => node.id)).toEqual([10, 30]);
expect(scene.nodes.find((node) => node.id === 30)?.parentId).toBe(10);
expect(scene.nodes.map((node) => node.backendNodeId)).toEqual([10, 30]);
expect(scene.nodes.find((node) => node.backendNodeId === 30)?.parentId).toBe(
scene.nodes.find((node) => node.backendNodeId === 10)!.id,
);
});
it("maps iframe sub-document controls to VomNodes", () => {
@@ -1024,7 +1031,7 @@ describe("buildVomScene", () => {
backendDOMNodeId: 20,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
nodes: [
{
@@ -1048,10 +1055,15 @@ describe("buildVomScene", () => {
pointerEvents: "auto",
},
],
iframeNodes: new Map([
[
20,
[
documents: [
{
frameId: "child",
contextScopeId: "child",
parentFrameId: "root",
ownerBackendNodeId: 20,
target: { tabId: 7 },
axNodes: [],
domNodes: [
{
backendNodeId: 101,
parentBackendNodeId: null,
@@ -1073,26 +1085,25 @@ describe("buildVomScene", () => {
pointerEvents: "auto",
},
],
],
]),
},
],
excludedBackendNodeIds: new Set(),
};
const iframeControls = buildVomScene(axNodes, captured).nodes.filter((node) =>
[101, 102].includes(node.id),
);
const scene = buildTestScene(axNodes, captured);
const iframeControls = scene.nodes.filter((node) => [101, 102].includes(node.backendNodeId!));
expect(iframeControls).toEqual([
expect.objectContaining({
id: 101,
parentId: 20,
backendNodeId: 101,
parentId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
role: "textbox",
name: "请输入手机号",
sensitive: false,
}),
expect.objectContaining({
id: 102,
parentId: 20,
backendNodeId: 102,
parentId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
role: "textbox",
name: "密码",
sensitive: true,
@@ -1113,11 +1124,9 @@ describe("buildVomScene", () => {
parentId: "1",
role: { type: "role", value: "Iframe" },
backendDOMNodeId: 20,
childIds: ["3"],
},
{
nodeId: "3",
parentId: "2",
role: { type: "role", value: "textbox" },
name: { type: "x", value: "输入密码" },
value: { value: "iframe-secret" },
@@ -1125,7 +1134,7 @@ describe("buildVomScene", () => {
properties: [{ name: "inputType", value: { value: "password" } }],
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
nodes: [
{
@@ -1149,10 +1158,15 @@ describe("buildVomScene", () => {
pointerEvents: "auto",
},
],
iframeNodes: new Map([
[
20,
[
documents: [
{
frameId: "child",
contextScopeId: "child",
parentFrameId: "root",
ownerBackendNodeId: 20,
target: { tabId: 7 },
axNodes: [axNodes[2]],
domNodes: [
{
backendNodeId: 102,
parentBackendNodeId: null,
@@ -1164,25 +1178,27 @@ describe("buildVomScene", () => {
pointerEvents: "auto",
},
],
],
]),
},
],
excludedBackendNodeIds: new Set(),
};
const passwordNodes = buildVomScene(axNodes, captured).nodes.filter(
const passwordNodes = buildTestScene(axNodes.slice(0, 2), captured).nodes.filter(
(node) => node.role === "textbox" && node.name === "输入密码",
);
expect(passwordNodes).toHaveLength(1);
expect(passwordNodes[0]).toEqual(expect.objectContaining({ id: 202, sensitive: true }));
expect(passwordNodes[0]).toEqual(
expect.objectContaining({ backendNodeId: 202, sensitive: true }),
);
expect(passwordNodes[0].value).toBeUndefined();
const rendered = renderVom(buildVomScene(axNodes, captured)).text;
const rendered = renderVom(buildTestScene(axNodes.slice(0, 2), captured)).text;
expect(rendered).toContain('textbox "输入密码" [filled] ="•••"');
expect(rendered).not.toContain("iframe-secret");
});
it("keeps unnamed iframe controls but skips unnamed iframe links", () => {
const scene = buildVomScene(
const scene = buildTestScene(
[
{
nodeId: "1",
@@ -1221,10 +1237,15 @@ describe("buildVomScene", () => {
pointerEvents: "auto",
},
],
iframeNodes: new Map([
[
20,
[
documents: [
{
frameId: "child",
contextScopeId: "child",
parentFrameId: "root",
ownerBackendNodeId: 20,
target: { tabId: 7 },
axNodes: [],
domNodes: [
{
backendNodeId: 201,
parentBackendNodeId: null,
@@ -1246,25 +1267,25 @@ describe("buildVomScene", () => {
pointerEvents: "auto",
},
],
],
]),
},
],
excludedBackendNodeIds: new Set(),
},
);
expect(scene.nodes.find((node) => node.id === 201)).toEqual(
expect(scene.nodes.find((node) => node.backendNodeId === 201)).toEqual(
expect.objectContaining({
id: 201,
parentId: 20,
backendNodeId: 201,
parentId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
role: "button",
}),
);
expect(scene.nodes.find((node) => node.id === 201)).not.toHaveProperty("name");
expect(scene.nodes.find((node) => node.id === 202)).toBeUndefined();
expect(scene.nodes.find((node) => node.backendNodeId === 201)).not.toHaveProperty("name");
expect(scene.nodes.find((node) => node.backendNodeId === 202)).toBeUndefined();
});
it("preserves captured-only iframe anchors for iframe sub-document controls", () => {
const scene = buildVomScene(
const scene = buildTestScene(
[
{
nodeId: "1",
@@ -1297,10 +1318,15 @@ describe("buildVomScene", () => {
pointerEvents: "auto",
},
],
iframeNodes: new Map<number, CapturedNode[]>([
[
20,
[
documents: [
{
frameId: "child",
contextScopeId: "child",
parentFrameId: "root",
ownerBackendNodeId: 20,
target: { tabId: 7 },
axNodes: [],
domNodes: [
{
backendNodeId: 201,
parentBackendNodeId: null,
@@ -1322,16 +1348,16 @@ describe("buildVomScene", () => {
pointerEvents: "auto",
},
],
],
]),
},
],
excludedBackendNodeIds: new Set(),
},
);
expect(scene.nodes.find((node) => node.id === 20)).toEqual(
expect(scene.nodes.find((node) => node.backendNodeId === 20)).toEqual(
expect.objectContaining({
id: 20,
parentId: 10,
backendNodeId: 20,
parentId: scene.nodes.find((node) => node.backendNodeId === 10)!.id,
role: "Iframe",
name: "支付验证",
}),
@@ -1352,9 +1378,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 10,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 320, height: 240 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1375,9 +1400,9 @@ describe("buildVomScene", () => {
],
};
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
expect(buildTestScene(axNodes, captured).nodes[0]).toEqual(
expect.objectContaining({
id: 10,
backendNodeId: 10,
sensitive: true,
}),
);
@@ -1392,9 +1417,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 10,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1411,9 +1435,9 @@ describe("buildVomScene", () => {
],
};
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
expect(buildTestScene(axNodes, captured).nodes[0]).toEqual(
expect.objectContaining({
id: 10,
backendNodeId: 10,
role: "textbox",
name: "+86",
placeholder: "输入手机号",
@@ -1431,9 +1455,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 10,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1450,8 +1473,13 @@ describe("buildVomScene", () => {
],
};
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
expect.objectContaining({ id: 10, role: "textbox", name: "Email", value: "a@b.com" }),
expect(buildTestScene(axNodes, captured).nodes[0]).toEqual(
expect.objectContaining({
backendNodeId: 10,
role: "textbox",
name: "Email",
value: "a@b.com",
}),
);
});
@@ -1470,9 +1498,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 20,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1510,9 +1537,9 @@ describe("buildVomScene", () => {
],
};
expect(buildVomScene(axNodes, captured).nodes.find((node) => node.id === 20)).toEqual(
expect.objectContaining({ role: "textbox", name: "验证码" }),
);
expect(
buildTestScene(axNodes, captured).nodes.find((node) => node.backendNodeId === 20),
).toEqual(expect.objectContaining({ role: "textbox", name: "验证码" }));
});
it("uses preceding AX static text as a form control label", () => {
@@ -1536,9 +1563,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 20,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1555,13 +1581,13 @@ describe("buildVomScene", () => {
],
};
expect(buildVomScene(axNodes, captured).nodes.find((node) => node.id === 20)).toEqual(
expect.objectContaining({ role: "textbox", name: "验证码", inputState: "empty" }),
);
expect(
buildTestScene(axNodes, captured).nodes.find((node) => node.backendNodeId === 20),
).toEqual(expect.objectContaining({ role: "textbox", name: "验证码", inputState: "empty" }));
});
it("marks runtime default input values without treating them as ordinary filled input", () => {
const scene = buildVomScene(
const scene = buildTestScene(
[
{
nodeId: "1",
@@ -1571,7 +1597,6 @@ describe("buildVomScene", () => {
],
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1615,9 +1640,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 20,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1667,9 +1691,14 @@ describe("buildVomScene", () => {
],
};
const scene = buildVomScene(axNodes, captured);
expect(scene.nodes.find((n) => n.id === 20)).toEqual(
expect.objectContaining({ id: 20, role: "button", name: "close", cursor: "pointer" }),
const scene = buildTestScene(axNodes, captured);
expect(scene.nodes.find((n) => n.backendNodeId === 20)).toEqual(
expect.objectContaining({
backendNodeId: 20,
role: "button",
name: "close",
cursor: "pointer",
}),
);
const rendered = renderVom(scene);
expect(rendered.text).toContain('@e1 button "close"');
@@ -1703,9 +1732,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 30,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1744,9 +1772,9 @@ describe("buildVomScene", () => {
],
};
const scene = buildVomScene(axNodes, captured);
expect(scene.nodes.find((n) => n.id === 20)).toBeUndefined();
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("link");
const scene = buildTestScene(axNodes, captured);
expect(scene.nodes.find((n) => n.backendNodeId === 20)).toBeUndefined();
expect(scene.nodes.find((n) => n.backendNodeId === 30)?.role).toBe("link");
});
it("promotes only the outermost clickable in a nested pointer chain", () => {
@@ -1771,9 +1799,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 30,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1812,11 +1839,15 @@ describe("buildVomScene", () => {
],
};
const scene = buildVomScene(axNodes, captured);
expect(scene.nodes.find((n) => n.id === 20)).toEqual(
expect.objectContaining({ id: 20, role: "button", attrs: { "aria-label": "收藏" } }),
const scene = buildTestScene(axNodes, captured);
expect(scene.nodes.find((n) => n.backendNodeId === 20)).toEqual(
expect.objectContaining({
backendNodeId: 20,
role: "button",
attrs: { "aria-label": "收藏" },
}),
);
expect(scene.nodes.find((n) => n.id === 30)).toBeUndefined();
expect(scene.nodes.find((n) => n.backendNodeId === 30)).toBeUndefined();
const rendered = renderVom(scene);
expect(rendered.text).toContain('@e1 button "收藏"');
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
@@ -1840,9 +1871,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 20,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1890,10 +1920,10 @@ describe("buildVomScene", () => {
],
};
const scene = buildVomScene(axNodes, captured);
const scene = buildTestScene(axNodes, captured);
expect(scene.activeScopeBlocks).toEqual([
{
triggerId: 20,
triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
label: "Reviews (12)",
lines: ["Jane - ear cups are small", "Bob - great sound"],
},
@@ -1917,11 +1947,10 @@ describe("buildVomScene", () => {
backendDOMNodeId: 20,
},
];
const scene = buildVomScene(
const scene = buildTestScene(
axNodes,
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -1954,7 +1983,11 @@ describe("buildVomScene", () => {
);
expect(scene.surfaces).toEqual([
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
{
triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
triggerAction: "hover",
subItems: ["Shoes", "Bags"],
},
]);
expect(renderVom(scene).text).toContain('@e1 button "Products" [hover first: Shoes | Bags]');
});
@@ -1975,11 +2008,10 @@ describe("buildVomScene", () => {
backendDOMNodeId: 21,
},
];
const scene = buildVomScene(
const scene = buildTestScene(
axNodes,
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -2040,11 +2072,10 @@ describe("buildVomScene", () => {
backendDOMNodeId: 21,
},
];
const scene = buildVomScene(
const scene = buildTestScene(
axNodes,
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -2089,7 +2120,11 @@ describe("buildVomScene", () => {
);
expect(scene.surfaces).toEqual([
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile"] },
{
triggerId: scene.nodes.find((node) => node.backendNodeId === 21)!.id,
triggerAction: "hover",
subItems: ["My profile"],
},
]);
});
@@ -2109,11 +2144,10 @@ describe("buildVomScene", () => {
backendDOMNodeId: 21,
},
];
const scene = buildVomScene(
const scene = buildTestScene(
axNodes,
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -2164,7 +2198,11 @@ describe("buildVomScene", () => {
);
expect(scene.surfaces).toEqual([
{ triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
{
triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
triggerAction: "hover",
subItems: ["My profile", "Sign out"],
},
]);
expect(renderVom(scene).text).toContain(
'@e1 button "image" [hover first: My profile | Sign out]',
@@ -2187,11 +2225,10 @@ describe("buildVomScene", () => {
backendDOMNodeId: 42,
},
];
const scene = buildVomScene(
const scene = buildTestScene(
axNodes,
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -2261,9 +2298,8 @@ describe("buildVomScene", () => {
properties: [{ name: "inputType", value: { value: "credit-card" } }],
},
];
const scene = buildVomScene(axNodes, {
const scene = buildTestScene(axNodes, {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -2310,7 +2346,7 @@ describe("buildVomScene", () => {
],
});
expect(scene.nodes.find((node) => node.id === 20)).toEqual(
expect(scene.nodes.find((node) => node.backendNodeId === 20)).toEqual(
expect.objectContaining({
name: "Products [expanded]",
attrs: expect.objectContaining({
@@ -2320,13 +2356,17 @@ describe("buildVomScene", () => {
}),
);
expect(scene.activeScopeBlocks).toEqual([
{ triggerId: 20, label: "Products [expanded]", lines: ["Shoes Bags"] },
{
triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
label: "Products [expanded]",
lines: ["Shoes Bags"],
},
]);
expect(scene.nodes.find((node) => node.id === 30)?.sensitive).toBe(true);
expect(scene.nodes.find((node) => node.backendNodeId === 30)?.sensitive).toBe(true);
});
it("aggregates AX virtual text into unnamed structural nodes", () => {
const scene = buildVomScene(
const scene = buildTestScene(
[
{
nodeId: "1",
@@ -2350,7 +2390,6 @@ describe("buildVomScene", () => {
],
{
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -2377,13 +2416,12 @@ describe("buildVomScene", () => {
},
);
expect(scene.nodes.find((node) => node.id === 20)?.name).toBe("Inline only text");
expect(scene.nodes.find((node) => node.backendNodeId === 20)?.name).toBe("Inline only text");
});
it("marks captured dialog elements as modal without AX role or aria-modal", () => {
const scene = buildVomScene([], {
const scene = buildTestScene([], {
viewport: { width: 640, height: 480 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set(),
nodes: [
{
@@ -2401,7 +2439,7 @@ describe("buildVomScene", () => {
expect(scene.nodes[0]).toEqual(
expect.objectContaining({
id: 50,
backendNodeId: 50,
modal: true,
}),
);
@@ -2430,9 +2468,8 @@ describe("buildVomScene", () => {
backendDOMNodeId: 202,
},
];
const captured: CapturedViewModel = {
const captured: SceneFixture = {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set([200, 201, 202]),
nodes: [
{
@@ -2448,9 +2485,9 @@ describe("buildVomScene", () => {
],
};
const scene = buildVomScene(axNodes, captured);
expect(scene.nodes.find((n) => n.id === 202)).toBeUndefined();
expect(scene.nodes.find((n) => n.id === 2)).toEqual(
const scene = buildTestScene(axNodes, captured);
expect(scene.nodes.find((n) => n.backendNodeId === 202)).toBeUndefined();
expect(scene.nodes.find((n) => n.backendNodeId === 2)).toEqual(
expect.objectContaining({ role: "button", name: "Real Button" }),
);
});
@@ -2478,13 +2515,12 @@ describe("buildVomScene", () => {
backendDOMNodeId: 203,
},
];
const scene = buildVomScene(axNodes, {
const scene = buildTestScene(axNodes, {
viewport: { width: 1000, height: 800 },
iframeNodes: new Map(),
excludedBackendNodeIds: new Set([200]),
nodes: [],
});
expect(scene.nodes.find((n) => n.id === 203)).toBeUndefined();
expect(scene.nodes.find((n) => n.backendNodeId === 203)).toBeUndefined();
});
});
@@ -2589,6 +2625,7 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0],
nodeName: [i("body"), i("button")],
@@ -2680,6 +2717,7 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0],
nodeName: [i("body"), i("button")],
@@ -2774,6 +2812,7 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0],
nodeName: [i("body"), i("button")],
@@ -3038,6 +3077,7 @@ describe("handleSnapshot", () => {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 2],
nodeName: [i("html"), i("body"), i("div"), i("input")],
@@ -3095,6 +3135,7 @@ describe("handleSnapshot", () => {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 1, 1, 1],
nodeName: [i("html"), i("body"), i("input"), i("input"), i("input"), i("input")],
@@ -3240,12 +3281,17 @@ describe("handleSnapshot", () => {
if (method === "Runtime.releaseObject") return {};
}
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
if (method === "DOMSnapshot.captureSnapshot")
return { ...snapshot, documents: [snapshot.documents[0]] };
if (method === "Accessibility.getFullAXTree") return { nodes: mainAx };
throw new Error(`unexpected root CDP method ${method}`);
});
const sendToTarget = vi.fn(async (_target, method: string) => {
if (method === "Accessibility.enable") return {};
if (method === "Page.getLayoutMetrics" && ownerGeometry !== "unavailable")
return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } };
if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
if (method === "DOMSnapshot.captureSnapshot")
return { ...snapshot, documents: [snapshot.documents[1]] };
if (method === "Accessibility.getFullAXTree") return { nodes: childAx };
throw new Error(`unexpected child CDP method ${method}`);
});
@@ -3346,6 +3392,7 @@ describe("handleSnapshot", () => {
strings,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 1, 3],
nodeName: [index("html"), index("body"), index("button"), index("div"), index("#text")],
@@ -3670,6 +3717,7 @@ describe("handleSnapshot", () => {
strings: S,
documents: [
{
frameId: "root",
scrollOffsetX: 0,
scrollOffsetY: 0,
nodes: {
@@ -3696,6 +3744,7 @@ describe("handleSnapshot", () => {
},
// documents[1]: cross-origin login form
{
frameId: "child",
scrollOffsetX: 0,
scrollOffsetY: 0,
nodes: {
@@ -3874,6 +3923,7 @@ describe("handleSnapshot", () => {
strings: S,
documents: [
{
frameId: "root",
nodes: {
parentIndex: [-1, 0, 1, 0, 3, 4, 5],
nodeName: [