mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
refactor(observation): organize capture around document facts
This commit is contained in:
@@ -4,9 +4,9 @@ import { CAPTURE_SUPPRESS, type CaptureSuppressMessage } from "@/lib/capture-sup
|
||||
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "@/lib/overlay-bridge";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import type { CdpRunner } from "@/tools/shared";
|
||||
import type { BuildVomSceneOptions, VomFrameDocument } from "../observation";
|
||||
import {
|
||||
buildFrameVomScene,
|
||||
buildVomScene,
|
||||
type CdpAxNode,
|
||||
captureVomObservation,
|
||||
handleGetHtml,
|
||||
@@ -17,7 +17,35 @@ import {
|
||||
type ScreenshotDeps,
|
||||
stripDataUrlPrefix,
|
||||
} from "../observation";
|
||||
import type { CapturedNode, CapturedViewModel } from "../vom/capture";
|
||||
import type { CapturedNode, CapturedSceneInput } from "../vom/facts";
|
||||
|
||||
interface SceneFixture extends CapturedSceneInput {
|
||||
documents?: VomFrameDocument[];
|
||||
}
|
||||
|
||||
// Only supplies the common root fixture; child documents declare their own ownership.
|
||||
function buildTestScene(
|
||||
axNodes: CdpAxNode[],
|
||||
captured: SceneFixture,
|
||||
options: BuildVomSceneOptions = {},
|
||||
) {
|
||||
const rootFrameId = captured.rootFrameId ?? "root";
|
||||
return buildFrameVomScene(
|
||||
[
|
||||
{
|
||||
frameId: rootFrameId,
|
||||
contextScopeId: rootFrameId,
|
||||
target: { tabId: 7 },
|
||||
url: options.pageUrl,
|
||||
domNodes: captured.nodes,
|
||||
axNodes,
|
||||
},
|
||||
...(captured.documents ?? []),
|
||||
],
|
||||
captured,
|
||||
options,
|
||||
);
|
||||
}
|
||||
|
||||
function fakeAgentWindow(ids: number[]) {
|
||||
let i = 0;
|
||||
@@ -522,10 +550,10 @@ describe("handleScreenshot overlay suppression", () => {
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// buildVomScene
|
||||
// Semantic scene fixtures
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("buildVomScene", () => {
|
||||
describe("buildFrameVomScene", () => {
|
||||
it("merges multiple sibling and nested frames without mixing their targets", () => {
|
||||
const node = (
|
||||
backendNodeId: number,
|
||||
@@ -563,25 +591,10 @@ describe("buildVomScene", () => {
|
||||
node(300, null, "nested", "body"),
|
||||
node(301, 300, "nested", "input", { placeholder: "Nested field" }),
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
const captured: SceneFixture = {
|
||||
viewport: { width: 1200, height: 800 },
|
||||
nodes: mainNodes,
|
||||
iframeNodes: new Map([
|
||||
[10, firstNodes],
|
||||
[20, secondNodes],
|
||||
[210, nestedNodes],
|
||||
]),
|
||||
frameNodes: new Map([
|
||||
["main", mainNodes],
|
||||
["first", firstNodes],
|
||||
["second", secondNodes],
|
||||
["nested", nestedNodes],
|
||||
]),
|
||||
frameOwnerBackendNodeIds: new Map([
|
||||
["first", 10],
|
||||
["second", 20],
|
||||
["nested", 210],
|
||||
]),
|
||||
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
@@ -680,11 +693,10 @@ describe("buildVomScene", () => {
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
const captured: CapturedViewModel = {
|
||||
const captured: SceneFixture = {
|
||||
viewport: { width: 800, height: 600 },
|
||||
nodes: [],
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([["child", [childNode]]]),
|
||||
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
@@ -746,14 +758,10 @@ describe("buildVomScene", () => {
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
const captured: SceneFixture = {
|
||||
viewport: { width: 800, height: 600 },
|
||||
nodes: mainNodes,
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([
|
||||
["main", mainNodes],
|
||||
["child", []],
|
||||
]),
|
||||
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
@@ -829,9 +837,8 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 200,
|
||||
},
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
const captured: SceneFixture = {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
@@ -857,11 +864,11 @@ describe("buildVomScene", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
const scene = buildTestScene(axNodes, captured);
|
||||
expect(scene.viewport).toEqual({ width: 1000, height: 800 });
|
||||
expect(scene.nodes[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
id: 100,
|
||||
backendNodeId: 100,
|
||||
parentId: null,
|
||||
role: "RootWebArea",
|
||||
name: "Example",
|
||||
@@ -873,14 +880,14 @@ describe("buildVomScene", () => {
|
||||
);
|
||||
expect(scene.nodes[1]).toEqual(
|
||||
expect.objectContaining({
|
||||
id: 200,
|
||||
parentId: 100,
|
||||
backendNodeId: 200,
|
||||
parentId: scene.nodes.find((node) => node.backendNodeId === 100)!.id,
|
||||
role: "button",
|
||||
name: "Submit",
|
||||
tag: "button",
|
||||
rect: { x: 20, y: 20, w: 120, h: 40 },
|
||||
domParentId: 100,
|
||||
domAncestorIds: [100],
|
||||
domParentId: scene.nodes.find((node) => node.backendNodeId === 100)!.id,
|
||||
domAncestorIds: [scene.nodes.find((node) => node.backendNodeId === 100)!.id],
|
||||
attrs: {},
|
||||
}),
|
||||
);
|
||||
@@ -909,9 +916,8 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 300,
|
||||
},
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
const captured: SceneFixture = {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
@@ -947,10 +953,10 @@ describe("buildVomScene", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const scene = buildVomScene(axNodes, captured, { pageUrl: "https://app.example.test/home" });
|
||||
const scene = buildTestScene(axNodes, captured, { pageUrl: "https://app.example.test/home" });
|
||||
|
||||
expect(scene.nodes.find((node) => node.id === 200)?.href).toBe("docs.example.org");
|
||||
expect(scene.nodes.find((node) => node.id === 300)?.href).toBeUndefined();
|
||||
expect(scene.nodes.find((node) => node.backendNodeId === 200)?.href).toBe("docs.example.org");
|
||||
expect(scene.nodes.find((node) => node.backendNodeId === 300)?.href).toBeUndefined();
|
||||
});
|
||||
|
||||
it("uses the nearest backend AX ancestor as parentId", () => {
|
||||
@@ -975,9 +981,8 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 30,
|
||||
},
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
const captured: SceneFixture = {
|
||||
viewport: { width: 400, height: 300 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
@@ -1003,10 +1008,12 @@ describe("buildVomScene", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
const scene = buildTestScene(axNodes, captured);
|
||||
|
||||
expect(scene.nodes.map((node) => node.id)).toEqual([10, 30]);
|
||||
expect(scene.nodes.find((node) => node.id === 30)?.parentId).toBe(10);
|
||||
expect(scene.nodes.map((node) => node.backendNodeId)).toEqual([10, 30]);
|
||||
expect(scene.nodes.find((node) => node.backendNodeId === 30)?.parentId).toBe(
|
||||
scene.nodes.find((node) => node.backendNodeId === 10)!.id,
|
||||
);
|
||||
});
|
||||
|
||||
it("maps iframe sub-document controls to VomNodes", () => {
|
||||
@@ -1024,7 +1031,7 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 20,
|
||||
},
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
const captured: SceneFixture = {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
nodes: [
|
||||
{
|
||||
@@ -1048,10 +1055,15 @@ describe("buildVomScene", () => {
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
iframeNodes: new Map([
|
||||
[
|
||||
20,
|
||||
[
|
||||
documents: [
|
||||
{
|
||||
frameId: "child",
|
||||
contextScopeId: "child",
|
||||
parentFrameId: "root",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 7 },
|
||||
axNodes: [],
|
||||
domNodes: [
|
||||
{
|
||||
backendNodeId: 101,
|
||||
parentBackendNodeId: null,
|
||||
@@ -1073,26 +1085,25 @@ describe("buildVomScene", () => {
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
],
|
||||
]),
|
||||
},
|
||||
],
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
|
||||
const iframeControls = buildVomScene(axNodes, captured).nodes.filter((node) =>
|
||||
[101, 102].includes(node.id),
|
||||
);
|
||||
const scene = buildTestScene(axNodes, captured);
|
||||
const iframeControls = scene.nodes.filter((node) => [101, 102].includes(node.backendNodeId!));
|
||||
|
||||
expect(iframeControls).toEqual([
|
||||
expect.objectContaining({
|
||||
id: 101,
|
||||
parentId: 20,
|
||||
backendNodeId: 101,
|
||||
parentId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
|
||||
role: "textbox",
|
||||
name: "请输入手机号",
|
||||
sensitive: false,
|
||||
}),
|
||||
expect.objectContaining({
|
||||
id: 102,
|
||||
parentId: 20,
|
||||
backendNodeId: 102,
|
||||
parentId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
|
||||
role: "textbox",
|
||||
name: "密码",
|
||||
sensitive: true,
|
||||
@@ -1113,11 +1124,9 @@ describe("buildVomScene", () => {
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "Iframe" },
|
||||
backendDOMNodeId: 20,
|
||||
childIds: ["3"],
|
||||
},
|
||||
{
|
||||
nodeId: "3",
|
||||
parentId: "2",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "x", value: "输入密码" },
|
||||
value: { value: "iframe-secret" },
|
||||
@@ -1125,7 +1134,7 @@ describe("buildVomScene", () => {
|
||||
properties: [{ name: "inputType", value: { value: "password" } }],
|
||||
},
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
const captured: SceneFixture = {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
nodes: [
|
||||
{
|
||||
@@ -1149,10 +1158,15 @@ describe("buildVomScene", () => {
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
iframeNodes: new Map([
|
||||
[
|
||||
20,
|
||||
[
|
||||
documents: [
|
||||
{
|
||||
frameId: "child",
|
||||
contextScopeId: "child",
|
||||
parentFrameId: "root",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 7 },
|
||||
axNodes: [axNodes[2]],
|
||||
domNodes: [
|
||||
{
|
||||
backendNodeId: 102,
|
||||
parentBackendNodeId: null,
|
||||
@@ -1164,25 +1178,27 @@ describe("buildVomScene", () => {
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
],
|
||||
]),
|
||||
},
|
||||
],
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
|
||||
const passwordNodes = buildVomScene(axNodes, captured).nodes.filter(
|
||||
const passwordNodes = buildTestScene(axNodes.slice(0, 2), captured).nodes.filter(
|
||||
(node) => node.role === "textbox" && node.name === "输入密码",
|
||||
);
|
||||
|
||||
expect(passwordNodes).toHaveLength(1);
|
||||
expect(passwordNodes[0]).toEqual(expect.objectContaining({ id: 202, sensitive: true }));
|
||||
expect(passwordNodes[0]).toEqual(
|
||||
expect.objectContaining({ backendNodeId: 202, sensitive: true }),
|
||||
);
|
||||
expect(passwordNodes[0].value).toBeUndefined();
|
||||
const rendered = renderVom(buildVomScene(axNodes, captured)).text;
|
||||
const rendered = renderVom(buildTestScene(axNodes.slice(0, 2), captured)).text;
|
||||
expect(rendered).toContain('textbox "输入密码" [filled] ="•••"');
|
||||
expect(rendered).not.toContain("iframe-secret");
|
||||
});
|
||||
|
||||
it("keeps unnamed iframe controls but skips unnamed iframe links", () => {
|
||||
const scene = buildVomScene(
|
||||
const scene = buildTestScene(
|
||||
[
|
||||
{
|
||||
nodeId: "1",
|
||||
@@ -1221,10 +1237,15 @@ describe("buildVomScene", () => {
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
iframeNodes: new Map([
|
||||
[
|
||||
20,
|
||||
[
|
||||
documents: [
|
||||
{
|
||||
frameId: "child",
|
||||
contextScopeId: "child",
|
||||
parentFrameId: "root",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 7 },
|
||||
axNodes: [],
|
||||
domNodes: [
|
||||
{
|
||||
backendNodeId: 201,
|
||||
parentBackendNodeId: null,
|
||||
@@ -1246,25 +1267,25 @@ describe("buildVomScene", () => {
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
],
|
||||
]),
|
||||
},
|
||||
],
|
||||
excludedBackendNodeIds: new Set(),
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.nodes.find((node) => node.id === 201)).toEqual(
|
||||
expect(scene.nodes.find((node) => node.backendNodeId === 201)).toEqual(
|
||||
expect.objectContaining({
|
||||
id: 201,
|
||||
parentId: 20,
|
||||
backendNodeId: 201,
|
||||
parentId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
|
||||
role: "button",
|
||||
}),
|
||||
);
|
||||
expect(scene.nodes.find((node) => node.id === 201)).not.toHaveProperty("name");
|
||||
expect(scene.nodes.find((node) => node.id === 202)).toBeUndefined();
|
||||
expect(scene.nodes.find((node) => node.backendNodeId === 201)).not.toHaveProperty("name");
|
||||
expect(scene.nodes.find((node) => node.backendNodeId === 202)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("preserves captured-only iframe anchors for iframe sub-document controls", () => {
|
||||
const scene = buildVomScene(
|
||||
const scene = buildTestScene(
|
||||
[
|
||||
{
|
||||
nodeId: "1",
|
||||
@@ -1297,10 +1318,15 @@ describe("buildVomScene", () => {
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
iframeNodes: new Map<number, CapturedNode[]>([
|
||||
[
|
||||
20,
|
||||
[
|
||||
documents: [
|
||||
{
|
||||
frameId: "child",
|
||||
contextScopeId: "child",
|
||||
parentFrameId: "root",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 7 },
|
||||
axNodes: [],
|
||||
domNodes: [
|
||||
{
|
||||
backendNodeId: 201,
|
||||
parentBackendNodeId: null,
|
||||
@@ -1322,16 +1348,16 @@ describe("buildVomScene", () => {
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
],
|
||||
]),
|
||||
},
|
||||
],
|
||||
excludedBackendNodeIds: new Set(),
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.nodes.find((node) => node.id === 20)).toEqual(
|
||||
expect(scene.nodes.find((node) => node.backendNodeId === 20)).toEqual(
|
||||
expect.objectContaining({
|
||||
id: 20,
|
||||
parentId: 10,
|
||||
backendNodeId: 20,
|
||||
parentId: scene.nodes.find((node) => node.backendNodeId === 10)!.id,
|
||||
role: "Iframe",
|
||||
name: "支付验证",
|
||||
}),
|
||||
@@ -1352,9 +1378,8 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 10,
|
||||
},
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
const captured: SceneFixture = {
|
||||
viewport: { width: 320, height: 240 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
@@ -1375,9 +1400,9 @@ describe("buildVomScene", () => {
|
||||
],
|
||||
};
|
||||
|
||||
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
|
||||
expect(buildTestScene(axNodes, captured).nodes[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
id: 10,
|
||||
backendNodeId: 10,
|
||||
sensitive: true,
|
||||
}),
|
||||
);
|
||||
@@ -1392,9 +1417,8 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 10,
|
||||
},
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
const captured: SceneFixture = {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
@@ -1411,9 +1435,9 @@ describe("buildVomScene", () => {
|
||||
],
|
||||
};
|
||||
|
||||
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
|
||||
expect(buildTestScene(axNodes, captured).nodes[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
id: 10,
|
||||
backendNodeId: 10,
|
||||
role: "textbox",
|
||||
name: "+86",
|
||||
placeholder: "输入手机号",
|
||||
@@ -1431,9 +1455,8 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 10,
|
||||
},
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
const captured: SceneFixture = {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
@@ -1450,8 +1473,13 @@ describe("buildVomScene", () => {
|
||||
],
|
||||
};
|
||||
|
||||
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
|
||||
expect.objectContaining({ id: 10, role: "textbox", name: "Email", value: "a@b.com" }),
|
||||
expect(buildTestScene(axNodes, captured).nodes[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
backendNodeId: 10,
|
||||
role: "textbox",
|
||||
name: "Email",
|
||||
value: "a@b.com",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -1470,9 +1498,8 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 20,
|
||||
},
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
const captured: SceneFixture = {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
@@ -1510,9 +1537,9 @@ describe("buildVomScene", () => {
|
||||
],
|
||||
};
|
||||
|
||||
expect(buildVomScene(axNodes, captured).nodes.find((node) => node.id === 20)).toEqual(
|
||||
expect.objectContaining({ role: "textbox", name: "验证码" }),
|
||||
);
|
||||
expect(
|
||||
buildTestScene(axNodes, captured).nodes.find((node) => node.backendNodeId === 20),
|
||||
).toEqual(expect.objectContaining({ role: "textbox", name: "验证码" }));
|
||||
});
|
||||
|
||||
it("uses preceding AX static text as a form control label", () => {
|
||||
@@ -1536,9 +1563,8 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 20,
|
||||
},
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
const captured: SceneFixture = {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
@@ -1555,13 +1581,13 @@ describe("buildVomScene", () => {
|
||||
],
|
||||
};
|
||||
|
||||
expect(buildVomScene(axNodes, captured).nodes.find((node) => node.id === 20)).toEqual(
|
||||
expect.objectContaining({ role: "textbox", name: "验证码", inputState: "empty" }),
|
||||
);
|
||||
expect(
|
||||
buildTestScene(axNodes, captured).nodes.find((node) => node.backendNodeId === 20),
|
||||
).toEqual(expect.objectContaining({ role: "textbox", name: "验证码", inputState: "empty" }));
|
||||
});
|
||||
|
||||
it("marks runtime default input values without treating them as ordinary filled input", () => {
|
||||
const scene = buildVomScene(
|
||||
const scene = buildTestScene(
|
||||
[
|
||||
{
|
||||
nodeId: "1",
|
||||
@@ -1571,7 +1597,6 @@ describe("buildVomScene", () => {
|
||||
],
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
@@ -1615,9 +1640,8 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 20,
|
||||
},
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
const captured: SceneFixture = {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
@@ -1667,9 +1691,14 @@ describe("buildVomScene", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
expect(scene.nodes.find((n) => n.id === 20)).toEqual(
|
||||
expect.objectContaining({ id: 20, role: "button", name: "close", cursor: "pointer" }),
|
||||
const scene = buildTestScene(axNodes, captured);
|
||||
expect(scene.nodes.find((n) => n.backendNodeId === 20)).toEqual(
|
||||
expect.objectContaining({
|
||||
backendNodeId: 20,
|
||||
role: "button",
|
||||
name: "close",
|
||||
cursor: "pointer",
|
||||
}),
|
||||
);
|
||||
const rendered = renderVom(scene);
|
||||
expect(rendered.text).toContain('@e1 button "close"');
|
||||
@@ -1703,9 +1732,8 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 30,
|
||||
},
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
const captured: SceneFixture = {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
@@ -1744,9 +1772,9 @@ describe("buildVomScene", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
expect(scene.nodes.find((n) => n.id === 20)).toBeUndefined();
|
||||
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("link");
|
||||
const scene = buildTestScene(axNodes, captured);
|
||||
expect(scene.nodes.find((n) => n.backendNodeId === 20)).toBeUndefined();
|
||||
expect(scene.nodes.find((n) => n.backendNodeId === 30)?.role).toBe("link");
|
||||
});
|
||||
|
||||
it("promotes only the outermost clickable in a nested pointer chain", () => {
|
||||
@@ -1771,9 +1799,8 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 30,
|
||||
},
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
const captured: SceneFixture = {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
@@ -1812,11 +1839,15 @@ describe("buildVomScene", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
expect(scene.nodes.find((n) => n.id === 20)).toEqual(
|
||||
expect.objectContaining({ id: 20, role: "button", attrs: { "aria-label": "收藏" } }),
|
||||
const scene = buildTestScene(axNodes, captured);
|
||||
expect(scene.nodes.find((n) => n.backendNodeId === 20)).toEqual(
|
||||
expect.objectContaining({
|
||||
backendNodeId: 20,
|
||||
role: "button",
|
||||
attrs: { "aria-label": "收藏" },
|
||||
}),
|
||||
);
|
||||
expect(scene.nodes.find((n) => n.id === 30)).toBeUndefined();
|
||||
expect(scene.nodes.find((n) => n.backendNodeId === 30)).toBeUndefined();
|
||||
const rendered = renderVom(scene);
|
||||
expect(rendered.text).toContain('@e1 button "收藏"');
|
||||
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
|
||||
@@ -1840,9 +1871,8 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 20,
|
||||
},
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
const captured: SceneFixture = {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
@@ -1890,10 +1920,10 @@ describe("buildVomScene", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
const scene = buildTestScene(axNodes, captured);
|
||||
expect(scene.activeScopeBlocks).toEqual([
|
||||
{
|
||||
triggerId: 20,
|
||||
triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
|
||||
label: "Reviews (12)",
|
||||
lines: ["Jane - ear cups are small", "Bob - great sound"],
|
||||
},
|
||||
@@ -1917,11 +1947,10 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 20,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(
|
||||
const scene = buildTestScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
@@ -1954,7 +1983,11 @@ describe("buildVomScene", () => {
|
||||
);
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
{
|
||||
triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
|
||||
triggerAction: "hover",
|
||||
subItems: ["Shoes", "Bags"],
|
||||
},
|
||||
]);
|
||||
expect(renderVom(scene).text).toContain('@e1 button "Products" [hover first: Shoes | Bags]');
|
||||
});
|
||||
@@ -1975,11 +2008,10 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(
|
||||
const scene = buildTestScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
@@ -2040,11 +2072,10 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(
|
||||
const scene = buildTestScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
@@ -2089,7 +2120,11 @@ describe("buildVomScene", () => {
|
||||
);
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile"] },
|
||||
{
|
||||
triggerId: scene.nodes.find((node) => node.backendNodeId === 21)!.id,
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile"],
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -2109,11 +2144,10 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(
|
||||
const scene = buildTestScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
@@ -2164,7 +2198,11 @@ describe("buildVomScene", () => {
|
||||
);
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
|
||||
{
|
||||
triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile", "Sign out"],
|
||||
},
|
||||
]);
|
||||
expect(renderVom(scene).text).toContain(
|
||||
'@e1 button "image" [hover first: My profile | Sign out]',
|
||||
@@ -2187,11 +2225,10 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 42,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(
|
||||
const scene = buildTestScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
@@ -2261,9 +2298,8 @@ describe("buildVomScene", () => {
|
||||
properties: [{ name: "inputType", value: { value: "credit-card" } }],
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
const scene = buildTestScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
@@ -2310,7 +2346,7 @@ describe("buildVomScene", () => {
|
||||
],
|
||||
});
|
||||
|
||||
expect(scene.nodes.find((node) => node.id === 20)).toEqual(
|
||||
expect(scene.nodes.find((node) => node.backendNodeId === 20)).toEqual(
|
||||
expect.objectContaining({
|
||||
name: "Products [expanded]",
|
||||
attrs: expect.objectContaining({
|
||||
@@ -2320,13 +2356,17 @@ describe("buildVomScene", () => {
|
||||
}),
|
||||
);
|
||||
expect(scene.activeScopeBlocks).toEqual([
|
||||
{ triggerId: 20, label: "Products [expanded]", lines: ["Shoes Bags"] },
|
||||
{
|
||||
triggerId: scene.nodes.find((node) => node.backendNodeId === 20)!.id,
|
||||
label: "Products [expanded]",
|
||||
lines: ["Shoes Bags"],
|
||||
},
|
||||
]);
|
||||
expect(scene.nodes.find((node) => node.id === 30)?.sensitive).toBe(true);
|
||||
expect(scene.nodes.find((node) => node.backendNodeId === 30)?.sensitive).toBe(true);
|
||||
});
|
||||
|
||||
it("aggregates AX virtual text into unnamed structural nodes", () => {
|
||||
const scene = buildVomScene(
|
||||
const scene = buildTestScene(
|
||||
[
|
||||
{
|
||||
nodeId: "1",
|
||||
@@ -2350,7 +2390,6 @@ describe("buildVomScene", () => {
|
||||
],
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
@@ -2377,13 +2416,12 @@ describe("buildVomScene", () => {
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.nodes.find((node) => node.id === 20)?.name).toBe("Inline only text");
|
||||
expect(scene.nodes.find((node) => node.backendNodeId === 20)?.name).toBe("Inline only text");
|
||||
});
|
||||
|
||||
it("marks captured dialog elements as modal without AX role or aria-modal", () => {
|
||||
const scene = buildVomScene([], {
|
||||
const scene = buildTestScene([], {
|
||||
viewport: { width: 640, height: 480 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
@@ -2401,7 +2439,7 @@ describe("buildVomScene", () => {
|
||||
|
||||
expect(scene.nodes[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
id: 50,
|
||||
backendNodeId: 50,
|
||||
modal: true,
|
||||
}),
|
||||
);
|
||||
@@ -2430,9 +2468,8 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 202,
|
||||
},
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
const captured: SceneFixture = {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set([200, 201, 202]),
|
||||
nodes: [
|
||||
{
|
||||
@@ -2448,9 +2485,9 @@ describe("buildVomScene", () => {
|
||||
],
|
||||
};
|
||||
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
expect(scene.nodes.find((n) => n.id === 202)).toBeUndefined();
|
||||
expect(scene.nodes.find((n) => n.id === 2)).toEqual(
|
||||
const scene = buildTestScene(axNodes, captured);
|
||||
expect(scene.nodes.find((n) => n.backendNodeId === 202)).toBeUndefined();
|
||||
expect(scene.nodes.find((n) => n.backendNodeId === 2)).toEqual(
|
||||
expect.objectContaining({ role: "button", name: "Real Button" }),
|
||||
);
|
||||
});
|
||||
@@ -2478,13 +2515,12 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 203,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
const scene = buildTestScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set([200]),
|
||||
nodes: [],
|
||||
});
|
||||
expect(scene.nodes.find((n) => n.id === 203)).toBeUndefined();
|
||||
expect(scene.nodes.find((n) => n.backendNodeId === 203)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2589,6 +2625,7 @@ describe("handleSnapshot", () => {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("button")],
|
||||
@@ -2680,6 +2717,7 @@ describe("handleSnapshot", () => {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("button")],
|
||||
@@ -2774,6 +2812,7 @@ describe("handleSnapshot", () => {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("button")],
|
||||
@@ -3038,6 +3077,7 @@ describe("handleSnapshot", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 2],
|
||||
nodeName: [i("html"), i("body"), i("div"), i("input")],
|
||||
@@ -3095,6 +3135,7 @@ describe("handleSnapshot", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 1, 1, 1],
|
||||
nodeName: [i("html"), i("body"), i("input"), i("input"), i("input"), i("input")],
|
||||
@@ -3240,12 +3281,17 @@ describe("handleSnapshot", () => {
|
||||
if (method === "Runtime.releaseObject") return {};
|
||||
}
|
||||
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "DOMSnapshot.captureSnapshot")
|
||||
return { ...snapshot, documents: [snapshot.documents[0]] };
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: mainAx };
|
||||
throw new Error(`unexpected root CDP method ${method}`);
|
||||
});
|
||||
const sendToTarget = vi.fn(async (_target, method: string) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Page.getLayoutMetrics" && ownerGeometry !== "unavailable")
|
||||
return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } };
|
||||
if (method === "Accessibility.enable" || method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot")
|
||||
return { ...snapshot, documents: [snapshot.documents[1]] };
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: childAx };
|
||||
throw new Error(`unexpected child CDP method ${method}`);
|
||||
});
|
||||
@@ -3346,6 +3392,7 @@ describe("handleSnapshot", () => {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 1, 3],
|
||||
nodeName: [index("html"), index("body"), index("button"), index("div"), index("#text")],
|
||||
@@ -3670,6 +3717,7 @@ describe("handleSnapshot", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
@@ -3696,6 +3744,7 @@ describe("handleSnapshot", () => {
|
||||
},
|
||||
// documents[1]: cross-origin login form
|
||||
{
|
||||
frameId: "child",
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
@@ -3874,6 +3923,7 @@ describe("handleSnapshot", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "root",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 0, 3, 4, 5],
|
||||
nodeName: [
|
||||
|
||||
Reference in New Issue
Block a user