mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(vom): fix name lost
This commit is contained in:
1 parent
41cb576256
commit
b5062ac19b
15 files changed
+1588
-304
No files matched your search
@@ -0,0 +1,161 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { CdpRunner } from "../../shared";
|
||||
import type { CapturedNode } from "../capture";
|
||||
import type { FrameDocument, FrameOwnedAxNode } from "../frame-document";
|
||||
import { probeTooltipNames } from "../name-enrichment";
|
||||
import { buildSemanticGraph, resolveSemanticGraph } from "../semantic-graph";
|
||||
import { stableIdentifierName } from "../semantic-graph/name-evidence";
|
||||
import { frameBackendKey } from "../semantic-graph/types";
|
||||
|
||||
function document(
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
accessibleName?: string,
|
||||
frameId = "child",
|
||||
x = 10,
|
||||
): FrameDocument<FrameOwnedAxNode> {
|
||||
const node: CapturedNode = {
|
||||
backendNodeId,
|
||||
parentBackendNodeId: null,
|
||||
frameId,
|
||||
tag: "button",
|
||||
attrs: {
|
||||
id: "toolInsertRecord",
|
||||
"data-test-id": "toolInsertRecord",
|
||||
...(accessibleName ? { "aria-label": accessibleName } : {}),
|
||||
},
|
||||
rect: { x, y: 20, w: 40, h: 30 },
|
||||
localRect: { x: 2, y: 3, w: 40, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
};
|
||||
return {
|
||||
frameId,
|
||||
parentFrameId: "main",
|
||||
contextScopeId: "child",
|
||||
target,
|
||||
axNodes: [],
|
||||
domNodes: [node],
|
||||
url: "https://example.com/frame",
|
||||
};
|
||||
}
|
||||
|
||||
function semantics(documents: FrameDocument<FrameOwnedAxNode>[]) {
|
||||
return resolveSemanticGraph(
|
||||
buildSemanticGraph({
|
||||
documents,
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: documents[0]?.frameId,
|
||||
}),
|
||||
{ identifierFallback: false },
|
||||
);
|
||||
}
|
||||
|
||||
describe("tooltip name enrichment", () => {
|
||||
it("uses the owning frame target and upgrades an identifier name from a tooltip", async () => {
|
||||
const target = { tabId: 7, sessionId: "oopif-session" };
|
||||
let hovered = false;
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method, params) => {
|
||||
if (method === "Input.dispatchMouseEvent") {
|
||||
hovered = (params as { x?: number }).x === 30;
|
||||
}
|
||||
return {};
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "Page.createIsolatedWorld") return { executionContextId: 11 };
|
||||
if (method === "DOM.resolveNode") return { object: { objectId: "button-501" } };
|
||||
if (method === "Runtime.callFunctionOn") {
|
||||
return { result: { value: hovered ? ["插入行"] : [] } };
|
||||
}
|
||||
return {};
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
|
||||
const documents = [document(target, 501)];
|
||||
const graph = semantics(documents);
|
||||
const names = await probeTooltipNames(cdp, 7, documents, graph);
|
||||
|
||||
expect(names.get(frameBackendKey("child", 501))).toBe("插入行");
|
||||
expect(cdp.sendToTarget).toHaveBeenCalledWith(
|
||||
target,
|
||||
"Page.createIsolatedWorld",
|
||||
expect.objectContaining({ frameId: "child" }),
|
||||
);
|
||||
|
||||
const sendsBeforeCache = vi.mocked(cdp.send).mock.calls.length;
|
||||
const targetSendsBeforeCache = vi.mocked(
|
||||
cdp.sendToTarget as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
).mock.calls.length;
|
||||
const cached = await probeTooltipNames(cdp, 7, documents, graph);
|
||||
expect(cached.get(frameBackendKey("child", 501))).toBe("插入行");
|
||||
expect(cdp.send).toHaveBeenCalledTimes(sendsBeforeCache);
|
||||
expect(cdp.sendToTarget).toHaveBeenCalledTimes(targetSendsBeforeCache);
|
||||
});
|
||||
|
||||
it("does not probe controls that already have a higher-confidence name", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async () => ({ executionContextId: 12 })) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
|
||||
const documents = [document({ tabId: 8 }, 601, "Add a new record")];
|
||||
const names = await probeTooltipNames(cdp, 8, documents, semantics(documents));
|
||||
|
||||
expect(names.size).toBe(0);
|
||||
expect(cdp.send).not.toHaveBeenCalled();
|
||||
expect(cdp.sendToTarget).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps tooltip evidence isolated across sibling frame sessions", async () => {
|
||||
let hoveredX = -10;
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method, params) => {
|
||||
if (method === "Input.dispatchMouseEvent") {
|
||||
hoveredX = (params as { x: number }).x;
|
||||
}
|
||||
return {};
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (target, method, params) => {
|
||||
if (method === "Page.createIsolatedWorld") {
|
||||
return { executionContextId: target.sessionId === "left-session" ? 21 : 22 };
|
||||
}
|
||||
if (method === "DOM.resolveNode") {
|
||||
return { object: { objectId: target.sessionId === "left-session" ? "left" : "right" } };
|
||||
}
|
||||
if (method === "Runtime.callFunctionOn") {
|
||||
const objectId = (params as { objectId: string }).objectId;
|
||||
const value =
|
||||
objectId === "left" && hoveredX === 30
|
||||
? ["Left action"]
|
||||
: objectId === "right" && hoveredX === 130
|
||||
? ["Right action"]
|
||||
: [];
|
||||
return { result: { value } };
|
||||
}
|
||||
return {};
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
const documents = [
|
||||
document({ tabId: 9, sessionId: "left-session" }, 701, undefined, "left", 10),
|
||||
document({ tabId: 9, sessionId: "right-session" }, 701, undefined, "right", 110),
|
||||
];
|
||||
|
||||
const names = await probeTooltipNames(cdp, 9, documents, semantics(documents));
|
||||
|
||||
expect(names.get(frameBackendKey("left", 701))).toBe("Left action");
|
||||
expect(names.get(frameBackendKey("right", 701))).toBe("Right action");
|
||||
});
|
||||
});
|
||||
|
||||
describe("stable identifier evidence", () => {
|
||||
it("accepts readable action identifiers and rejects framework-generated ids", () => {
|
||||
expect(stableIdentifierName({ id: "toolInsertRecord" })).toBe("Insert record");
|
||||
expect(stableIdentifierName({ id: "reactivateAccount" })).toBe("Reactivate account");
|
||||
expect(stableIdentifierName({ id: "headlessui-menu-button-r1" })).toBeUndefined();
|
||||
expect(stableIdentifierName({ id: "7f15a3bc91" })).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -141,6 +141,64 @@ describe("semantic VOM graph", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("uses stable identifiers only as a final name fallback", () => {
|
||||
const base = document(
|
||||
"main",
|
||||
[
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["fallback", "labelled", "container"],
|
||||
},
|
||||
{
|
||||
nodeId: "fallback",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 2,
|
||||
role: { type: "role", value: "button" },
|
||||
},
|
||||
{
|
||||
nodeId: "labelled",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 3,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Localized name" },
|
||||
},
|
||||
{
|
||||
nodeId: "container",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 4,
|
||||
role: { type: "role", value: "generic" },
|
||||
},
|
||||
],
|
||||
[
|
||||
dom(1, null, "body"),
|
||||
dom(2, 1, "button", { "data-test-id": "toolInsertRecord" }),
|
||||
dom(3, 1, "button", { id: "toolIgnoredFallback" }),
|
||||
dom(4, 1, "div", { id: "toolbarContainer" }),
|
||||
],
|
||||
);
|
||||
const staticScene = buildSemanticVomScene({
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: "main",
|
||||
documents: [base],
|
||||
});
|
||||
const enrichedScene = buildSemanticVomScene({
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: "main",
|
||||
documents: [base],
|
||||
supplementalNames: new Map([["main\u00002", "插入行"]]),
|
||||
});
|
||||
|
||||
expect(staticScene.nodes.find((node) => node.backendNodeId === 2)?.name).toBe("Insert record");
|
||||
expect(staticScene.nodes.find((node) => node.backendNodeId === 3)?.name).toBe("Localized name");
|
||||
expect(staticScene.nodes.find((node) => node.backendNodeId === 4)?.name).toBeUndefined();
|
||||
expect(enrichedScene.nodes.find((node) => node.backendNodeId === 2)?.name).toBe("插入行");
|
||||
expect(enrichedScene.nodes.find((node) => node.backendNodeId === 3)?.name).toBe(
|
||||
"Localized name",
|
||||
);
|
||||
});
|
||||
|
||||
it("isolates equal backend node ids in sibling frames", () => {
|
||||
const scene = buildSemanticVomScene({
|
||||
viewport: { width: 1000, height: 800 },
|
||||
@@ -212,4 +270,279 @@ describe("semantic VOM graph", () => {
|
||||
["frame-b", 5],
|
||||
]);
|
||||
});
|
||||
|
||||
it("lets explicit DOM structure override ignored AX metadata", () => {
|
||||
const scene = buildSemanticVomScene({
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: "main",
|
||||
documents: [
|
||||
document(
|
||||
"main",
|
||||
[
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["toolbar"],
|
||||
},
|
||||
{
|
||||
nodeId: "toolbar",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 2,
|
||||
ignored: true,
|
||||
role: { type: "role", value: "generic" },
|
||||
childIds: ["wrapper"],
|
||||
},
|
||||
{
|
||||
nodeId: "wrapper",
|
||||
parentId: "toolbar",
|
||||
backendDOMNodeId: 5,
|
||||
role: { type: "role", value: "generic" },
|
||||
childIds: ["bold", "italic"],
|
||||
},
|
||||
{
|
||||
nodeId: "bold",
|
||||
parentId: "wrapper",
|
||||
backendDOMNodeId: 3,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Bold" },
|
||||
},
|
||||
{
|
||||
nodeId: "italic",
|
||||
parentId: "wrapper",
|
||||
backendDOMNodeId: 4,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Italic" },
|
||||
},
|
||||
],
|
||||
[
|
||||
dom(1, null, "body"),
|
||||
dom(2, 1, "div", { role: "toolbar" }),
|
||||
dom(5, 2, "div"),
|
||||
dom(3, 5, "button"),
|
||||
dom(4, 5, "button"),
|
||||
],
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
const toolbar = scene.nodes.find((node) => node.backendNodeId === 2);
|
||||
const buttons = scene.nodes.filter((node) => [3, 4].includes(node.backendNodeId ?? -1));
|
||||
expect(toolbar).toEqual(expect.objectContaining({ role: "toolbar" }));
|
||||
expect(toolbar && isVomReferenceNode(toolbar)).toBe(false);
|
||||
expect(scene.nodes.some((node) => node.role === "group")).toBe(false);
|
||||
expect(buttons.map((node) => node.parentId)).toEqual([toolbar?.id, toolbar?.id]);
|
||||
});
|
||||
|
||||
it("infers an unnamed group only from independent interaction branches", () => {
|
||||
const scene = buildSemanticVomScene({
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: "main",
|
||||
documents: [
|
||||
document(
|
||||
"main",
|
||||
[
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["wrapper"],
|
||||
},
|
||||
{
|
||||
nodeId: "wrapper",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 2,
|
||||
role: { type: "role", value: "generic" },
|
||||
childIds: ["first", "second"],
|
||||
},
|
||||
{
|
||||
nodeId: "first",
|
||||
parentId: "wrapper",
|
||||
backendDOMNodeId: 3,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "First" },
|
||||
},
|
||||
{
|
||||
nodeId: "second",
|
||||
parentId: "wrapper",
|
||||
backendDOMNodeId: 4,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Second" },
|
||||
},
|
||||
],
|
||||
[dom(1, null, "body"), dom(2, 1, "div"), dom(3, 2, "button"), dom(4, 2, "button")],
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
const group = scene.nodes.find((node) => node.backendNodeId === 2);
|
||||
const buttons = scene.nodes.filter((node) => [3, 4].includes(node.backendNodeId ?? -1));
|
||||
expect(group).toEqual(expect.objectContaining({ role: "group" }));
|
||||
expect(group?.name).toBeUndefined();
|
||||
expect(group && isVomReferenceNode(group)).toBe(false);
|
||||
expect(buttons.map((node) => node.parentId)).toEqual([group?.id, group?.id]);
|
||||
});
|
||||
|
||||
it("keeps uncertain single-branch wrappers flat", () => {
|
||||
const scene = buildSemanticVomScene({
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: "main",
|
||||
documents: [
|
||||
document(
|
||||
"main",
|
||||
[
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["wrapper"],
|
||||
},
|
||||
{
|
||||
nodeId: "wrapper",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 2,
|
||||
role: { type: "role", value: "generic" },
|
||||
childIds: ["button"],
|
||||
},
|
||||
{
|
||||
nodeId: "button",
|
||||
parentId: "wrapper",
|
||||
backendDOMNodeId: 3,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Only action" },
|
||||
},
|
||||
],
|
||||
[dom(1, null, "body"), dom(2, 1, "div"), dom(3, 2, "button")],
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
expect(scene.nodes.some((node) => node.role === "group")).toBe(false);
|
||||
expect(scene.nodes.find((node) => node.backendNodeId === 3)?.parentId).toBe(
|
||||
scene.nodes.find((node) => node.backendNodeId === 1)?.id,
|
||||
);
|
||||
});
|
||||
|
||||
it("does not infer across mixed content boundaries", () => {
|
||||
const scene = buildSemanticVomScene({
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: "main",
|
||||
documents: [
|
||||
document(
|
||||
"main",
|
||||
[
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["wrapper"],
|
||||
},
|
||||
{
|
||||
nodeId: "wrapper",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 2,
|
||||
role: { type: "role", value: "generic" },
|
||||
childIds: ["first", "second", "paragraph"],
|
||||
},
|
||||
{
|
||||
nodeId: "first",
|
||||
parentId: "wrapper",
|
||||
backendDOMNodeId: 3,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "First" },
|
||||
},
|
||||
{
|
||||
nodeId: "second",
|
||||
parentId: "wrapper",
|
||||
backendDOMNodeId: 4,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Second" },
|
||||
},
|
||||
{
|
||||
nodeId: "paragraph",
|
||||
parentId: "wrapper",
|
||||
backendDOMNodeId: 5,
|
||||
role: { type: "role", value: "paragraph" },
|
||||
},
|
||||
],
|
||||
[
|
||||
dom(1, null, "body"),
|
||||
dom(2, 1, "div"),
|
||||
dom(3, 2, "button"),
|
||||
dom(4, 2, "button"),
|
||||
dom(5, 2, "p", {}, "Description"),
|
||||
],
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
expect(scene.nodes.some((node) => node.role === "group")).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps only the innermost reliable inferred group", () => {
|
||||
const scene = buildSemanticVomScene({
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: "main",
|
||||
documents: [
|
||||
document(
|
||||
"main",
|
||||
[
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["outer"],
|
||||
},
|
||||
{
|
||||
nodeId: "outer",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 2,
|
||||
role: { type: "role", value: "generic" },
|
||||
childIds: ["inner", "third"],
|
||||
},
|
||||
{
|
||||
nodeId: "inner",
|
||||
parentId: "outer",
|
||||
backendDOMNodeId: 3,
|
||||
role: { type: "role", value: "generic" },
|
||||
childIds: ["first", "second"],
|
||||
},
|
||||
{
|
||||
nodeId: "first",
|
||||
parentId: "inner",
|
||||
backendDOMNodeId: 4,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "First" },
|
||||
},
|
||||
{
|
||||
nodeId: "second",
|
||||
parentId: "inner",
|
||||
backendDOMNodeId: 5,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Second" },
|
||||
},
|
||||
{
|
||||
nodeId: "third",
|
||||
parentId: "outer",
|
||||
backendDOMNodeId: 6,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Third" },
|
||||
},
|
||||
],
|
||||
[
|
||||
dom(1, null, "body"),
|
||||
dom(2, 1, "div"),
|
||||
dom(3, 2, "div"),
|
||||
dom(4, 3, "button"),
|
||||
dom(5, 3, "button"),
|
||||
dom(6, 2, "button"),
|
||||
],
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
const groups = scene.nodes.filter((node) => node.role === "group");
|
||||
expect(groups.map((node) => node.backendNodeId)).toEqual([3]);
|
||||
expect(scene.nodes.find((node) => node.backendNodeId === 6)?.parentId).not.toBe(groups[0]?.id);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user