mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(vom): function refactor
This commit is contained in:
Ljy-0827
@@ -1656,7 +1656,7 @@ describe("buildVomScene", () => {
|
||||
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
expect(scene.nodes.find((n) => n.id === 20)).toEqual(
|
||||
expect.objectContaining({ id: 20, role: "generic", cursor: "pointer" }),
|
||||
expect.objectContaining({ id: 20, role: "button", name: "close", cursor: "pointer" }),
|
||||
);
|
||||
const rendered = renderVom(scene);
|
||||
expect(rendered.text).toContain('@e1 button "close"');
|
||||
@@ -1732,7 +1732,7 @@ describe("buildVomScene", () => {
|
||||
};
|
||||
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
expect(scene.nodes.find((n) => n.id === 20)?.role).toBe("generic");
|
||||
expect(scene.nodes.find((n) => n.id === 20)).toBeUndefined();
|
||||
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("link");
|
||||
});
|
||||
|
||||
@@ -1801,9 +1801,9 @@ describe("buildVomScene", () => {
|
||||
|
||||
const scene = buildVomScene(axNodes, captured);
|
||||
expect(scene.nodes.find((n) => n.id === 20)).toEqual(
|
||||
expect.objectContaining({ id: 20, role: "generic", attrs: { "aria-label": "收藏" } }),
|
||||
expect.objectContaining({ id: 20, role: "button", attrs: { "aria-label": "收藏" } }),
|
||||
);
|
||||
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic");
|
||||
expect(scene.nodes.find((n) => n.id === 30)).toBeUndefined();
|
||||
const rendered = renderVom(scene);
|
||||
expect(rendered.text).toContain('@e1 button "收藏"');
|
||||
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
|
||||
@@ -2131,7 +2131,7 @@ describe("buildVomScene", () => {
|
||||
});
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
|
||||
{ triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
|
||||
]);
|
||||
expect(renderVom(scene).text).toContain(
|
||||
'@e1 button "image" [hover first: My profile | Sign out]',
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -132,4 +132,67 @@ describe("captureFrameData", () => {
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps OOPIF semantics when viewport projection is unavailable", async () => {
|
||||
const owner = { ...ownerNode(10, 50), rect: null, localRect: null };
|
||||
const captured: CapturedViewModel = {
|
||||
nodes: [owner],
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([["main", [owner]]]),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
if (method === "DOM.getBoxModel") throw new Error("owner geometry unavailable");
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
|
||||
}
|
||||
throw new Error(`unexpected root ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") throw new Error("viewport unavailable");
|
||||
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return childSnapshot("child", 101);
|
||||
if (method === "Accessibility.getFullAXTree") {
|
||||
return {
|
||||
nodes: [
|
||||
{
|
||||
nodeId: "button",
|
||||
backendDOMNodeId: 101,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Continue" },
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const documents = await captureFrameData(cdp, 4, captured);
|
||||
const childDocument = documents.find((document) => document.frameId === "child");
|
||||
|
||||
expect(childDocument?.axNodes).toEqual([
|
||||
expect.objectContaining({ backendDOMNodeId: 101, frameId: "child" }),
|
||||
]);
|
||||
expect(childDocument?.domNodes.find((node) => node.backendNodeId === 101)).toEqual(
|
||||
expect.objectContaining({ rect: null, localRect: { x: 10, y: 20, w: 100, h: 40 } }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
import { isVomReferenceNode, renderVom } from "@browser-skill/vom";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { CapturedNode } from "../capture";
|
||||
import type { FrameDocument } from "../frame-document";
|
||||
import { buildSemanticVomScene, type SemanticAxNode } from "../semantic-graph";
|
||||
|
||||
function dom(
|
||||
backendNodeId: number,
|
||||
parentBackendNodeId: number | null,
|
||||
tag: string,
|
||||
attrs: Record<string, string> = {},
|
||||
textContent?: string,
|
||||
): CapturedNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
parentBackendNodeId,
|
||||
tag,
|
||||
attrs,
|
||||
rect: { x: 0, y: 0, w: 100, h: 30 },
|
||||
paintOrder: backendNodeId,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
...(textContent ? { textContent } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function document(
|
||||
frameId: string,
|
||||
axNodes: SemanticAxNode[],
|
||||
domNodes: CapturedNode[],
|
||||
frame?: Pick<FrameDocument<SemanticAxNode>, "parentFrameId" | "ownerBackendNodeId">,
|
||||
): FrameDocument<SemanticAxNode> {
|
||||
return {
|
||||
frameId,
|
||||
contextScopeId: frameId,
|
||||
target: { tabId: 1 },
|
||||
axNodes,
|
||||
domNodes,
|
||||
...frame,
|
||||
};
|
||||
}
|
||||
|
||||
describe("semantic VOM graph", () => {
|
||||
it("preserves backend-less AX structure without making it referenceable", () => {
|
||||
const scene = buildSemanticVomScene({
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: "main",
|
||||
documents: [
|
||||
document(
|
||||
"main",
|
||||
[
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["toolbar"],
|
||||
},
|
||||
{
|
||||
nodeId: "toolbar",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "toolbar" },
|
||||
name: { type: "computedString", value: "Formatting" },
|
||||
childIds: ["bold", "italic"],
|
||||
},
|
||||
{
|
||||
nodeId: "bold",
|
||||
parentId: "toolbar",
|
||||
backendDOMNodeId: 2,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Bold" },
|
||||
},
|
||||
{
|
||||
nodeId: "italic",
|
||||
parentId: "toolbar",
|
||||
backendDOMNodeId: 3,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Italic" },
|
||||
},
|
||||
],
|
||||
[dom(1, null, "body"), dom(2, 1, "button"), dom(3, 1, "button")],
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
const toolbar = scene.nodes.find((node) => node.role === "toolbar");
|
||||
const buttons = scene.nodes.filter((node) => node.role === "button");
|
||||
expect(toolbar).toEqual(expect.objectContaining({ name: "Formatting", referenceable: false }));
|
||||
expect(buttons.map((node) => node.parentId)).toEqual([toolbar?.id, toolbar?.id]);
|
||||
expect(toolbar && isVomReferenceNode(toolbar)).toBe(false);
|
||||
expect(renderVom(scene).refs.map((ref) => ref.backendNodeId)).toEqual([2, 3]);
|
||||
});
|
||||
|
||||
it("resolves missing AX names through one DOM naming pipeline", () => {
|
||||
const scene = buildSemanticVomScene({
|
||||
viewport: { width: 800, height: 600 },
|
||||
rootFrameId: "main",
|
||||
documents: [
|
||||
document(
|
||||
"main",
|
||||
[
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["labelled", "titled", "text"],
|
||||
},
|
||||
{
|
||||
nodeId: "labelled",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 2,
|
||||
role: { type: "role", value: "button" },
|
||||
},
|
||||
{
|
||||
nodeId: "titled",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 3,
|
||||
role: { type: "role", value: "button" },
|
||||
},
|
||||
{
|
||||
nodeId: "text",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 4,
|
||||
role: { type: "role", value: "button" },
|
||||
},
|
||||
],
|
||||
[
|
||||
dom(1, null, "body"),
|
||||
dom(2, 1, "button", { "aria-label": "Add row" }),
|
||||
dom(3, 1, "button", { title: "Settings" }),
|
||||
dom(4, 1, "button"),
|
||||
dom(5, 4, "span", {}, "Save"),
|
||||
],
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
expect(scene.nodes.filter((node) => node.role === "button").map((node) => node.name)).toEqual([
|
||||
"Add row",
|
||||
"Settings",
|
||||
"Save",
|
||||
]);
|
||||
});
|
||||
|
||||
it("isolates equal backend node ids in sibling frames", () => {
|
||||
const scene = buildSemanticVomScene({
|
||||
viewport: { width: 1000, height: 800 },
|
||||
rootFrameId: "main",
|
||||
documents: [
|
||||
document(
|
||||
"main",
|
||||
[
|
||||
{
|
||||
nodeId: "root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
childIds: ["owner-a", "owner-b"],
|
||||
},
|
||||
{
|
||||
nodeId: "owner-a",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 10,
|
||||
role: { type: "role", value: "Iframe" },
|
||||
},
|
||||
{
|
||||
nodeId: "owner-b",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 20,
|
||||
role: { type: "role", value: "Iframe" },
|
||||
},
|
||||
],
|
||||
[dom(1, null, "body"), dom(10, 1, "iframe"), dom(20, 1, "iframe")],
|
||||
),
|
||||
document(
|
||||
"frame-a",
|
||||
[
|
||||
{
|
||||
nodeId: "button-a",
|
||||
backendDOMNodeId: 5,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Accept" },
|
||||
},
|
||||
],
|
||||
[dom(5, null, "button")],
|
||||
{ parentFrameId: "main", ownerBackendNodeId: 10 },
|
||||
),
|
||||
document(
|
||||
"frame-b",
|
||||
[
|
||||
{
|
||||
nodeId: "button-b",
|
||||
backendDOMNodeId: 5,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Decline" },
|
||||
},
|
||||
],
|
||||
[dom(5, null, "button")],
|
||||
{ parentFrameId: "main", ownerBackendNodeId: 20 },
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
const frameButtons = scene.nodes.filter((node) => node.backendNodeId === 5);
|
||||
const owners = new Map(
|
||||
scene.nodes
|
||||
.filter((node) => node.backendNodeId === 10 || node.backendNodeId === 20)
|
||||
.map((node) => [node.backendNodeId, node.id]),
|
||||
);
|
||||
expect(frameButtons.map((node) => node.frameId)).toEqual(["frame-a", "frame-b"]);
|
||||
expect(frameButtons.map((node) => node.parentId)).toEqual([owners.get(10), owners.get(20)]);
|
||||
expect(renderVom(scene).refs.map((ref) => [ref.frameId, ref.backendNodeId])).toEqual([
|
||||
["frame-a", 5],
|
||||
["frame-b", 5],
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -1020,7 +1020,13 @@ export async function captureViewModel(
|
||||
options: CaptureViewModelOptions = {},
|
||||
): Promise<CapturedViewModel> {
|
||||
throwIfAborted(options.signal);
|
||||
const metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
let metrics: LayoutMetricsReply = {};
|
||||
try {
|
||||
metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
console.debug("[bsk capture] layout metrics unavailable", error);
|
||||
}
|
||||
throwIfAborted(options.signal);
|
||||
const dpr = devicePixelRatio(metrics);
|
||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
|
||||
@@ -33,25 +33,19 @@ async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFra
|
||||
|
||||
function transformFrameNodes(
|
||||
nodes: CapturedNode[],
|
||||
projection: GeometryProjection,
|
||||
projection: GeometryProjection | null,
|
||||
): CapturedNode[] {
|
||||
return nodes.map((node) => ({
|
||||
...node,
|
||||
rect: (() => {
|
||||
const bounds = projectRectToViewport(node.rect, projection);
|
||||
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||
})(),
|
||||
rect: projection
|
||||
? (() => {
|
||||
const bounds = projectRectToViewport(node.rect, projection);
|
||||
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||
})()
|
||||
: null,
|
||||
}));
|
||||
}
|
||||
|
||||
function frameOwnerNode(captured: CapturedViewModel, frame: CdpFrame): CapturedNode | undefined {
|
||||
if (frame.ownerBackendNodeId === undefined) return undefined;
|
||||
const parentNodes = frame.parentFrameId
|
||||
? captured.frameNodes?.get(frame.parentFrameId)
|
||||
: captured.nodes;
|
||||
return parentNodes?.find((node) => node.backendNodeId === frame.ownerBackendNodeId);
|
||||
}
|
||||
|
||||
async function captureMissingFrameDocuments(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
@@ -65,14 +59,20 @@ async function captureMissingFrameDocuments(
|
||||
|
||||
for (const frame of graph.frames) {
|
||||
if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue;
|
||||
const owner = frameOwnerNode(captured, frame);
|
||||
if (!owner?.rect || frame.ownerBackendNodeId === undefined) continue;
|
||||
try {
|
||||
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
|
||||
signal,
|
||||
});
|
||||
const projection = await resolveFrameProjection(cdp, graph, frame.frameId);
|
||||
if (!projection) continue;
|
||||
let projection: GeometryProjection | null = null;
|
||||
try {
|
||||
projection = await resolveFrameProjection(cdp, graph, frame.frameId);
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
console.debug("[bsk observation] child frame geometry projection failed", {
|
||||
frameId: frame.frameId,
|
||||
err,
|
||||
});
|
||||
}
|
||||
const rootFrameId = child.rootFrameId ?? frame.frameId;
|
||||
const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
|
||||
if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
|
||||
@@ -85,11 +85,13 @@ async function captureMissingFrameDocuments(
|
||||
for (const [childFrameId, nodes] of childFrames) {
|
||||
captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
|
||||
}
|
||||
captured.iframeNodes.set(
|
||||
frame.ownerBackendNodeId,
|
||||
captured.frameNodes.get(frame.frameId) ?? [],
|
||||
);
|
||||
captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
|
||||
if (frame.ownerBackendNodeId !== undefined) {
|
||||
captured.iframeNodes.set(
|
||||
frame.ownerBackendNodeId,
|
||||
captured.frameNodes.get(frame.frameId) ?? [],
|
||||
);
|
||||
captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
|
||||
}
|
||||
if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId);
|
||||
for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) {
|
||||
captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId);
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
import type { Viewport } from "@browser-skill/vom";
|
||||
import type { FrameDocument } from "../frame-document";
|
||||
import {
|
||||
frameAxKey,
|
||||
frameBackendKey,
|
||||
type SemanticAxNode,
|
||||
type SemanticFrame,
|
||||
type SemanticGraph,
|
||||
type SemanticGraphNode,
|
||||
type SemanticNodeId,
|
||||
} from "./types";
|
||||
|
||||
export interface BuildSemanticGraphInput {
|
||||
documents: FrameDocument<SemanticAxNode>[];
|
||||
viewport: Viewport;
|
||||
rootFrameId?: string;
|
||||
excludedBackendNodeIds?: ReadonlySet<number>;
|
||||
}
|
||||
|
||||
function domNodeId(frameId: string, backendNodeId: number): SemanticNodeId {
|
||||
return JSON.stringify([frameId, "dom", backendNodeId]);
|
||||
}
|
||||
|
||||
function axNodeId(frameId: string, nodeId: string): SemanticNodeId {
|
||||
return JSON.stringify([frameId, "ax", nodeId]);
|
||||
}
|
||||
|
||||
export function buildSemanticGraph(input: BuildSemanticGraphInput): SemanticGraph {
|
||||
const rootFrameId =
|
||||
input.rootFrameId ??
|
||||
input.documents.find((document) => !document.parentFrameId)?.frameId ??
|
||||
input.documents[0]?.frameId ??
|
||||
"root";
|
||||
const frames = new Map<string, SemanticFrame>();
|
||||
const nodes = new Map<SemanticNodeId, SemanticGraphNode>();
|
||||
const nodeIdsByFrame = new Map<string, SemanticNodeId[]>();
|
||||
const nodeByFrameBackend = new Map<string, SemanticNodeId>();
|
||||
const nodeByFrameAx = new Map<string, SemanticNodeId>();
|
||||
|
||||
for (const [frameOrder, document] of input.documents.entries()) {
|
||||
frames.set(document.frameId, {
|
||||
frameId: document.frameId,
|
||||
...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
|
||||
...(document.ownerBackendNodeId !== undefined
|
||||
? { ownerBackendNodeId: document.ownerBackendNodeId }
|
||||
: {}),
|
||||
contextScopeId: document.contextScopeId,
|
||||
target: document.target,
|
||||
...(document.url ? { url: document.url } : {}),
|
||||
order: frameOrder,
|
||||
});
|
||||
|
||||
const frameNodeIds: SemanticNodeId[] = [];
|
||||
nodeIdsByFrame.set(document.frameId, frameNodeIds);
|
||||
for (const [index, dom] of document.domNodes.entries()) {
|
||||
const id = domNodeId(document.frameId, dom.backendNodeId);
|
||||
const node: SemanticGraphNode = {
|
||||
id,
|
||||
frameId: document.frameId,
|
||||
backendNodeId: dom.backendNodeId,
|
||||
dom,
|
||||
sourceOrder: index,
|
||||
excluded:
|
||||
document.frameId === rootFrameId &&
|
||||
input.excludedBackendNodeIds?.has(dom.backendNodeId) === true,
|
||||
};
|
||||
nodes.set(id, node);
|
||||
frameNodeIds.push(id);
|
||||
nodeByFrameBackend.set(frameBackendKey(document.frameId, dom.backendNodeId), id);
|
||||
}
|
||||
|
||||
for (const [index, ax] of document.axNodes.entries()) {
|
||||
const joinedId =
|
||||
typeof ax.backendDOMNodeId === "number"
|
||||
? nodeByFrameBackend.get(frameBackendKey(document.frameId, ax.backendDOMNodeId))
|
||||
: undefined;
|
||||
const id = joinedId ?? axNodeId(document.frameId, ax.nodeId);
|
||||
const existing = nodes.get(id);
|
||||
if (existing) {
|
||||
existing.ax = ax;
|
||||
existing.axNodeId = ax.nodeId;
|
||||
} else {
|
||||
nodes.set(id, {
|
||||
id,
|
||||
frameId: document.frameId,
|
||||
...(typeof ax.backendDOMNodeId === "number"
|
||||
? { backendNodeId: ax.backendDOMNodeId }
|
||||
: {}),
|
||||
axNodeId: ax.nodeId,
|
||||
ax,
|
||||
sourceOrder: document.domNodes.length + index,
|
||||
excluded:
|
||||
document.frameId === rootFrameId &&
|
||||
typeof ax.backendDOMNodeId === "number" &&
|
||||
input.excludedBackendNodeIds?.has(ax.backendDOMNodeId) === true,
|
||||
});
|
||||
frameNodeIds.push(id);
|
||||
}
|
||||
nodeByFrameAx.set(frameAxKey(document.frameId, ax.nodeId), id);
|
||||
}
|
||||
}
|
||||
|
||||
for (const document of input.documents) {
|
||||
for (const dom of document.domNodes) {
|
||||
const id = nodeByFrameBackend.get(frameBackendKey(document.frameId, dom.backendNodeId));
|
||||
const node = id ? nodes.get(id) : undefined;
|
||||
if (!node || dom.parentBackendNodeId === null) continue;
|
||||
node.domParentId = nodeByFrameBackend.get(
|
||||
frameBackendKey(document.frameId, dom.parentBackendNodeId),
|
||||
);
|
||||
}
|
||||
for (const ax of document.axNodes) {
|
||||
const id = nodeByFrameAx.get(frameAxKey(document.frameId, ax.nodeId));
|
||||
const node = id ? nodes.get(id) : undefined;
|
||||
if (!node || !ax.parentId) continue;
|
||||
node.axParentId = nodeByFrameAx.get(frameAxKey(document.frameId, ax.parentId));
|
||||
}
|
||||
}
|
||||
|
||||
const axChildren = new Map<SemanticNodeId, SemanticNodeId[]>();
|
||||
const excludedQueue: SemanticNodeId[] = [];
|
||||
for (const node of nodes.values()) {
|
||||
if (node.axParentId) {
|
||||
const children = axChildren.get(node.axParentId) ?? [];
|
||||
children.push(node.id);
|
||||
axChildren.set(node.axParentId, children);
|
||||
}
|
||||
if (node.excluded) excludedQueue.push(node.id);
|
||||
}
|
||||
for (let index = 0; index < excludedQueue.length; index += 1) {
|
||||
for (const childId of axChildren.get(excludedQueue[index]) ?? []) {
|
||||
const child = nodes.get(childId);
|
||||
if (!child || child.excluded) continue;
|
||||
child.excluded = true;
|
||||
excludedQueue.push(childId);
|
||||
}
|
||||
}
|
||||
|
||||
for (const frame of frames.values()) {
|
||||
if (!frame.parentFrameId || frame.ownerBackendNodeId === undefined) continue;
|
||||
frame.ownerNodeId = nodeByFrameBackend.get(
|
||||
frameBackendKey(frame.parentFrameId, frame.ownerBackendNodeId),
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
viewport: input.viewport,
|
||||
rootFrameId,
|
||||
frames,
|
||||
nodes,
|
||||
nodeIdsByFrame,
|
||||
nodeByFrameBackend,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import type { Viewport, VomScene } from "@browser-skill/vom";
|
||||
import type { FrameDocument } from "../frame-document";
|
||||
import { buildSemanticGraph } from "./build";
|
||||
import { projectSemanticGraph } from "./project";
|
||||
import { resolveSemanticGraph } from "./resolve";
|
||||
import type { SemanticAxNode } from "./types";
|
||||
|
||||
export type { SemanticAxNode } from "./types";
|
||||
|
||||
export function buildSemanticVomScene(input: {
|
||||
documents: FrameDocument<SemanticAxNode>[];
|
||||
viewport: Viewport;
|
||||
rootFrameId?: string;
|
||||
excludedBackendNodeIds?: ReadonlySet<number>;
|
||||
}): VomScene {
|
||||
return projectSemanticGraph(
|
||||
resolveSemanticGraph(
|
||||
buildSemanticGraph({
|
||||
documents: input.documents,
|
||||
viewport: input.viewport,
|
||||
rootFrameId: input.rootFrameId,
|
||||
excludedBackendNodeIds: input.excludedBackendNodeIds,
|
||||
}),
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
import {
|
||||
applyVomInteractionRecovery,
|
||||
isVomReferenceNode,
|
||||
type VomNode,
|
||||
type VomScene,
|
||||
} from "@browser-skill/vom";
|
||||
import type { ResolvedSemanticGraph, ResolvedSemanticNode, SemanticNodeId } from "./types";
|
||||
|
||||
const TRANSPARENT_ROLES = new Set(["", "generic", "none", "presentation", "inlinetextbox"]);
|
||||
|
||||
function connectedFrameIds(graph: ResolvedSemanticGraph): Set<string> {
|
||||
const connected = new Set<string>([graph.rootFrameId]);
|
||||
const childrenByParent = new Map<string, string[]>();
|
||||
for (const frame of graph.frames.values()) {
|
||||
if (!frame.parentFrameId) continue;
|
||||
const children = childrenByParent.get(frame.parentFrameId) ?? [];
|
||||
children.push(frame.frameId);
|
||||
childrenByParent.set(frame.parentFrameId, children);
|
||||
}
|
||||
const queue = [graph.rootFrameId];
|
||||
for (let index = 0; index < queue.length; index += 1) {
|
||||
for (const frameId of childrenByParent.get(queue[index]) ?? []) {
|
||||
if (connected.has(frameId)) continue;
|
||||
const frame = graph.frames.get(frameId);
|
||||
if (!frame?.ownerNodeId) continue;
|
||||
const owner = graph.nodes.get(frame.ownerNodeId);
|
||||
if (!owner || owner.excluded) continue;
|
||||
connected.add(frame.frameId);
|
||||
queue.push(frame.frameId);
|
||||
}
|
||||
}
|
||||
return connected;
|
||||
}
|
||||
|
||||
function semanticParentId(
|
||||
graph: ResolvedSemanticGraph,
|
||||
node: ResolvedSemanticNode,
|
||||
): SemanticNodeId | undefined {
|
||||
let localParent = node.axParentId ?? node.domParentId;
|
||||
if (node.axParentId && node.domParentId && node.axParentId !== node.domParentId) {
|
||||
const axParent = graph.nodes.get(node.axParentId);
|
||||
const axParentRole = axParent?.vom.role?.toLowerCase() ?? "";
|
||||
const axParentIsRoot = axParentRole === "rootwebarea" || axParentRole === "webarea";
|
||||
const axParentIsStructural =
|
||||
!TRANSPARENT_ROLES.has(axParentRole) ||
|
||||
Boolean(
|
||||
axParent?.vom.name ||
|
||||
axParent?.vom.text ||
|
||||
(axParent ? hasMeaningfulRelations(axParent) : false),
|
||||
);
|
||||
localParent = !axParentIsRoot && axParentIsStructural ? node.axParentId : node.domParentId;
|
||||
}
|
||||
if (localParent) return localParent;
|
||||
if (node.frameId === graph.rootFrameId) return undefined;
|
||||
return graph.frames.get(node.frameId)?.ownerNodeId;
|
||||
}
|
||||
|
||||
function isChildDocumentRoot(graph: ResolvedSemanticGraph, node: ResolvedSemanticNode): boolean {
|
||||
if (node.frameId === graph.rootFrameId) return false;
|
||||
if (semanticParentId(graph, node) !== graph.frames.get(node.frameId)?.ownerNodeId) return false;
|
||||
const role = node.vom.role?.toLowerCase() ?? "";
|
||||
return role === "rootwebarea" || role === "webarea";
|
||||
}
|
||||
|
||||
function hasMeaningfulRelations(node: ResolvedSemanticNode): boolean {
|
||||
const attrs = node.vom.attrs ?? {};
|
||||
return ["aria-controls", "aria-labelledby", "aria-owns", "aria-describedby"].some((name) =>
|
||||
attrs[name]?.trim(),
|
||||
);
|
||||
}
|
||||
|
||||
function shouldKeepNode(
|
||||
graph: ResolvedSemanticGraph,
|
||||
node: ResolvedSemanticNode,
|
||||
recovered: VomNode,
|
||||
): boolean {
|
||||
if (node.excluded || isChildDocumentRoot(graph, node)) return false;
|
||||
if (node.ax?.ignored) return false;
|
||||
const role = recovered.role?.toLowerCase() ?? "";
|
||||
if (!TRANSPARENT_ROLES.has(role)) return true;
|
||||
return Boolean(
|
||||
recovered.name ||
|
||||
recovered.text ||
|
||||
recovered.value ||
|
||||
recovered.placeholder ||
|
||||
hasMeaningfulRelations(node) ||
|
||||
(recovered.rect && ["fixed", "sticky"].includes(recovered.position)),
|
||||
);
|
||||
}
|
||||
|
||||
function redundantSourceNodes(
|
||||
graph: ResolvedSemanticGraph,
|
||||
nodes: ResolvedSemanticNode[],
|
||||
): Set<SemanticNodeId> {
|
||||
const byKey = new Map<string, ResolvedSemanticNode[]>();
|
||||
for (const node of nodes) {
|
||||
const name = node.vom.name?.trim().toLowerCase() ?? "";
|
||||
if (!name || !isVomReferenceNode({ ...node.vom, id: 0, parentId: null, referenceable: true })) {
|
||||
continue;
|
||||
}
|
||||
const key = `${node.frameId}\u0000${semanticParentId(graph, node) ?? ""}\u0000${node.vom.role?.toLowerCase() ?? ""}\u0000${name}\u0000${node.vom.sensitive === true}`;
|
||||
const matches = byKey.get(key) ?? [];
|
||||
matches.push(node);
|
||||
byKey.set(key, matches);
|
||||
}
|
||||
const duplicates = new Set<SemanticNodeId>();
|
||||
for (const matches of byKey.values()) {
|
||||
const axOnly = matches.filter((node) => node.ax && !node.dom);
|
||||
const domOnly = matches.filter((node) => node.dom && !node.ax);
|
||||
if (matches.length === 2 && axOnly.length === 1 && domOnly.length === 1) {
|
||||
duplicates.add(domOnly[0].id);
|
||||
}
|
||||
}
|
||||
return duplicates;
|
||||
}
|
||||
|
||||
function nearestKeptParent(
|
||||
graph: ResolvedSemanticGraph,
|
||||
nodeId: SemanticNodeId,
|
||||
kept: ReadonlySet<SemanticNodeId>,
|
||||
memo: Map<SemanticNodeId, SemanticNodeId | undefined>,
|
||||
): SemanticNodeId | undefined {
|
||||
if (memo.has(nodeId)) return memo.get(nodeId);
|
||||
let current = semanticParentId(graph, graph.nodes.get(nodeId) as ResolvedSemanticNode);
|
||||
const seen = new Set<SemanticNodeId>();
|
||||
const path: SemanticNodeId[] = [];
|
||||
let result: SemanticNodeId | undefined;
|
||||
while (current && !seen.has(current)) {
|
||||
if (kept.has(current)) {
|
||||
result = current;
|
||||
break;
|
||||
}
|
||||
if (memo.has(current)) {
|
||||
result = memo.get(current);
|
||||
break;
|
||||
}
|
||||
seen.add(current);
|
||||
path.push(current);
|
||||
const parent = graph.nodes.get(current);
|
||||
current = parent ? semanticParentId(graph, parent) : undefined;
|
||||
}
|
||||
memo.set(nodeId, result);
|
||||
for (const id of path) memo.set(id, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
function domAncestors(
|
||||
graph: ResolvedSemanticGraph,
|
||||
node: ResolvedSemanticNode,
|
||||
numericId: ReadonlyMap<SemanticNodeId, number>,
|
||||
): number[] {
|
||||
const ancestors: number[] = [];
|
||||
const seen = new Set<SemanticNodeId>();
|
||||
let current = node.domParentId;
|
||||
while (current && !seen.has(current)) {
|
||||
seen.add(current);
|
||||
const id = numericId.get(current);
|
||||
if (id !== undefined) ancestors.push(id);
|
||||
current = graph.nodes.get(current)?.domParentId;
|
||||
}
|
||||
return ancestors;
|
||||
}
|
||||
|
||||
export function projectSemanticGraph(graph: ResolvedSemanticGraph): VomScene {
|
||||
const frameOwnerIds = new Set(
|
||||
[...graph.frames.values()].flatMap((frame) => frame.ownerNodeId ?? []),
|
||||
);
|
||||
const connectedFrames = connectedFrameIds(graph);
|
||||
const orderedNodes = [...graph.frames.values()]
|
||||
.filter((frame) => connectedFrames.has(frame.frameId))
|
||||
.sort((a, b) => a.order - b.order)
|
||||
.flatMap((frame) =>
|
||||
(graph.nodeIdsByFrame.get(frame.frameId) ?? [])
|
||||
.map((id) => graph.nodes.get(id))
|
||||
.filter((node): node is ResolvedSemanticNode => node !== undefined)
|
||||
.sort((a, b) => a.sourceOrder - b.sourceOrder),
|
||||
);
|
||||
|
||||
const provisionalNumericId = new Map<SemanticNodeId, number>();
|
||||
orderedNodes.forEach((node, index) => provisionalNumericId.set(node.id, index + 1));
|
||||
const provisionalNodes: VomNode[] = orderedNodes.map((node) => ({
|
||||
...node.vom,
|
||||
id: provisionalNumericId.get(node.id) as number,
|
||||
parentId: (() => {
|
||||
const parent = semanticParentId(graph, node);
|
||||
return parent ? (provisionalNumericId.get(parent) ?? null) : null;
|
||||
})(),
|
||||
...(node.backendNodeId !== undefined ? { backendNodeId: node.backendNodeId } : {}),
|
||||
frameId: node.frameId,
|
||||
contextScopeId: graph.frames.get(node.frameId)?.contextScopeId ?? node.frameId,
|
||||
referenceable: node.referenceable,
|
||||
}));
|
||||
const recoveredById = new Map(
|
||||
applyVomInteractionRecovery(provisionalNodes).map((node) => [node.id, node]),
|
||||
);
|
||||
const duplicates = redundantSourceNodes(graph, orderedNodes);
|
||||
const kept = new Set<SemanticNodeId>();
|
||||
for (const node of orderedNodes) {
|
||||
if (duplicates.has(node.id)) continue;
|
||||
const id = provisionalNumericId.get(node.id) as number;
|
||||
const recovered = recoveredById.get(id) as VomNode;
|
||||
if ((!node.excluded && frameOwnerIds.has(node.id)) || shouldKeepNode(graph, node, recovered)) {
|
||||
kept.add(node.id);
|
||||
}
|
||||
}
|
||||
|
||||
const numericId = new Map<SemanticNodeId, number>();
|
||||
let nextId = 1;
|
||||
for (const node of orderedNodes) {
|
||||
if (!kept.has(node.id)) continue;
|
||||
numericId.set(node.id, nextId);
|
||||
nextId += 1;
|
||||
}
|
||||
|
||||
const nodes: VomNode[] = [];
|
||||
const keptParentMemo = new Map<SemanticNodeId, SemanticNodeId | undefined>();
|
||||
for (const node of orderedNodes) {
|
||||
const id = numericId.get(node.id);
|
||||
if (id === undefined) continue;
|
||||
const provisional = recoveredById.get(provisionalNumericId.get(node.id) as number) as VomNode;
|
||||
const parent = nearestKeptParent(graph, node.id, kept, keptParentMemo);
|
||||
const domParent = node.domParentId ? numericId.get(node.domParentId) : undefined;
|
||||
nodes.push({
|
||||
...provisional,
|
||||
id,
|
||||
parentId: parent ? (numericId.get(parent) ?? null) : null,
|
||||
...(node.backendNodeId !== undefined ? { backendNodeId: node.backendNodeId } : {}),
|
||||
...(node.dom
|
||||
? {
|
||||
domParentId: node.domParentId === undefined ? null : (domParent ?? null),
|
||||
domAncestorIds: domAncestors(graph, node, numericId),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
viewport: graph.viewport,
|
||||
nodes,
|
||||
rootFrameId: graph.rootFrameId,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,527 @@
|
||||
import type { VomNode } from "@browser-skill/vom";
|
||||
import type {
|
||||
ResolvedSemanticGraph,
|
||||
ResolvedSemanticNode,
|
||||
SemanticAxNode,
|
||||
SemanticGraph,
|
||||
SemanticGraphNode,
|
||||
SemanticNodeId,
|
||||
} from "./types";
|
||||
|
||||
const FORM_TAGS = new Set(["input", "textarea", "select"]);
|
||||
const NATIVE_CONTROL_TAGS = new Set(["button", "input", "select", "textarea"]);
|
||||
const TEXT_AX_ROLES = new Set(["statictext", "inlinetextbox", "labeltext", "text"]);
|
||||
const INTERACTIVE_ROLES = new Set([
|
||||
"button",
|
||||
"checkbox",
|
||||
"combobox",
|
||||
"link",
|
||||
"listbox",
|
||||
"menuitem",
|
||||
"menuitemcheckbox",
|
||||
"menuitemradio",
|
||||
"option",
|
||||
"radio",
|
||||
"searchbox",
|
||||
"slider",
|
||||
"spinbutton",
|
||||
"switch",
|
||||
"tab",
|
||||
"textbox",
|
||||
"treeitem",
|
||||
]);
|
||||
const SENSITIVE_INPUT_TYPES = new Set([
|
||||
"password",
|
||||
"credit-card",
|
||||
"one-time-code",
|
||||
"current-password",
|
||||
"new-password",
|
||||
]);
|
||||
|
||||
function clean(value: string | undefined): string | undefined {
|
||||
const normalized = value?.replace(/\s+/g, " ").trim();
|
||||
return normalized || undefined;
|
||||
}
|
||||
|
||||
function axValue(field?: { value?: string | number | boolean }): string | undefined {
|
||||
return field?.value === undefined ? undefined : clean(String(field.value));
|
||||
}
|
||||
|
||||
function axProperty(node: SemanticAxNode | undefined, name: string): string | undefined {
|
||||
return axValue(node?.properties?.find((property) => property.name === name)?.value);
|
||||
}
|
||||
|
||||
function normalizeRole(value: string | undefined): string | undefined {
|
||||
const role = clean(value)?.toLowerCase();
|
||||
return role || undefined;
|
||||
}
|
||||
|
||||
function nativeRole(tag: string, attrs: Record<string, string>): string | undefined {
|
||||
const explicit = normalizeRole(attrs.role);
|
||||
if (explicit) return explicit;
|
||||
if (tag === "button") return "button";
|
||||
if (tag === "textarea") return "textbox";
|
||||
if (tag === "select") return attrs.multiple === undefined ? "combobox" : "listbox";
|
||||
if (tag === "a" && attrs.href) return "link";
|
||||
if (tag === "iframe") return "Iframe";
|
||||
if (tag === "dialog") return "dialog";
|
||||
if (tag === "img") return "img";
|
||||
if (tag === "table") return "table";
|
||||
if (tag === "tr") return "row";
|
||||
if (tag === "th") return "columnheader";
|
||||
if (tag === "td") return "cell";
|
||||
if (tag === "ul" || tag === "ol") return "list";
|
||||
if (tag === "li") return "listitem";
|
||||
if (/^h[1-6]$/.test(tag)) return "heading";
|
||||
if (tag === "nav") return "navigation";
|
||||
if (tag === "main") return "main";
|
||||
if (tag === "form") return "form";
|
||||
if (tag === "input") {
|
||||
const type = (attrs.type ?? "text").toLowerCase();
|
||||
if (type === "hidden") return undefined;
|
||||
if (type === "checkbox") return "checkbox";
|
||||
if (type === "radio") return "radio";
|
||||
if (type === "range") return "slider";
|
||||
if (type === "number") return "spinbutton";
|
||||
if (type === "search") return "searchbox";
|
||||
if (["button", "submit", "reset", "image"].includes(type)) return "button";
|
||||
return "textbox";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function resolvedRole(node: SemanticGraphNode): string | undefined {
|
||||
const rawAxRole = clean(axValue(node.ax?.role));
|
||||
const axRole = normalizeRole(rawAxRole);
|
||||
const domRole = nativeRole(node.dom?.tag.toLowerCase() ?? "", node.dom?.attrs ?? {});
|
||||
if (!axRole || ["generic", "none", "presentation"].includes(axRole)) {
|
||||
return domRole ?? rawAxRole;
|
||||
}
|
||||
return rawAxRole;
|
||||
}
|
||||
|
||||
function sensitive(node: SemanticGraphNode): boolean {
|
||||
const attrs = node.dom?.attrs ?? {};
|
||||
const type = (attrs.type ?? "").toLowerCase();
|
||||
const autocomplete = (attrs.autocomplete ?? "").toLowerCase();
|
||||
return (
|
||||
type === "password" ||
|
||||
autocomplete.startsWith("cc-") ||
|
||||
SENSITIVE_INPUT_TYPES.has(autocomplete) ||
|
||||
SENSITIVE_INPUT_TYPES.has(axProperty(node.ax, "inputType") ?? "")
|
||||
);
|
||||
}
|
||||
|
||||
function inputState(
|
||||
node: SemanticGraphNode,
|
||||
role: string | undefined,
|
||||
value: string | undefined,
|
||||
isSensitive: boolean,
|
||||
): VomNode["inputState"] {
|
||||
if (node.dom?.formState) return node.dom.formState;
|
||||
const tag = node.dom?.tag.toLowerCase() ?? "";
|
||||
if (!FORM_TAGS.has(tag) && !["textbox", "searchbox"].includes(role ?? "")) return undefined;
|
||||
if (isSensitive) return value ? "filled" : "empty";
|
||||
if (node.dom?.formValue !== undefined) {
|
||||
if (node.dom.formValue === "") return "empty";
|
||||
return node.dom.formValue === (node.dom.formDefaultValue ?? "") ? "default" : "filled";
|
||||
}
|
||||
return value ? "filled" : "empty";
|
||||
}
|
||||
|
||||
function externalHrefHost(
|
||||
href: string | undefined,
|
||||
pageUrl: string | undefined,
|
||||
): string | undefined {
|
||||
if (!href || !pageUrl) return undefined;
|
||||
try {
|
||||
const page = new URL(pageUrl);
|
||||
const target = new URL(href, page);
|
||||
return target.origin === page.origin ? undefined : target.hostname;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function iconKeyword(value: string | undefined): string | undefined {
|
||||
const token = clean(value)
|
||||
?.split(/[#/.\s:_-]+/)
|
||||
.filter(Boolean)
|
||||
.pop();
|
||||
return token && /^[a-z][a-z0-9_-]{1,32}$/i.test(token) ? token.toLowerCase() : undefined;
|
||||
}
|
||||
|
||||
interface ResolveIndexes {
|
||||
domChildren: Map<SemanticNodeId, SemanticNodeId[]>;
|
||||
axChildren: Map<SemanticNodeId, SemanticNodeId[]>;
|
||||
domIdIndex: Map<string, SemanticNodeId>;
|
||||
labelForIndex: Map<string, SemanticNodeId[]>;
|
||||
domText: (nodeId: SemanticNodeId) => string | undefined;
|
||||
axText: (nodeId: SemanticNodeId) => string | undefined;
|
||||
hasNativeDescendant: (nodeId: SemanticNodeId) => boolean;
|
||||
}
|
||||
|
||||
function buildIndexes(graph: SemanticGraph): ResolveIndexes {
|
||||
const domChildren = new Map<SemanticNodeId, SemanticNodeId[]>();
|
||||
const axChildren = new Map<SemanticNodeId, SemanticNodeId[]>();
|
||||
const domIdIndex = new Map<string, SemanticNodeId>();
|
||||
const labelForIndex = new Map<string, SemanticNodeId[]>();
|
||||
for (const node of graph.nodes.values()) {
|
||||
if (node.domParentId) {
|
||||
const children = domChildren.get(node.domParentId) ?? [];
|
||||
children.push(node.id);
|
||||
domChildren.set(node.domParentId, children);
|
||||
}
|
||||
if (node.axParentId) {
|
||||
const children = axChildren.get(node.axParentId) ?? [];
|
||||
children.push(node.id);
|
||||
axChildren.set(node.axParentId, children);
|
||||
}
|
||||
const id = clean(node.dom?.attrs.id);
|
||||
if (id) domIdIndex.set(`${node.frameId}\u0000${id}`, node.id);
|
||||
const targetId = clean(node.dom?.attrs.for);
|
||||
if (node.dom?.tag.toLowerCase() === "label" && targetId) {
|
||||
const key = `${node.frameId}\u0000${targetId}`;
|
||||
const labels = labelForIndex.get(key) ?? [];
|
||||
labels.push(node.id);
|
||||
labelForIndex.set(key, labels);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [parentId, children] of axChildren) {
|
||||
const order = graph.nodes.get(parentId)?.ax?.childIds ?? [];
|
||||
const orderByAxId = new Map(order.map((nodeId, index) => [nodeId, index]));
|
||||
children.sort((a, b) => {
|
||||
const aIndex = orderByAxId.get(graph.nodes.get(a)?.axNodeId ?? "") ?? -1;
|
||||
const bIndex = orderByAxId.get(graph.nodes.get(b)?.axNodeId ?? "") ?? -1;
|
||||
return (
|
||||
(aIndex < 0 ? Number.MAX_SAFE_INTEGER : aIndex) -
|
||||
(bIndex < 0 ? Number.MAX_SAFE_INTEGER : bIndex)
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
const textMemo = new Map<SemanticNodeId, string | undefined>();
|
||||
const resolvingText = new Set<SemanticNodeId>();
|
||||
const domText = (nodeId: SemanticNodeId): string | undefined => {
|
||||
if (textMemo.has(nodeId)) return textMemo.get(nodeId);
|
||||
if (resolvingText.has(nodeId)) return undefined;
|
||||
resolvingText.add(nodeId);
|
||||
const node = graph.nodes.get(nodeId);
|
||||
const parts: string[] = [];
|
||||
const own = clean(node?.dom?.textContent);
|
||||
if (own) parts.push(own);
|
||||
for (const childId of domChildren.get(nodeId) ?? []) {
|
||||
const child = graph.nodes.get(childId);
|
||||
const childRole = child
|
||||
? nativeRole(child.dom?.tag.toLowerCase() ?? "", child.dom?.attrs ?? {})
|
||||
: undefined;
|
||||
if (childRole && INTERACTIVE_ROLES.has(childRole.toLowerCase())) continue;
|
||||
const childText = domText(childId);
|
||||
if (childText) parts.push(childText);
|
||||
if (parts.join(" ").length >= 240) break;
|
||||
}
|
||||
resolvingText.delete(nodeId);
|
||||
const result = clean(parts.join(" ").slice(0, 240));
|
||||
textMemo.set(nodeId, result);
|
||||
return result;
|
||||
};
|
||||
|
||||
const axTextMemo = new Map<SemanticNodeId, string | undefined>();
|
||||
const resolvingAxText = new Set<SemanticNodeId>();
|
||||
const axText = (nodeId: SemanticNodeId): string | undefined => {
|
||||
if (axTextMemo.has(nodeId)) return axTextMemo.get(nodeId);
|
||||
if (resolvingAxText.has(nodeId)) return undefined;
|
||||
resolvingAxText.add(nodeId);
|
||||
const node = graph.nodes.get(nodeId);
|
||||
const role = normalizeRole(axValue(node?.ax?.role)) ?? "";
|
||||
const parts: string[] = [];
|
||||
if (TEXT_AX_ROLES.has(role)) {
|
||||
const own = clean(axValue(node?.ax?.name) ?? axValue(node?.ax?.value));
|
||||
if (own) parts.push(own);
|
||||
}
|
||||
for (const childId of axChildren.get(nodeId) ?? []) {
|
||||
const child = graph.nodes.get(childId);
|
||||
const childRole = normalizeRole(axValue(child?.ax?.role)) ?? "";
|
||||
if (INTERACTIVE_ROLES.has(childRole)) continue;
|
||||
const childText = axText(childId);
|
||||
if (childText) parts.push(childText);
|
||||
if (parts.join(" ").length >= 240) break;
|
||||
}
|
||||
resolvingAxText.delete(nodeId);
|
||||
const result = clean(parts.join(" ").slice(0, 240));
|
||||
axTextMemo.set(nodeId, result);
|
||||
return result;
|
||||
};
|
||||
|
||||
const nativeMemo = new Map<SemanticNodeId, boolean>();
|
||||
const hasNativeDescendant = (nodeId: SemanticNodeId): boolean => {
|
||||
const cached = nativeMemo.get(nodeId);
|
||||
if (cached !== undefined) return cached;
|
||||
nativeMemo.set(nodeId, false);
|
||||
for (const childId of domChildren.get(nodeId) ?? []) {
|
||||
const child = graph.nodes.get(childId);
|
||||
const childRole =
|
||||
normalizeRole(axValue(child?.ax?.role)) ??
|
||||
(child
|
||||
? nativeRole(child.dom?.tag.toLowerCase() ?? "", child.dom?.attrs ?? {})
|
||||
: undefined);
|
||||
if (
|
||||
(child?.dom && NATIVE_CONTROL_TAGS.has(child.dom.tag.toLowerCase())) ||
|
||||
INTERACTIVE_ROLES.has(childRole ?? "") ||
|
||||
hasNativeDescendant(childId)
|
||||
) {
|
||||
nativeMemo.set(nodeId, true);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
return {
|
||||
domChildren,
|
||||
axChildren,
|
||||
domIdIndex,
|
||||
labelForIndex,
|
||||
domText,
|
||||
axText,
|
||||
hasNativeDescendant,
|
||||
};
|
||||
}
|
||||
|
||||
function ariaLabelledText(node: SemanticGraphNode, indexes: ResolveIndexes): string | undefined {
|
||||
const ids = (node.dom?.attrs["aria-labelledby"] ?? "").split(/\s+/).filter(Boolean);
|
||||
const parts = ids.flatMap((id) => {
|
||||
const target = indexes.domIdIndex.get(`${node.frameId}\u0000${id}`);
|
||||
return target ? (indexes.domText(target) ?? []) : [];
|
||||
});
|
||||
return clean(parts.join(" "));
|
||||
}
|
||||
|
||||
function nativeLabelText(
|
||||
node: SemanticGraphNode,
|
||||
graph: SemanticGraph,
|
||||
indexes: ResolveIndexes,
|
||||
): string | undefined {
|
||||
const id = clean(node.dom?.attrs.id);
|
||||
if (id) {
|
||||
const labels = indexes.labelForIndex.get(`${node.frameId}\u0000${id}`) ?? [];
|
||||
const explicit = clean(
|
||||
labels
|
||||
.map((labelId) => indexes.domText(labelId))
|
||||
.filter(Boolean)
|
||||
.join(" "),
|
||||
);
|
||||
if (explicit) return explicit;
|
||||
}
|
||||
let parentId = node.domParentId;
|
||||
let guard = 0;
|
||||
while (parentId && guard <= graph.nodes.size) {
|
||||
const parent = graph.nodes.get(parentId);
|
||||
if (!parent) break;
|
||||
if (parent.dom?.tag.toLowerCase() === "label") return indexes.domText(parentId);
|
||||
parentId = parent.domParentId;
|
||||
guard += 1;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function precedingAxText(
|
||||
node: SemanticGraphNode,
|
||||
graph: SemanticGraph,
|
||||
indexes: ResolveIndexes,
|
||||
): string | undefined {
|
||||
if (!node.axParentId) return undefined;
|
||||
const siblings = indexes.axChildren.get(node.axParentId) ?? [];
|
||||
const index = siblings.indexOf(node.id);
|
||||
for (let offset = 1; offset <= 4 && index - offset >= 0; offset += 1) {
|
||||
const sibling = graph.nodes.get(siblings[index - offset]);
|
||||
const role = normalizeRole(axValue(sibling?.ax?.role)) ?? "";
|
||||
if (INTERACTIVE_ROLES.has(role)) break;
|
||||
if (!TEXT_AX_ROLES.has(role) && role !== "generic") continue;
|
||||
const text = clean(axValue(sibling?.ax?.name) ?? axValue(sibling?.ax?.value))?.replace(
|
||||
/[：:]\s*$/,
|
||||
"",
|
||||
);
|
||||
if (text && text.length <= 80) return text;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function precedingDomText(
|
||||
node: SemanticGraphNode,
|
||||
graph: SemanticGraph,
|
||||
indexes: ResolveIndexes,
|
||||
): string | undefined {
|
||||
if (!node.domParentId) return undefined;
|
||||
const siblings = indexes.domChildren.get(node.domParentId) ?? [];
|
||||
const index = siblings.indexOf(node.id);
|
||||
for (let offset = 1; offset <= 4 && index - offset >= 0; offset += 1) {
|
||||
const sibling = graph.nodes.get(siblings[index - offset]);
|
||||
const siblingRole = sibling
|
||||
? nativeRole(sibling.dom?.tag.toLowerCase() ?? "", sibling.dom?.attrs ?? {})
|
||||
: undefined;
|
||||
if (siblingRole && INTERACTIVE_ROLES.has(siblingRole.toLowerCase())) break;
|
||||
const text = indexes.domText(siblings[index - offset])?.replace(/[：:]\s*$/, "");
|
||||
if (text && text.length <= 80) return text;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function iconHint(
|
||||
nodeId: SemanticNodeId,
|
||||
graph: SemanticGraph,
|
||||
indexes: ResolveIndexes,
|
||||
depth = 0,
|
||||
): string | undefined {
|
||||
if (depth > 3) return undefined;
|
||||
for (const childId of indexes.domChildren.get(nodeId) ?? []) {
|
||||
const child = graph.nodes.get(childId);
|
||||
const attrs = child?.dom?.attrs ?? {};
|
||||
const hint =
|
||||
iconKeyword(attrs["aria-label"]) ??
|
||||
iconKeyword(attrs.title) ??
|
||||
iconKeyword(attrs.href) ??
|
||||
iconKeyword(attrs["xlink:href"]) ??
|
||||
iconKeyword(axValue(child?.ax?.name)) ??
|
||||
iconKeyword(attrs.class) ??
|
||||
iconKeyword(child?.dom?.textContent);
|
||||
if (hint) return hint;
|
||||
const nested = iconHint(childId, graph, indexes, depth + 1);
|
||||
if (nested) return nested;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function resolvedName(
|
||||
node: SemanticGraphNode,
|
||||
role: string | undefined,
|
||||
graph: SemanticGraph,
|
||||
indexes: ResolveIndexes,
|
||||
): string | undefined {
|
||||
const attrs = node.dom?.attrs ?? {};
|
||||
const formControl = FORM_TAGS.has(node.dom?.tag.toLowerCase() ?? "");
|
||||
const descendantText = indexes.domText(node.id);
|
||||
const canUseIconHint =
|
||||
!indexes.hasNativeDescendant(node.id) &&
|
||||
(INTERACTIVE_ROLES.has(role?.toLowerCase() ?? "") ||
|
||||
node.dom?.cursor === "pointer" ||
|
||||
attrs.onclick !== undefined ||
|
||||
attrs.tabindex !== undefined);
|
||||
const candidates = [
|
||||
axValue(node.ax?.name),
|
||||
indexes.axText(node.id),
|
||||
ariaLabelledText(node, indexes),
|
||||
clean(attrs["aria-label"]),
|
||||
nativeLabelText(node, graph, indexes),
|
||||
clean(attrs.title),
|
||||
clean(attrs.alt),
|
||||
descendantText,
|
||||
formControl ? precedingAxText(node, graph, indexes) : undefined,
|
||||
formControl ? precedingDomText(node, graph, indexes) : undefined,
|
||||
formControl ? clean(node.dom?.formPlaceholder ?? attrs.placeholder) : undefined,
|
||||
canUseIconHint ? iconHint(node.id, graph, indexes) : undefined,
|
||||
];
|
||||
let name = candidates.find((candidate) => candidate !== undefined);
|
||||
const hasPopup = axProperty(node.ax, "hasPopup");
|
||||
if (name && hasPopup && hasPopup !== "false") {
|
||||
name =
|
||||
axProperty(node.ax, "expanded") === "true" ? `${name} [expanded]` : `${name} [has-submenu]`;
|
||||
}
|
||||
if (!name && role?.toLowerCase() === "iframe") return clean(attrs.id);
|
||||
return name;
|
||||
}
|
||||
|
||||
function insideNative(node: SemanticGraphNode, graph: SemanticGraph): boolean {
|
||||
let parentId = node.domParentId;
|
||||
let guard = 0;
|
||||
while (parentId && guard <= graph.nodes.size) {
|
||||
const parent = graph.nodes.get(parentId);
|
||||
if (!parent) break;
|
||||
if (parent.dom && NATIVE_CONTROL_TAGS.has(parent.dom.tag.toLowerCase())) return true;
|
||||
parentId = parent.domParentId;
|
||||
guard += 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function nearbyText(
|
||||
node: SemanticGraphNode,
|
||||
graph: SemanticGraph,
|
||||
indexes: ResolveIndexes,
|
||||
): string | undefined {
|
||||
if (!node.domParentId) return undefined;
|
||||
const siblings = indexes.domChildren.get(node.domParentId) ?? [];
|
||||
const index = siblings.indexOf(node.id);
|
||||
if (index < 0) return undefined;
|
||||
const texts = siblings
|
||||
.slice(Math.max(0, index - 3), index + 4)
|
||||
.filter((id) => id !== node.id)
|
||||
.flatMap((id) => indexes.domText(id) ?? [])
|
||||
.slice(0, 3);
|
||||
return clean(texts.join(" "));
|
||||
}
|
||||
|
||||
export function resolveSemanticGraph(graph: SemanticGraph): ResolvedSemanticGraph {
|
||||
const indexes = buildIndexes(graph);
|
||||
const nodes = new Map<SemanticNodeId, ResolvedSemanticNode>();
|
||||
for (const node of graph.nodes.values()) {
|
||||
const role = resolvedRole(node);
|
||||
const name = resolvedName(node, role, graph, indexes);
|
||||
const isSensitive = sensitive(node);
|
||||
const sourceValue = node.dom?.formValue ?? axValue(node.ax?.value);
|
||||
const value = isSensitive ? undefined : sourceValue;
|
||||
const attrs = { ...(node.dom?.attrs ?? {}) };
|
||||
if (isSensitive) delete attrs.value;
|
||||
const frame = graph.frames.get(node.frameId);
|
||||
const tag = node.dom?.tag.toLowerCase() ?? "";
|
||||
const vom: ResolvedSemanticNode["vom"] = {
|
||||
tag,
|
||||
rect: node.dom?.rect ?? null,
|
||||
paintOrder: node.dom?.paintOrder ?? 0,
|
||||
position: node.dom?.position || "static",
|
||||
pointerEvents: node.dom?.pointerEvents || "auto",
|
||||
...(node.dom?.cursor ? { cursor: node.dom.cursor } : {}),
|
||||
...(role ? { role } : {}),
|
||||
...(name ? { name } : {}),
|
||||
...(value !== undefined ? { value } : {}),
|
||||
...(clean(node.dom?.formPlaceholder ?? attrs.placeholder)
|
||||
? { placeholder: clean(node.dom?.formPlaceholder ?? attrs.placeholder) }
|
||||
: {}),
|
||||
attrs,
|
||||
...(clean(node.dom?.textContent) ? { text: clean(node.dom?.textContent) } : {}),
|
||||
...(nearbyText(node, graph, indexes) ? { nearbyText: nearbyText(node, graph, indexes) } : {}),
|
||||
inputState: inputState(node, role, sourceValue, isSensitive),
|
||||
sensitive: isSensitive,
|
||||
modal:
|
||||
["dialog", "alertdialog"].includes(role?.toLowerCase() ?? "") ||
|
||||
tag === "dialog" ||
|
||||
(attrs["aria-modal"] ?? "").toLowerCase() === "true",
|
||||
disabled:
|
||||
Object.prototype.hasOwnProperty.call(attrs, "disabled") ||
|
||||
(attrs["aria-disabled"] ?? "").toLowerCase() === "true",
|
||||
inert: Object.prototype.hasOwnProperty.call(attrs, "inert"),
|
||||
hasNativeDescendant: indexes.hasNativeDescendant(node.id),
|
||||
insideNative: insideNative(node, graph),
|
||||
...(role?.toLowerCase() === "link" ? { href: externalHrefHost(attrs.href, frame?.url) } : {}),
|
||||
};
|
||||
const selected = axProperty(node.ax, "selected");
|
||||
const expanded = axProperty(node.ax, "expanded");
|
||||
const controls = axProperty(node.ax, "controls");
|
||||
if (selected === "true" && !vom.attrs?.["aria-selected"]) {
|
||||
vom.attrs = { ...(vom.attrs ?? {}), "aria-selected": "true" };
|
||||
}
|
||||
if (expanded === "true" && !vom.attrs?.["aria-expanded"]) {
|
||||
vom.attrs = { ...(vom.attrs ?? {}), "aria-expanded": "true" };
|
||||
}
|
||||
if (controls && !vom.attrs?.["aria-controls"]) {
|
||||
vom.attrs = { ...(vom.attrs ?? {}), "aria-controls": controls };
|
||||
}
|
||||
nodes.set(node.id, {
|
||||
...node,
|
||||
vom,
|
||||
referenceable: node.backendNodeId !== undefined,
|
||||
});
|
||||
}
|
||||
return { ...graph, nodes };
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import type { Viewport, VomNode } from "@browser-skill/vom";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { CapturedNode } from "../capture";
|
||||
|
||||
export interface SemanticAxNode {
|
||||
nodeId: string;
|
||||
frameId?: string;
|
||||
parentId?: string;
|
||||
backendDOMNodeId?: number;
|
||||
ignored?: boolean;
|
||||
role?: { type: string; value?: string };
|
||||
name?: { type: string; value?: string };
|
||||
description?: { value?: string };
|
||||
value?: { value?: string | number | boolean };
|
||||
properties?: Array<{ name?: string; value?: { value?: string | number | boolean } }>;
|
||||
childIds?: string[];
|
||||
}
|
||||
|
||||
export type SemanticNodeId = string;
|
||||
|
||||
export interface SemanticFrame {
|
||||
frameId: string;
|
||||
parentFrameId?: string;
|
||||
ownerBackendNodeId?: number;
|
||||
ownerNodeId?: SemanticNodeId;
|
||||
contextScopeId: string;
|
||||
target: CdpTarget;
|
||||
url?: string;
|
||||
order: number;
|
||||
}
|
||||
|
||||
export interface SemanticGraphNode {
|
||||
id: SemanticNodeId;
|
||||
frameId: string;
|
||||
backendNodeId?: number;
|
||||
axNodeId?: string;
|
||||
ax?: SemanticAxNode;
|
||||
dom?: CapturedNode;
|
||||
axParentId?: SemanticNodeId;
|
||||
domParentId?: SemanticNodeId;
|
||||
sourceOrder: number;
|
||||
excluded: boolean;
|
||||
}
|
||||
|
||||
export interface SemanticGraph {
|
||||
viewport: Viewport;
|
||||
rootFrameId: string;
|
||||
frames: Map<string, SemanticFrame>;
|
||||
nodes: Map<SemanticNodeId, SemanticGraphNode>;
|
||||
nodeIdsByFrame: Map<string, SemanticNodeId[]>;
|
||||
nodeByFrameBackend: Map<string, SemanticNodeId>;
|
||||
}
|
||||
|
||||
export interface ResolvedSemanticNode extends SemanticGraphNode {
|
||||
vom: Omit<VomNode, "id" | "parentId" | "backendNodeId" | "frameId" | "contextScopeId">;
|
||||
referenceable: boolean;
|
||||
}
|
||||
|
||||
export interface ResolvedSemanticGraph extends Omit<SemanticGraph, "nodes"> {
|
||||
nodes: Map<SemanticNodeId, ResolvedSemanticNode>;
|
||||
}
|
||||
|
||||
export function frameBackendKey(frameId: string, backendNodeId: number): string {
|
||||
return `${frameId}\u0000${backendNodeId}`;
|
||||
}
|
||||
|
||||
export function frameAxKey(frameId: string, axNodeId: string): string {
|
||||
return `${frameId}\u0000${axNodeId}`;
|
||||
}
|
||||
@@ -199,7 +199,10 @@ function shouldRender(node: VomNode): boolean {
|
||||
}
|
||||
|
||||
export function isVomReferenceNode(node: VomNode): boolean {
|
||||
return INTERACTIVE_ROLES.has(normalizedRole(node)) || isUrlBearingStructuralAction(node);
|
||||
return (
|
||||
node.referenceable !== false &&
|
||||
(INTERACTIVE_ROLES.has(normalizedRole(node)) || isUrlBearingStructuralAction(node))
|
||||
);
|
||||
}
|
||||
|
||||
function hasRecoveryHandler(node: VomNode): boolean {
|
||||
|
||||
@@ -24,6 +24,8 @@ export interface VomNode {
|
||||
backendNodeId?: number;
|
||||
frameId?: string;
|
||||
contextScopeId?: string;
|
||||
/** False for semantic-only nodes that cannot be addressed through CDP. */
|
||||
referenceable?: boolean;
|
||||
|
||||
role?: string;
|
||||
name?: string;
|
||||
|
||||
Reference in New Issue
Block a user