feat(vom): fix name lost

This commit is contained in:
Ljy-0827 authored and GitHub committed 2026-08-26 15:28:27 +08:00
1 parent 41cb576256
commit b5062ac19b
15 files changed
+1588 -304

No files matched your search

@@ -1,25 +1,58 @@
import type { Viewport, VomScene } from "@browser-skill/vom";
import type { FrameDocument } from "../frame-document";
import { buildSemanticGraph } from "./build";
import { projectSemanticGraph } from "./project";
import { resolveSemanticGraph } from "./resolve";
import type { SemanticAxNode } from "./types";
import { buildSemanticGraph as buildGraph } from "./build";
import { projectSemanticGraph as projectGraph } from "./project";
import { resolveSemanticGraph as resolveGraph } from "./resolve";
import { normalizeSemanticStructure as normalizeStructure } from "./structure";
import type {
ResolvedSemanticGraph,
SemanticAxNode,
SemanticGraph,
StructuredSemanticGraph,
} from "./types";
export type { SemanticAxNode } from "./types";
export function buildSemanticVomScene(input: {
export interface SemanticVomInput {
documents: FrameDocument<SemanticAxNode>[];
viewport: Viewport;
rootFrameId?: string;
excludedBackendNodeIds?: ReadonlySet<number>;
}): VomScene {
}
export function buildSemanticGraph(input: SemanticVomInput): SemanticGraph {
return buildGraph(input);
}
export function resolveSemanticGraph(
graph: SemanticGraph,
options: {
supplementalNames?: ReadonlyMap<string, string>;
identifierFallback?: boolean;
} = {},
): ResolvedSemanticGraph {
return resolveGraph(graph, options);
}
export function normalizeSemanticStructure(graph: ResolvedSemanticGraph): StructuredSemanticGraph {
return normalizeStructure(graph);
}
export function projectSemanticGraph(graph: StructuredSemanticGraph): VomScene {
return projectGraph(graph);
}
export function buildSemanticVomScene(
input: SemanticVomInput & {
supplementalNames?: ReadonlyMap<string, string>;
identifierFallback?: boolean;
},
): VomScene {
return projectSemanticGraph(
resolveSemanticGraph(
buildSemanticGraph({
documents: input.documents,
viewport: input.viewport,
rootFrameId: input.rootFrameId,
excludedBackendNodeIds: input.excludedBackendNodeIds,
normalizeSemanticStructure(
resolveSemanticGraph(buildSemanticGraph(input), {
supplementalNames: input.supplementalNames,
identifierFallback: input.identifierFallback,
}),
),
);
@@ -0,0 +1,34 @@
const IDENTIFIER_ATTRS = ["data-testid", "data-test-id", "id"] as const;
const GENERIC_TOKENS = new Set(["btn", "button", "control", "icon", "item", "test", "tool"]);
const GENERATED_PREFIXES = new Set(["ember", "headlessui", "mui", "radix", "react", "rc"]);
function identifierTokens(value: string): string[] {
return value
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
.split(/[^a-zA-Z0-9]+/)
.map((token) => token.toLowerCase())
.filter(Boolean);
}
function readableIdentifier(value: string | undefined): string | undefined {
const raw = value?.trim();
if (!raw || raw.length < 3 || raw.length > 80) return undefined;
if (/^[a-f0-9]{8,}$/i.test(raw)) return undefined;
const tokens = identifierTokens(raw);
if (GENERATED_PREFIXES.has(tokens[0])) return undefined;
while (tokens.length > 1 && GENERIC_TOKENS.has(tokens[0])) tokens.shift();
while (tokens.length > 1 && GENERIC_TOKENS.has(tokens[tokens.length - 1])) tokens.pop();
if (tokens.length === 0 || !tokens.some((token) => /[a-z]{3,}/.test(token))) return undefined;
return tokens
.map((token, index) => (index === 0 ? `${token[0].toUpperCase()}${token.slice(1)}` : token))
.join(" ");
}
export function stableIdentifierName(attrs: Readonly<Record<string, string>>): string | undefined {
for (const name of IDENTIFIER_ATTRS) {
const label = readableIdentifier(attrs[name]);
if (label) return label;
}
return undefined;
}
@@ -1,14 +1,11 @@
import {
applyVomInteractionRecovery,
isVomReferenceNode,
type VomNode,
type VomScene,
} from "@browser-skill/vom";
import type { ResolvedSemanticGraph, ResolvedSemanticNode, SemanticNodeId } from "./types";
import type { VomNode, VomScene } from "@browser-skill/vom";
import type { SemanticNodeId, StructuredSemanticGraph, StructuredSemanticNode } from "./types";
const TRANSPARENT_ROLES = new Set(["", "generic", "none", "presentation", "inlinetextbox"]);
function retained(node: StructuredSemanticNode): boolean {
return node.structure.kind === "keep" || node.structure.kind === "inferred-group";
}
function connectedFrameIds(graph: ResolvedSemanticGraph): Set<string> {
function connectedFrameIds(graph: StructuredSemanticGraph): Set<string> {
const connected = new Set<string>([graph.rootFrameId]);
const childrenByParent = new Map<string, string[]>();
for (const frame of graph.frames.values()) {
@@ -24,7 +21,7 @@ function connectedFrameIds(graph: ResolvedSemanticGraph): Set<string> {
const frame = graph.frames.get(frameId);
if (!frame?.ownerNodeId) continue;
const owner = graph.nodes.get(frame.ownerNodeId);
if (!owner || owner.excluded) continue;
if (!owner || !retained(owner)) continue;
connected.add(frame.frameId);
queue.push(frame.frameId);
}
@@ -32,121 +29,9 @@ function connectedFrameIds(graph: ResolvedSemanticGraph): Set<string> {
return connected;
}
function semanticParentId(
graph: ResolvedSemanticGraph,
node: ResolvedSemanticNode,
): SemanticNodeId | undefined {
let localParent = node.axParentId ?? node.domParentId;
if (node.axParentId && node.domParentId && node.axParentId !== node.domParentId) {
const axParent = graph.nodes.get(node.axParentId);
const axParentRole = axParent?.vom.role?.toLowerCase() ?? "";
const axParentIsRoot = axParentRole === "rootwebarea" || axParentRole === "webarea";
const axParentIsStructural =
!TRANSPARENT_ROLES.has(axParentRole) ||
Boolean(
axParent?.vom.name ||
axParent?.vom.text ||
(axParent ? hasMeaningfulRelations(axParent) : false),
);
localParent = !axParentIsRoot && axParentIsStructural ? node.axParentId : node.domParentId;
}
if (localParent) return localParent;
if (node.frameId === graph.rootFrameId) return undefined;
return graph.frames.get(node.frameId)?.ownerNodeId;
}
function isChildDocumentRoot(graph: ResolvedSemanticGraph, node: ResolvedSemanticNode): boolean {
if (node.frameId === graph.rootFrameId) return false;
if (semanticParentId(graph, node) !== graph.frames.get(node.frameId)?.ownerNodeId) return false;
const role = node.vom.role?.toLowerCase() ?? "";
return role === "rootwebarea" || role === "webarea";
}
function hasMeaningfulRelations(node: ResolvedSemanticNode): boolean {
const attrs = node.vom.attrs ?? {};
return ["aria-controls", "aria-labelledby", "aria-owns", "aria-describedby"].some((name) =>
attrs[name]?.trim(),
);
}
function shouldKeepNode(
graph: ResolvedSemanticGraph,
node: ResolvedSemanticNode,
recovered: VomNode,
): boolean {
if (node.excluded || isChildDocumentRoot(graph, node)) return false;
if (node.ax?.ignored) return false;
const role = recovered.role?.toLowerCase() ?? "";
if (!TRANSPARENT_ROLES.has(role)) return true;
return Boolean(
recovered.name ||
recovered.text ||
recovered.value ||
recovered.placeholder ||
hasMeaningfulRelations(node) ||
(recovered.rect && ["fixed", "sticky"].includes(recovered.position)),
);
}
function redundantSourceNodes(
graph: ResolvedSemanticGraph,
nodes: ResolvedSemanticNode[],
): Set<SemanticNodeId> {
const byKey = new Map<string, ResolvedSemanticNode[]>();
for (const node of nodes) {
const name = node.vom.name?.trim().toLowerCase() ?? "";
if (!name || !isVomReferenceNode({ ...node.vom, id: 0, parentId: null, referenceable: true })) {
continue;
}
const key = `${node.frameId}\u0000${semanticParentId(graph, node) ?? ""}\u0000${node.vom.role?.toLowerCase() ?? ""}\u0000${name}\u0000${node.vom.sensitive === true}`;
const matches = byKey.get(key) ?? [];
matches.push(node);
byKey.set(key, matches);
}
const duplicates = new Set<SemanticNodeId>();
for (const matches of byKey.values()) {
const axOnly = matches.filter((node) => node.ax && !node.dom);
const domOnly = matches.filter((node) => node.dom && !node.ax);
if (matches.length === 2 && axOnly.length === 1 && domOnly.length === 1) {
duplicates.add(domOnly[0].id);
}
}
return duplicates;
}
function nearestKeptParent(
graph: ResolvedSemanticGraph,
nodeId: SemanticNodeId,
kept: ReadonlySet<SemanticNodeId>,
memo: Map<SemanticNodeId, SemanticNodeId | undefined>,
): SemanticNodeId | undefined {
if (memo.has(nodeId)) return memo.get(nodeId);
let current = semanticParentId(graph, graph.nodes.get(nodeId) as ResolvedSemanticNode);
const seen = new Set<SemanticNodeId>();
const path: SemanticNodeId[] = [];
let result: SemanticNodeId | undefined;
while (current && !seen.has(current)) {
if (kept.has(current)) {
result = current;
break;
}
if (memo.has(current)) {
result = memo.get(current);
break;
}
seen.add(current);
path.push(current);
const parent = graph.nodes.get(current);
current = parent ? semanticParentId(graph, parent) : undefined;
}
memo.set(nodeId, result);
for (const id of path) memo.set(id, result);
return result;
}
function domAncestors(
graph: ResolvedSemanticGraph,
node: ResolvedSemanticNode,
graph: StructuredSemanticGraph,
node: StructuredSemanticNode,
numericId: ReadonlyMap<SemanticNodeId, number>,
): number[] {
const ancestors: number[] = [];
@@ -161,10 +46,7 @@ function domAncestors(
return ancestors;
}
export function projectSemanticGraph(graph: ResolvedSemanticGraph): VomScene {
const frameOwnerIds = new Set(
[...graph.frames.values()].flatMap((frame) => frame.ownerNodeId ?? []),
);
export function projectSemanticGraph(graph: StructuredSemanticGraph): VomScene {
const connectedFrames = connectedFrameIds(graph);
const orderedNodes = [...graph.frames.values()]
.filter((frame) => connectedFrames.has(frame.frameId))
@@ -172,67 +54,28 @@ export function projectSemanticGraph(graph: ResolvedSemanticGraph): VomScene {
.flatMap((frame) =>
(graph.nodeIdsByFrame.get(frame.frameId) ?? [])
.map((id) => graph.nodes.get(id))
.filter((node): node is ResolvedSemanticNode => node !== undefined)
.filter((node): node is StructuredSemanticNode => node !== undefined && retained(node))
.sort((a, b) => a.sourceOrder - b.sourceOrder),
);
const provisionalNumericId = new Map<SemanticNodeId, number>();
orderedNodes.forEach((node, index) => provisionalNumericId.set(node.id, index + 1));
const provisionalNodes: VomNode[] = orderedNodes.map((node) => ({
const numericId = new Map<SemanticNodeId, number>();
orderedNodes.forEach((node, index) => numericId.set(node.id, index + 1));
const nodes: VomNode[] = orderedNodes.map((node) => ({
...node.vom,
id: provisionalNumericId.get(node.id) as number,
parentId: (() => {
const parent = semanticParentId(graph, node);
return parent ? (provisionalNumericId.get(parent) ?? null) : null;
})(),
id: numericId.get(node.id) as number,
parentId: node.semanticParentId ? (numericId.get(node.semanticParentId) ?? null) : null,
...(node.backendNodeId !== undefined ? { backendNodeId: node.backendNodeId } : {}),
frameId: node.frameId,
contextScopeId: graph.frames.get(node.frameId)?.contextScopeId ?? node.frameId,
referenceable: node.referenceable,
...(node.dom
? {
domParentId:
node.domParentId === undefined ? null : (numericId.get(node.domParentId) ?? null),
domAncestorIds: domAncestors(graph, node, numericId),
}
: {}),
}));
const recoveredById = new Map(
applyVomInteractionRecovery(provisionalNodes).map((node) => [node.id, node]),
);
const duplicates = redundantSourceNodes(graph, orderedNodes);
const kept = new Set<SemanticNodeId>();
for (const node of orderedNodes) {
if (duplicates.has(node.id)) continue;
const id = provisionalNumericId.get(node.id) as number;
const recovered = recoveredById.get(id) as VomNode;
if ((!node.excluded && frameOwnerIds.has(node.id)) || shouldKeepNode(graph, node, recovered)) {
kept.add(node.id);
}
}
const numericId = new Map<SemanticNodeId, number>();
let nextId = 1;
for (const node of orderedNodes) {
if (!kept.has(node.id)) continue;
numericId.set(node.id, nextId);
nextId += 1;
}
const nodes: VomNode[] = [];
const keptParentMemo = new Map<SemanticNodeId, SemanticNodeId | undefined>();
for (const node of orderedNodes) {
const id = numericId.get(node.id);
if (id === undefined) continue;
const provisional = recoveredById.get(provisionalNumericId.get(node.id) as number) as VomNode;
const parent = nearestKeptParent(graph, node.id, kept, keptParentMemo);
const domParent = node.domParentId ? numericId.get(node.domParentId) : undefined;
nodes.push({
...provisional,
id,
parentId: parent ? (numericId.get(parent) ?? null) : null,
...(node.backendNodeId !== undefined ? { backendNodeId: node.backendNodeId } : {}),
...(node.dom
? {
domParentId: node.domParentId === undefined ? null : (domParent ?? null),
domAncestorIds: domAncestors(graph, node, numericId),
}
: {}),
});
}
return {
viewport: graph.viewport,
@@ -1,4 +1,5 @@
import type { VomNode } from "@browser-skill/vom";
import { stableIdentifierName } from "./name-evidence";
import type {
ResolvedSemanticGraph,
ResolvedSemanticNode,
@@ -7,6 +8,7 @@ import type {
SemanticGraphNode,
SemanticNodeId,
} from "./types";
import { frameBackendKey } from "./types";
const FORM_TAGS = new Set(["input", "textarea", "select"]);
const NATIVE_CONTROL_TAGS = new Set(["button", "input", "select", "textarea"]);
@@ -57,8 +59,6 @@ function normalizeRole(value: string | undefined): string | undefined {
}
function nativeRole(tag: string, attrs: Record<string, string>): string | undefined {
const explicit = normalizeRole(attrs.role);
if (explicit) return explicit;
if (tag === "button") return "button";
if (tag === "textarea") return "textbox";
if (tag === "select") return attrs.multiple === undefined ? "combobox" : "listbox";
@@ -90,14 +90,29 @@ function nativeRole(tag: string, attrs: Record<string, string>): string | undefi
return undefined;
}
function resolvedRole(node: SemanticGraphNode): string | undefined {
function domRole(tag: string, attrs: Record<string, string>): string | undefined {
return normalizeRole(attrs.role) ?? nativeRole(tag, attrs);
}
function resolvedRole(node: SemanticGraphNode): {
value?: string;
source: ResolvedSemanticNode["roleSource"];
} {
const rawAxRole = clean(axValue(node.ax?.role));
const axRole = normalizeRole(rawAxRole);
const domRole = nativeRole(node.dom?.tag.toLowerCase() ?? "", node.dom?.attrs ?? {});
if (!axRole || ["generic", "none", "presentation"].includes(axRole)) {
return domRole ?? rawAxRole;
const attrs = node.dom?.attrs ?? {};
const explicitDomRole = normalizeRole(attrs.role);
const nativeDomRole = nativeRole(node.dom?.tag.toLowerCase() ?? "", attrs);
const usableAxRole = node.ax?.ignored !== true;
if (usableAxRole && axRole && !["generic", "none", "presentation"].includes(axRole)) {
return { value: rawAxRole, source: "ax" };
}
return rawAxRole;
if (explicitDomRole) return { value: explicitDomRole, source: "dom-explicit" };
if (nativeDomRole) return { value: nativeDomRole, source: "dom-native" };
if (rawAxRole) {
return { value: rawAxRole, source: usableAxRole ? "ax" : "ax-ignored" };
}
return { source: "none" };
}
function sensitive(node: SemanticGraphNode): boolean {
@@ -143,14 +158,6 @@ function externalHrefHost(
}
}
function iconKeyword(value: string | undefined): string | undefined {
const token = clean(value)
?.split(/[#/.\s:_-]+/)
.filter(Boolean)
.pop();
return token && /^[a-z][a-z0-9_-]{1,32}$/i.test(token) ? token.toLowerCase() : undefined;
}
interface ResolveIndexes {
domChildren: Map<SemanticNodeId, SemanticNodeId[]>;
axChildren: Map<SemanticNodeId, SemanticNodeId[]>;
@@ -214,7 +221,7 @@ function buildIndexes(graph: SemanticGraph): ResolveIndexes {
for (const childId of domChildren.get(nodeId) ?? []) {
const child = graph.nodes.get(childId);
const childRole = child
? nativeRole(child.dom?.tag.toLowerCase() ?? "", child.dom?.attrs ?? {})
? domRole(child.dom?.tag.toLowerCase() ?? "", child.dom?.attrs ?? {})
: undefined;
if (childRole && INTERACTIVE_ROLES.has(childRole.toLowerCase())) continue;
const childText = domText(childId);
@@ -262,10 +269,8 @@ function buildIndexes(graph: SemanticGraph): ResolveIndexes {
for (const childId of domChildren.get(nodeId) ?? []) {
const child = graph.nodes.get(childId);
const childRole =
normalizeRole(axValue(child?.ax?.role)) ??
(child
? nativeRole(child.dom?.tag.toLowerCase() ?? "", child.dom?.attrs ?? {})
: undefined);
(child?.ax?.ignored === true ? undefined : normalizeRole(axValue(child?.ax?.role))) ??
(child ? domRole(child.dom?.tag.toLowerCase() ?? "", child.dom?.attrs ?? {}) : undefined);
if (
(child?.dom && NATIVE_CONTROL_TAGS.has(child.dom.tag.toLowerCase())) ||
INTERACTIVE_ROLES.has(childRole ?? "") ||
@@ -359,7 +364,7 @@ function precedingDomText(
for (let offset = 1; offset <= 4 && index - offset >= 0; offset += 1) {
const sibling = graph.nodes.get(siblings[index - offset]);
const siblingRole = sibling
? nativeRole(sibling.dom?.tag.toLowerCase() ?? "", sibling.dom?.attrs ?? {})
? domRole(sibling.dom?.tag.toLowerCase() ?? "", sibling.dom?.attrs ?? {})
: undefined;
if (siblingRole && INTERACTIVE_ROLES.has(siblingRole.toLowerCase())) break;
const text = indexes.domText(siblings[index - offset])?.replace(/[::]\s*$/, "");
@@ -368,49 +373,23 @@ function precedingDomText(
return undefined;
}
function iconHint(
nodeId: SemanticNodeId,
graph: SemanticGraph,
indexes: ResolveIndexes,
depth = 0,
): string | undefined {
if (depth > 3) return undefined;
for (const childId of indexes.domChildren.get(nodeId) ?? []) {
const child = graph.nodes.get(childId);
const attrs = child?.dom?.attrs ?? {};
const hint =
iconKeyword(attrs["aria-label"]) ??
iconKeyword(attrs.title) ??
iconKeyword(attrs.href) ??
iconKeyword(attrs["xlink:href"]) ??
iconKeyword(axValue(child?.ax?.name)) ??
iconKeyword(attrs.class) ??
iconKeyword(child?.dom?.textContent);
if (hint) return hint;
const nested = iconHint(childId, graph, indexes, depth + 1);
if (nested) return nested;
}
return undefined;
}
function resolvedName(
node: SemanticGraphNode,
role: string | undefined,
graph: SemanticGraph,
indexes: ResolveIndexes,
supplementalNames: ReadonlyMap<string, string>,
identifierFallback: boolean,
): string | undefined {
const attrs = node.dom?.attrs ?? {};
const formControl = FORM_TAGS.has(node.dom?.tag.toLowerCase() ?? "");
const descendantText = indexes.domText(node.id);
const canUseIconHint =
!indexes.hasNativeDescendant(node.id) &&
(INTERACTIVE_ROLES.has(role?.toLowerCase() ?? "") ||
node.dom?.cursor === "pointer" ||
attrs.onclick !== undefined ||
attrs.tabindex !== undefined);
const interactive =
INTERACTIVE_ROLES.has(role?.toLowerCase() ?? "") ||
NATIVE_CONTROL_TAGS.has(node.dom?.tag.toLowerCase() ?? "");
const candidates = [
axValue(node.ax?.name),
indexes.axText(node.id),
node.ax?.ignored === true ? undefined : axValue(node.ax?.name),
node.ax?.ignored === true ? undefined : indexes.axText(node.id),
ariaLabelledText(node, indexes),
clean(attrs["aria-label"]),
nativeLabelText(node, graph, indexes),
@@ -420,7 +399,10 @@ function resolvedName(
formControl ? precedingAxText(node, graph, indexes) : undefined,
formControl ? precedingDomText(node, graph, indexes) : undefined,
formControl ? clean(node.dom?.formPlaceholder ?? attrs.placeholder) : undefined,
canUseIconHint ? iconHint(node.id, graph, indexes) : undefined,
!interactive || node.backendNodeId === undefined
? undefined
: clean(supplementalNames.get(frameBackendKey(node.frameId, node.backendNodeId))),
identifierFallback && interactive ? stableIdentifierName(attrs) : undefined,
];
let name = candidates.find((candidate) => candidate !== undefined);
const hasPopup = axProperty(node.ax, "hasPopup");
@@ -462,12 +444,21 @@ function nearbyText(
return clean(texts.join(" "));
}
export function resolveSemanticGraph(graph: SemanticGraph): ResolvedSemanticGraph {
export function resolveSemanticGraph(
graph: SemanticGraph,
options: {
supplementalNames?: ReadonlyMap<string, string>;
identifierFallback?: boolean;
} = {},
): ResolvedSemanticGraph {
const indexes = buildIndexes(graph);
const supplementalNames = options.supplementalNames ?? new Map<string, string>();
const identifierFallback = options.identifierFallback ?? true;
const nodes = new Map<SemanticNodeId, ResolvedSemanticNode>();
for (const node of graph.nodes.values()) {
const role = resolvedRole(node);
const name = resolvedName(node, role, graph, indexes);
const roleResolution = resolvedRole(node);
const role = roleResolution.value;
const name = resolvedName(node, role, graph, indexes, supplementalNames, identifierFallback);
const isSensitive = sensitive(node);
const sourceValue = node.dom?.formValue ?? axValue(node.ax?.value);
const value = isSensitive ? undefined : sourceValue;
@@ -505,9 +496,9 @@ export function resolveSemanticGraph(graph: SemanticGraph): ResolvedSemanticGrap
insideNative: insideNative(node, graph),
...(role?.toLowerCase() === "link" ? { href: externalHrefHost(attrs.href, frame?.url) } : {}),
};
const selected = axProperty(node.ax, "selected");
const expanded = axProperty(node.ax, "expanded");
const controls = axProperty(node.ax, "controls");
const selected = node.ax?.ignored === true ? undefined : axProperty(node.ax, "selected");
const expanded = node.ax?.ignored === true ? undefined : axProperty(node.ax, "expanded");
const controls = node.ax?.ignored === true ? undefined : axProperty(node.ax, "controls");
if (selected === "true" && !vom.attrs?.["aria-selected"]) {
vom.attrs = { ...(vom.attrs ?? {}), "aria-selected": "true" };
}
@@ -521,6 +512,7 @@ export function resolveSemanticGraph(graph: SemanticGraph): ResolvedSemanticGrap
...node,
vom,
referenceable: node.backendNodeId !== undefined,
roleSource: roleResolution.source,
});
}
return { ...graph, nodes };
@@ -0,0 +1,436 @@
import {
applyVomInteractionRecovery,
isVomReferenceNode,
isVomStructuralRole,
type VomNode,
} from "@browser-skill/vom";
import type {
ResolvedSemanticGraph,
ResolvedSemanticNode,
SemanticNodeId,
StructureDecision,
StructuredSemanticGraph,
StructuredSemanticNode,
} from "./types";
const TRANSPARENT_ROLES = new Set(["", "generic", "none", "presentation", "inlinetextbox"]);
const ROOT_ROLES = new Set(["rootwebarea", "webarea"]);
const CONTENT_BOUNDARY_ROLES = new Set([
"article",
"dialog",
"document",
"feed",
"form",
"grid",
"heading",
"iframe",
"list",
"main",
"navigation",
"paragraph",
"region",
"section",
"table",
"tree",
]);
function hasMeaningfulRelations(node: ResolvedSemanticNode): boolean {
const attrs = node.vom.attrs ?? {};
return ["aria-controls", "aria-labelledby", "aria-owns", "aria-describedby"].some((name) =>
attrs[name]?.trim(),
);
}
function explicitStructural(node: ResolvedSemanticNode): boolean {
return (
isVomStructuralRole(node.vom.role) &&
node.roleSource !== "none" &&
node.roleSource !== "ax-ignored"
);
}
function initialParentId(
graph: ResolvedSemanticGraph,
node: ResolvedSemanticNode,
): SemanticNodeId | undefined {
let parentId = node.axParentId ?? node.domParentId;
if (node.axParentId && node.domParentId && node.axParentId !== node.domParentId) {
const axParent = graph.nodes.get(node.axParentId);
const axRole = axParent?.vom.role?.toLowerCase() ?? "";
const meaningfulAxParent =
axParent !== undefined &&
!ROOT_ROLES.has(axRole) &&
(explicitStructural(axParent) ||
!TRANSPARENT_ROLES.has(axRole) ||
Boolean(axParent.vom.name || axParent.vom.text || hasMeaningfulRelations(axParent)));
parentId = meaningfulAxParent ? node.axParentId : node.domParentId;
}
if (parentId) return parentId;
if (node.frameId === graph.rootFrameId) return undefined;
return graph.frames.get(node.frameId)?.ownerNodeId;
}
function orderedNodes(graph: ResolvedSemanticGraph): ResolvedSemanticNode[] {
return [...graph.frames.values()]
.sort((a, b) => a.order - b.order)
.flatMap((frame) =>
(graph.nodeIdsByFrame.get(frame.frameId) ?? [])
.map((id) => graph.nodes.get(id))
.filter((node): node is ResolvedSemanticNode => node !== undefined)
.sort((a, b) => a.sourceOrder - b.sourceOrder),
);
}
function recoveredNodes(
graph: ResolvedSemanticGraph,
nodes: ResolvedSemanticNode[],
): Map<SemanticNodeId, VomNode> {
const numericId = new Map(nodes.map((node, index) => [node.id, index + 1]));
const semanticId = new Map([...numericId].map(([id, numeric]) => [numeric, id]));
const provisional = nodes.map((node) => ({
...node.vom,
id: numericId.get(node.id) as number,
parentId: (() => {
const parent = initialParentId(graph, node);
return parent ? (numericId.get(parent) ?? null) : null;
})(),
...(node.backendNodeId !== undefined ? { backendNodeId: node.backendNodeId } : {}),
frameId: node.frameId,
contextScopeId: graph.frames.get(node.frameId)?.contextScopeId ?? node.frameId,
referenceable: node.referenceable,
}));
return new Map(
applyVomInteractionRecovery(provisional).flatMap((node) => {
const id = semanticId.get(node.id);
return id ? [[id, node] as const] : [];
}),
);
}
function redundantSourceNodes(
graph: ResolvedSemanticGraph,
nodes: ResolvedSemanticNode[],
recovered: ReadonlyMap<SemanticNodeId, VomNode>,
): Set<SemanticNodeId> {
const byKey = new Map<string, ResolvedSemanticNode[]>();
for (const node of nodes) {
const output = recovered.get(node.id) as VomNode;
const name = output.name?.trim().toLowerCase() ?? "";
if (!name || !isVomReferenceNode({ ...output, referenceable: true })) continue;
const key = `${node.frameId}\u0000${initialParentId(graph, node) ?? ""}\u0000${output.role?.toLowerCase() ?? ""}\u0000${name}\u0000${output.sensitive === true}`;
const matches = byKey.get(key) ?? [];
matches.push(node);
byKey.set(key, matches);
}
const redundant = new Set<SemanticNodeId>();
for (const matches of byKey.values()) {
const axOnly = matches.filter((node) => node.ax && !node.dom);
const domOnly = matches.filter((node) => node.dom && !node.ax);
if (matches.length === 2 && axOnly.length === 1 && domOnly.length === 1) {
redundant.add(domOnly[0].id);
}
}
return redundant;
}
function baseDecision(
graph: ResolvedSemanticGraph,
node: ResolvedSemanticNode,
output: VomNode,
frameOwnerIds: ReadonlySet<SemanticNodeId>,
redundant: ReadonlySet<SemanticNodeId>,
): StructureDecision {
if (node.excluded) return { kind: "excluded", reason: "excluded" };
if (redundant.has(node.id)) return { kind: "excluded", reason: "redundant-source" };
if (frameOwnerIds.has(node.id)) return { kind: "keep", reason: "frame-owner" };
if (node.frameId !== graph.rootFrameId && ROOT_ROLES.has(output.role?.toLowerCase() ?? "")) {
return { kind: "transparent", reason: "child-document-root" };
}
if (node.ax?.ignored === true && !node.dom) {
return { kind: "transparent", reason: "ax-ignored" };
}
const role = output.role?.toLowerCase() ?? "";
if (!TRANSPARENT_ROLES.has(role)) return { kind: "keep", reason: "semantic" };
if (
output.name ||
output.text ||
output.value ||
output.placeholder ||
hasMeaningfulRelations(node) ||
(output.rect && ["fixed", "sticky"].includes(output.position))
) {
return { kind: "keep", reason: "semantic" };
}
return { kind: "transparent", reason: "wrapper" };
}
function buildChildren(
nodes: ResolvedSemanticNode[],
relation: "axParentId" | "domParentId",
): Map<SemanticNodeId, SemanticNodeId[]> {
const children = new Map<SemanticNodeId, SemanticNodeId[]>();
for (const node of nodes) {
const parentId = node[relation];
if (!parentId) continue;
const siblings = children.get(parentId) ?? [];
siblings.push(node.id);
children.set(parentId, siblings);
}
return children;
}
function explicitDomAncestors(
graph: ResolvedSemanticGraph,
nodes: ResolvedSemanticNode[],
domChildren: ReadonlyMap<SemanticNodeId, SemanticNodeId[]>,
): Set<SemanticNodeId> {
const inside = new Set<SemanticNodeId>();
const roots = nodes.filter((node) => !node.domParentId);
const queue = roots.map((node) => node.id);
for (let index = 0; index < queue.length; index += 1) {
const parentId = queue[index];
const parent = graph.nodes.get(parentId);
const parentIsBoundary =
parent !== undefined &&
explicitStructural(parent) &&
!ROOT_ROLES.has(parent.vom.role?.toLowerCase() ?? "");
for (const childId of domChildren.get(parentId) ?? []) {
if (inside.has(parentId) || parentIsBoundary) inside.add(childId);
queue.push(childId);
}
}
return inside;
}
function reliableGroupCandidates(
graph: ResolvedSemanticGraph,
nodes: ResolvedSemanticNode[],
recovered: ReadonlyMap<SemanticNodeId, VomNode>,
decisions: ReadonlyMap<SemanticNodeId, StructureDecision>,
domChildren: ReadonlyMap<SemanticNodeId, SemanticNodeId[]>,
): Set<SemanticNodeId> {
const insideExplicitStructure = explicitDomAncestors(graph, nodes, domChildren);
const remainingChildren = new Map(
nodes.map((node) => [node.id, domChildren.get(node.id)?.length ?? 0]),
);
const summaries = new Map<SemanticNodeId, { blocked: boolean; interactions: number }>();
const queue = nodes
.filter((node) => (remainingChildren.get(node.id) ?? 0) === 0)
.map((node) => node.id);
for (let index = 0; index < queue.length; index += 1) {
const nodeId = queue[index];
const node = graph.nodes.get(nodeId);
const output = recovered.get(nodeId);
if (!node || !output) continue;
const role = output.role?.toLowerCase() ?? "";
const excluded = decisions.get(nodeId)?.kind === "excluded";
const blocked = excluded || explicitStructural(node) || CONTENT_BOUNDARY_ROLES.has(role);
const summary = excluded
? { blocked: true, interactions: 0 }
: isVomReferenceNode(output)
? { blocked: false, interactions: 1 }
: blocked
? { blocked: true, interactions: 0 }
: (domChildren.get(nodeId) ?? []).reduce(
(combined, childId) => {
const child = summaries.get(childId);
return {
blocked: combined.blocked || !child || child.blocked,
interactions: Math.min(2, combined.interactions + (child?.interactions ?? 0)),
};
},
{ blocked: false, interactions: 0 },
);
summaries.set(nodeId, summary);
const parentId = node.domParentId;
if (!parentId) continue;
const remaining = (remainingChildren.get(parentId) ?? 1) - 1;
remainingChildren.set(parentId, remaining);
if (remaining === 0) queue.push(parentId);
}
const candidates = new Set<SemanticNodeId>();
for (const node of nodes) {
const output = recovered.get(node.id) as VomNode;
const role = output.role?.toLowerCase() ?? "";
if (
!node.dom ||
node.dom.tag.toLowerCase() !== "div" ||
!TRANSPARENT_ROLES.has(role) ||
insideExplicitStructure.has(node.id) ||
decisions.get(node.id)?.kind === "excluded" ||
node.dom.attrs.hidden !== undefined ||
(node.dom.attrs["aria-hidden"] ?? "").toLowerCase() === "true" ||
node.dom.attrs.inert !== undefined
) {
continue;
}
const branches = (domChildren.get(node.id) ?? []).map((id) => summaries.get(id));
const blocked = branches.some((summary) => !summary || summary.blocked);
const contributingBranches = branches.filter(
(summary) => (summary?.interactions ?? 0) > 0,
).length;
const interactions = branches.reduce(
(count, summary) => Math.min(2, count + (summary?.interactions ?? 0)),
0,
);
if (!blocked && contributingBranches >= 2 && interactions >= 2) {
candidates.add(node.id);
}
}
return candidates;
}
function leafCandidates(
nodes: ResolvedSemanticNode[],
candidates: ReadonlySet<SemanticNodeId>,
domChildren: ReadonlyMap<SemanticNodeId, SemanticNodeId[]>,
): Set<SemanticNodeId> {
const parentById = new Map(
nodes.flatMap((node) => (node.domParentId ? ([[node.id, node.domParentId]] as const) : [])),
);
const childCount = new Map(nodes.map((node) => [node.id, domChildren.get(node.id)?.length ?? 0]));
const containsCandidate = new Set<SemanticNodeId>();
const queue = nodes.filter((node) => (childCount.get(node.id) ?? 0) === 0).map((node) => node.id);
const selected = new Set<SemanticNodeId>();
for (let index = 0; index < queue.length; index += 1) {
const nodeId = queue[index];
if (candidates.has(nodeId) && !containsCandidate.has(nodeId)) selected.add(nodeId);
const parentId = parentById.get(nodeId);
if (!parentId) continue;
if (candidates.has(nodeId) || containsCandidate.has(nodeId)) containsCandidate.add(parentId);
const remaining = (childCount.get(parentId) ?? 1) - 1;
childCount.set(parentId, remaining);
if (remaining === 0) queue.push(parentId);
}
return selected;
}
function nearestKept(
graph: ResolvedSemanticGraph,
startId: SemanticNodeId | undefined,
relation: "axParentId" | "domParentId",
kept: ReadonlySet<SemanticNodeId>,
memo: Map<SemanticNodeId, SemanticNodeId | undefined>,
): SemanticNodeId | undefined {
let currentId = startId;
const path: SemanticNodeId[] = [];
const seen = new Set<SemanticNodeId>();
let result: SemanticNodeId | undefined;
while (currentId && !seen.has(currentId)) {
if (kept.has(currentId)) {
result = currentId;
break;
}
if (memo.has(currentId)) {
result = memo.get(currentId);
break;
}
seen.add(currentId);
path.push(currentId);
currentId = graph.nodes.get(currentId)?.[relation];
}
for (const id of path) memo.set(id, result);
return result;
}
function finalParentId(
graph: ResolvedSemanticGraph,
node: ResolvedSemanticNode,
kept: ReadonlySet<SemanticNodeId>,
decisions: ReadonlyMap<SemanticNodeId, StructureDecision>,
axParentMemo: Map<SemanticNodeId, SemanticNodeId | undefined>,
domParentMemo: Map<SemanticNodeId, SemanticNodeId | undefined>,
): SemanticNodeId | undefined {
const axParent = nearestKept(graph, node.axParentId, "axParentId", kept, axParentMemo);
const domParent = nearestKept(graph, node.domParentId, "domParentId", kept, domParentMemo);
const axNode = axParent ? graph.nodes.get(axParent) : undefined;
const domNode = domParent ? graph.nodes.get(domParent) : undefined;
const axIsRoot = ROOT_ROLES.has(axNode?.vom.role?.toLowerCase() ?? "");
const axIsExplicit = axNode ? explicitStructural(axNode) && !axIsRoot : false;
const domIsStructural =
domNode !== undefined &&
(explicitStructural(domNode) || decisions.get(domNode.id)?.kind === "inferred-group");
if (axIsExplicit) return axParent;
if (domIsStructural) return domParent;
if (axParent && !axIsRoot) return axParent;
if (domParent) return domParent;
if (axParent) return axParent;
if (node.frameId === graph.rootFrameId) return undefined;
return graph.frames.get(node.frameId)?.ownerNodeId;
}
function recoveredVom(node: VomNode, inferredGroup: boolean): ResolvedSemanticNode["vom"] {
const {
id: _id,
parentId: _parentId,
backendNodeId: _backendNodeId,
frameId: _frameId,
contextScopeId: _contextScopeId,
referenceable: _referenceable,
...vom
} = node;
if (!inferredGroup) return vom;
const { name: _name, text: _text, nearbyText: _nearbyText, ...groupVom } = vom;
return { ...groupVom, role: "group" };
}
export function normalizeSemanticStructure(graph: ResolvedSemanticGraph): StructuredSemanticGraph {
const nodes = orderedNodes(graph);
const recovered = recoveredNodes(graph, nodes);
const redundant = redundantSourceNodes(graph, nodes, recovered);
const frameOwnerIds = new Set(
[...graph.frames.values()].flatMap((frame) => frame.ownerNodeId ?? []),
);
const decisions = new Map<SemanticNodeId, StructureDecision>();
for (const node of nodes) {
decisions.set(
node.id,
baseDecision(graph, node, recovered.get(node.id) as VomNode, frameOwnerIds, redundant),
);
}
const domChildren = buildChildren(nodes, "domParentId");
const candidates = reliableGroupCandidates(graph, nodes, recovered, decisions, domChildren);
for (const nodeId of leafCandidates(nodes, candidates, domChildren)) {
decisions.set(nodeId, {
kind: "inferred-group",
reason: "independent-interaction-branches",
});
}
const kept = new Set(
nodes
.filter((node) => {
const kind = decisions.get(node.id)?.kind;
return kind === "keep" || kind === "inferred-group";
})
.map((node) => node.id),
);
const axParentMemo = new Map<SemanticNodeId, SemanticNodeId | undefined>();
const domParentMemo = new Map<SemanticNodeId, SemanticNodeId | undefined>();
const structuredNodes = new Map<SemanticNodeId, StructuredSemanticNode>();
for (const node of nodes) {
const structure = decisions.get(node.id) as StructureDecision;
const inferredGroup = structure.kind === "inferred-group";
structuredNodes.set(node.id, {
...node,
vom: recoveredVom(recovered.get(node.id) as VomNode, inferredGroup),
structure,
...(kept.has(node.id)
? {
semanticParentId: finalParentId(
graph,
node,
kept,
decisions,
axParentMemo,
domParentMemo,
),
}
: {}),
});
}
return { ...graph, nodes: structuredNodes };
}
@@ -54,12 +54,28 @@ export interface SemanticGraph {
export interface ResolvedSemanticNode extends SemanticGraphNode {
vom: Omit<VomNode, "id" | "parentId" | "backendNodeId" | "frameId" | "contextScopeId">;
referenceable: boolean;
roleSource: "ax" | "ax-ignored" | "dom-explicit" | "dom-native" | "none";
}
export interface ResolvedSemanticGraph extends Omit<SemanticGraph, "nodes"> {
nodes: Map<SemanticNodeId, ResolvedSemanticNode>;
}
export type StructureDecision =
| { kind: "keep"; reason: "semantic" | "frame-owner" }
| { kind: "inferred-group"; reason: "independent-interaction-branches" }
| { kind: "transparent"; reason: "wrapper" | "child-document-root" | "ax-ignored" }
| { kind: "excluded"; reason: "excluded" | "redundant-source" };
export interface StructuredSemanticNode extends ResolvedSemanticNode {
structure: StructureDecision;
semanticParentId?: SemanticNodeId;
}
export interface StructuredSemanticGraph extends Omit<ResolvedSemanticGraph, "nodes"> {
nodes: Map<SemanticNodeId, StructuredSemanticNode>;
}
export function frameBackendKey(frameId: string, backendNodeId: number): string {
return `${frameId}\u0000${backendNodeId}`;
}