feat(vom): fix name lost

This commit is contained in:
Ljy-0827 authored and GitHub committed 2026-08-26 15:28:27 +08:00
1 parent 41cb576256
commit b5062ac19b
15 files changed
+1588 -304

No files matched your search

@@ -1,14 +1,11 @@
import {
applyVomInteractionRecovery,
isVomReferenceNode,
type VomNode,
type VomScene,
} from "@browser-skill/vom";
import type { ResolvedSemanticGraph, ResolvedSemanticNode, SemanticNodeId } from "./types";
import type { VomNode, VomScene } from "@browser-skill/vom";
import type { SemanticNodeId, StructuredSemanticGraph, StructuredSemanticNode } from "./types";
const TRANSPARENT_ROLES = new Set(["", "generic", "none", "presentation", "inlinetextbox"]);
function retained(node: StructuredSemanticNode): boolean {
return node.structure.kind === "keep" || node.structure.kind === "inferred-group";
}
function connectedFrameIds(graph: ResolvedSemanticGraph): Set<string> {
function connectedFrameIds(graph: StructuredSemanticGraph): Set<string> {
const connected = new Set<string>([graph.rootFrameId]);
const childrenByParent = new Map<string, string[]>();
for (const frame of graph.frames.values()) {
@@ -24,7 +21,7 @@ function connectedFrameIds(graph: ResolvedSemanticGraph): Set<string> {
const frame = graph.frames.get(frameId);
if (!frame?.ownerNodeId) continue;
const owner = graph.nodes.get(frame.ownerNodeId);
if (!owner || owner.excluded) continue;
if (!owner || !retained(owner)) continue;
connected.add(frame.frameId);
queue.push(frame.frameId);
}
@@ -32,121 +29,9 @@ function connectedFrameIds(graph: ResolvedSemanticGraph): Set<string> {
return connected;
}
function semanticParentId(
graph: ResolvedSemanticGraph,
node: ResolvedSemanticNode,
): SemanticNodeId | undefined {
let localParent = node.axParentId ?? node.domParentId;
if (node.axParentId && node.domParentId && node.axParentId !== node.domParentId) {
const axParent = graph.nodes.get(node.axParentId);
const axParentRole = axParent?.vom.role?.toLowerCase() ?? "";
const axParentIsRoot = axParentRole === "rootwebarea" || axParentRole === "webarea";
const axParentIsStructural =
!TRANSPARENT_ROLES.has(axParentRole) ||
Boolean(
axParent?.vom.name ||
axParent?.vom.text ||
(axParent ? hasMeaningfulRelations(axParent) : false),
);
localParent = !axParentIsRoot && axParentIsStructural ? node.axParentId : node.domParentId;
}
if (localParent) return localParent;
if (node.frameId === graph.rootFrameId) return undefined;
return graph.frames.get(node.frameId)?.ownerNodeId;
}
function isChildDocumentRoot(graph: ResolvedSemanticGraph, node: ResolvedSemanticNode): boolean {
if (node.frameId === graph.rootFrameId) return false;
if (semanticParentId(graph, node) !== graph.frames.get(node.frameId)?.ownerNodeId) return false;
const role = node.vom.role?.toLowerCase() ?? "";
return role === "rootwebarea" || role === "webarea";
}
function hasMeaningfulRelations(node: ResolvedSemanticNode): boolean {
const attrs = node.vom.attrs ?? {};
return ["aria-controls", "aria-labelledby", "aria-owns", "aria-describedby"].some((name) =>
attrs[name]?.trim(),
);
}
function shouldKeepNode(
graph: ResolvedSemanticGraph,
node: ResolvedSemanticNode,
recovered: VomNode,
): boolean {
if (node.excluded || isChildDocumentRoot(graph, node)) return false;
if (node.ax?.ignored) return false;
const role = recovered.role?.toLowerCase() ?? "";
if (!TRANSPARENT_ROLES.has(role)) return true;
return Boolean(
recovered.name ||
recovered.text ||
recovered.value ||
recovered.placeholder ||
hasMeaningfulRelations(node) ||
(recovered.rect && ["fixed", "sticky"].includes(recovered.position)),
);
}
function redundantSourceNodes(
graph: ResolvedSemanticGraph,
nodes: ResolvedSemanticNode[],
): Set<SemanticNodeId> {
const byKey = new Map<string, ResolvedSemanticNode[]>();
for (const node of nodes) {
const name = node.vom.name?.trim().toLowerCase() ?? "";
if (!name || !isVomReferenceNode({ ...node.vom, id: 0, parentId: null, referenceable: true })) {
continue;
}
const key = `${node.frameId}\u0000${semanticParentId(graph, node) ?? ""}\u0000${node.vom.role?.toLowerCase() ?? ""}\u0000${name}\u0000${node.vom.sensitive === true}`;
const matches = byKey.get(key) ?? [];
matches.push(node);
byKey.set(key, matches);
}
const duplicates = new Set<SemanticNodeId>();
for (const matches of byKey.values()) {
const axOnly = matches.filter((node) => node.ax && !node.dom);
const domOnly = matches.filter((node) => node.dom && !node.ax);
if (matches.length === 2 && axOnly.length === 1 && domOnly.length === 1) {
duplicates.add(domOnly[0].id);
}
}
return duplicates;
}
function nearestKeptParent(
graph: ResolvedSemanticGraph,
nodeId: SemanticNodeId,
kept: ReadonlySet<SemanticNodeId>,
memo: Map<SemanticNodeId, SemanticNodeId | undefined>,
): SemanticNodeId | undefined {
if (memo.has(nodeId)) return memo.get(nodeId);
let current = semanticParentId(graph, graph.nodes.get(nodeId) as ResolvedSemanticNode);
const seen = new Set<SemanticNodeId>();
const path: SemanticNodeId[] = [];
let result: SemanticNodeId | undefined;
while (current && !seen.has(current)) {
if (kept.has(current)) {
result = current;
break;
}
if (memo.has(current)) {
result = memo.get(current);
break;
}
seen.add(current);
path.push(current);
const parent = graph.nodes.get(current);
current = parent ? semanticParentId(graph, parent) : undefined;
}
memo.set(nodeId, result);
for (const id of path) memo.set(id, result);
return result;
}
function domAncestors(
graph: ResolvedSemanticGraph,
node: ResolvedSemanticNode,
graph: StructuredSemanticGraph,
node: StructuredSemanticNode,
numericId: ReadonlyMap<SemanticNodeId, number>,
): number[] {
const ancestors: number[] = [];
@@ -161,10 +46,7 @@ function domAncestors(
return ancestors;
}
export function projectSemanticGraph(graph: ResolvedSemanticGraph): VomScene {
const frameOwnerIds = new Set(
[...graph.frames.values()].flatMap((frame) => frame.ownerNodeId ?? []),
);
export function projectSemanticGraph(graph: StructuredSemanticGraph): VomScene {
const connectedFrames = connectedFrameIds(graph);
const orderedNodes = [...graph.frames.values()]
.filter((frame) => connectedFrames.has(frame.frameId))
@@ -172,67 +54,28 @@ export function projectSemanticGraph(graph: ResolvedSemanticGraph): VomScene {
.flatMap((frame) =>
(graph.nodeIdsByFrame.get(frame.frameId) ?? [])
.map((id) => graph.nodes.get(id))
.filter((node): node is ResolvedSemanticNode => node !== undefined)
.filter((node): node is StructuredSemanticNode => node !== undefined && retained(node))
.sort((a, b) => a.sourceOrder - b.sourceOrder),
);
const provisionalNumericId = new Map<SemanticNodeId, number>();
orderedNodes.forEach((node, index) => provisionalNumericId.set(node.id, index + 1));
const provisionalNodes: VomNode[] = orderedNodes.map((node) => ({
const numericId = new Map<SemanticNodeId, number>();
orderedNodes.forEach((node, index) => numericId.set(node.id, index + 1));
const nodes: VomNode[] = orderedNodes.map((node) => ({
...node.vom,
id: provisionalNumericId.get(node.id) as number,
parentId: (() => {
const parent = semanticParentId(graph, node);
return parent ? (provisionalNumericId.get(parent) ?? null) : null;
})(),
id: numericId.get(node.id) as number,
parentId: node.semanticParentId ? (numericId.get(node.semanticParentId) ?? null) : null,
...(node.backendNodeId !== undefined ? { backendNodeId: node.backendNodeId } : {}),
frameId: node.frameId,
contextScopeId: graph.frames.get(node.frameId)?.contextScopeId ?? node.frameId,
referenceable: node.referenceable,
...(node.dom
? {
domParentId:
node.domParentId === undefined ? null : (numericId.get(node.domParentId) ?? null),
domAncestorIds: domAncestors(graph, node, numericId),
}
: {}),
}));
const recoveredById = new Map(
applyVomInteractionRecovery(provisionalNodes).map((node) => [node.id, node]),
);
const duplicates = redundantSourceNodes(graph, orderedNodes);
const kept = new Set<SemanticNodeId>();
for (const node of orderedNodes) {
if (duplicates.has(node.id)) continue;
const id = provisionalNumericId.get(node.id) as number;
const recovered = recoveredById.get(id) as VomNode;
if ((!node.excluded && frameOwnerIds.has(node.id)) || shouldKeepNode(graph, node, recovered)) {
kept.add(node.id);
}
}
const numericId = new Map<SemanticNodeId, number>();
let nextId = 1;
for (const node of orderedNodes) {
if (!kept.has(node.id)) continue;
numericId.set(node.id, nextId);
nextId += 1;
}
const nodes: VomNode[] = [];
const keptParentMemo = new Map<SemanticNodeId, SemanticNodeId | undefined>();
for (const node of orderedNodes) {
const id = numericId.get(node.id);
if (id === undefined) continue;
const provisional = recoveredById.get(provisionalNumericId.get(node.id) as number) as VomNode;
const parent = nearestKeptParent(graph, node.id, kept, keptParentMemo);
const domParent = node.domParentId ? numericId.get(node.domParentId) : undefined;
nodes.push({
...provisional,
id,
parentId: parent ? (numericId.get(parent) ?? null) : null,
...(node.backendNodeId !== undefined ? { backendNodeId: node.backendNodeId } : {}),
...(node.dom
? {
domParentId: node.domParentId === undefined ? null : (domParent ?? null),
domAncestorIds: domAncestors(graph, node, numericId),
}
: {}),
});
}
return {
viewport: graph.viewport,