mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(vom): fix name lost
This commit is contained in:
1 parent
41cb576256
commit
b5062ac19b
15 files changed
+1588
-304
No files matched your search
@@ -1,25 +1,58 @@
|
||||
import type { Viewport, VomScene } from "@browser-skill/vom";
|
||||
import type { FrameDocument } from "../frame-document";
|
||||
import { buildSemanticGraph } from "./build";
|
||||
import { projectSemanticGraph } from "./project";
|
||||
import { resolveSemanticGraph } from "./resolve";
|
||||
import type { SemanticAxNode } from "./types";
|
||||
import { buildSemanticGraph as buildGraph } from "./build";
|
||||
import { projectSemanticGraph as projectGraph } from "./project";
|
||||
import { resolveSemanticGraph as resolveGraph } from "./resolve";
|
||||
import { normalizeSemanticStructure as normalizeStructure } from "./structure";
|
||||
import type {
|
||||
ResolvedSemanticGraph,
|
||||
SemanticAxNode,
|
||||
SemanticGraph,
|
||||
StructuredSemanticGraph,
|
||||
} from "./types";
|
||||
|
||||
export type { SemanticAxNode } from "./types";
|
||||
|
||||
export function buildSemanticVomScene(input: {
|
||||
export interface SemanticVomInput {
|
||||
documents: FrameDocument<SemanticAxNode>[];
|
||||
viewport: Viewport;
|
||||
rootFrameId?: string;
|
||||
excludedBackendNodeIds?: ReadonlySet<number>;
|
||||
}): VomScene {
|
||||
}
|
||||
|
||||
export function buildSemanticGraph(input: SemanticVomInput): SemanticGraph {
|
||||
return buildGraph(input);
|
||||
}
|
||||
|
||||
export function resolveSemanticGraph(
|
||||
graph: SemanticGraph,
|
||||
options: {
|
||||
supplementalNames?: ReadonlyMap<string, string>;
|
||||
identifierFallback?: boolean;
|
||||
} = {},
|
||||
): ResolvedSemanticGraph {
|
||||
return resolveGraph(graph, options);
|
||||
}
|
||||
|
||||
export function normalizeSemanticStructure(graph: ResolvedSemanticGraph): StructuredSemanticGraph {
|
||||
return normalizeStructure(graph);
|
||||
}
|
||||
|
||||
export function projectSemanticGraph(graph: StructuredSemanticGraph): VomScene {
|
||||
return projectGraph(graph);
|
||||
}
|
||||
|
||||
export function buildSemanticVomScene(
|
||||
input: SemanticVomInput & {
|
||||
supplementalNames?: ReadonlyMap<string, string>;
|
||||
identifierFallback?: boolean;
|
||||
},
|
||||
): VomScene {
|
||||
return projectSemanticGraph(
|
||||
resolveSemanticGraph(
|
||||
buildSemanticGraph({
|
||||
documents: input.documents,
|
||||
viewport: input.viewport,
|
||||
rootFrameId: input.rootFrameId,
|
||||
excludedBackendNodeIds: input.excludedBackendNodeIds,
|
||||
normalizeSemanticStructure(
|
||||
resolveSemanticGraph(buildSemanticGraph(input), {
|
||||
supplementalNames: input.supplementalNames,
|
||||
identifierFallback: input.identifierFallback,
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
const IDENTIFIER_ATTRS = ["data-testid", "data-test-id", "id"] as const;
|
||||
const GENERIC_TOKENS = new Set(["btn", "button", "control", "icon", "item", "test", "tool"]);
|
||||
const GENERATED_PREFIXES = new Set(["ember", "headlessui", "mui", "radix", "react", "rc"]);
|
||||
|
||||
function identifierTokens(value: string): string[] {
|
||||
return value
|
||||
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
|
||||
.split(/[^a-zA-Z0-9]+/)
|
||||
.map((token) => token.toLowerCase())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function readableIdentifier(value: string | undefined): string | undefined {
|
||||
const raw = value?.trim();
|
||||
if (!raw || raw.length < 3 || raw.length > 80) return undefined;
|
||||
if (/^[a-f0-9]{8,}$/i.test(raw)) return undefined;
|
||||
|
||||
const tokens = identifierTokens(raw);
|
||||
if (GENERATED_PREFIXES.has(tokens[0])) return undefined;
|
||||
while (tokens.length > 1 && GENERIC_TOKENS.has(tokens[0])) tokens.shift();
|
||||
while (tokens.length > 1 && GENERIC_TOKENS.has(tokens[tokens.length - 1])) tokens.pop();
|
||||
if (tokens.length === 0 || !tokens.some((token) => /[a-z]{3,}/.test(token))) return undefined;
|
||||
return tokens
|
||||
.map((token, index) => (index === 0 ? `${token[0].toUpperCase()}${token.slice(1)}` : token))
|
||||
.join(" ");
|
||||
}
|
||||
|
||||
export function stableIdentifierName(attrs: Readonly<Record<string, string>>): string | undefined {
|
||||
for (const name of IDENTIFIER_ATTRS) {
|
||||
const label = readableIdentifier(attrs[name]);
|
||||
if (label) return label;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@@ -1,14 +1,11 @@
|
||||
import {
|
||||
applyVomInteractionRecovery,
|
||||
isVomReferenceNode,
|
||||
type VomNode,
|
||||
type VomScene,
|
||||
} from "@browser-skill/vom";
|
||||
import type { ResolvedSemanticGraph, ResolvedSemanticNode, SemanticNodeId } from "./types";
|
||||
import type { VomNode, VomScene } from "@browser-skill/vom";
|
||||
import type { SemanticNodeId, StructuredSemanticGraph, StructuredSemanticNode } from "./types";
|
||||
|
||||
const TRANSPARENT_ROLES = new Set(["", "generic", "none", "presentation", "inlinetextbox"]);
|
||||
function retained(node: StructuredSemanticNode): boolean {
|
||||
return node.structure.kind === "keep" || node.structure.kind === "inferred-group";
|
||||
}
|
||||
|
||||
function connectedFrameIds(graph: ResolvedSemanticGraph): Set<string> {
|
||||
function connectedFrameIds(graph: StructuredSemanticGraph): Set<string> {
|
||||
const connected = new Set<string>([graph.rootFrameId]);
|
||||
const childrenByParent = new Map<string, string[]>();
|
||||
for (const frame of graph.frames.values()) {
|
||||
@@ -24,7 +21,7 @@ function connectedFrameIds(graph: ResolvedSemanticGraph): Set<string> {
|
||||
const frame = graph.frames.get(frameId);
|
||||
if (!frame?.ownerNodeId) continue;
|
||||
const owner = graph.nodes.get(frame.ownerNodeId);
|
||||
if (!owner || owner.excluded) continue;
|
||||
if (!owner || !retained(owner)) continue;
|
||||
connected.add(frame.frameId);
|
||||
queue.push(frame.frameId);
|
||||
}
|
||||
@@ -32,121 +29,9 @@ function connectedFrameIds(graph: ResolvedSemanticGraph): Set<string> {
|
||||
return connected;
|
||||
}
|
||||
|
||||
function semanticParentId(
|
||||
graph: ResolvedSemanticGraph,
|
||||
node: ResolvedSemanticNode,
|
||||
): SemanticNodeId | undefined {
|
||||
let localParent = node.axParentId ?? node.domParentId;
|
||||
if (node.axParentId && node.domParentId && node.axParentId !== node.domParentId) {
|
||||
const axParent = graph.nodes.get(node.axParentId);
|
||||
const axParentRole = axParent?.vom.role?.toLowerCase() ?? "";
|
||||
const axParentIsRoot = axParentRole === "rootwebarea" || axParentRole === "webarea";
|
||||
const axParentIsStructural =
|
||||
!TRANSPARENT_ROLES.has(axParentRole) ||
|
||||
Boolean(
|
||||
axParent?.vom.name ||
|
||||
axParent?.vom.text ||
|
||||
(axParent ? hasMeaningfulRelations(axParent) : false),
|
||||
);
|
||||
localParent = !axParentIsRoot && axParentIsStructural ? node.axParentId : node.domParentId;
|
||||
}
|
||||
if (localParent) return localParent;
|
||||
if (node.frameId === graph.rootFrameId) return undefined;
|
||||
return graph.frames.get(node.frameId)?.ownerNodeId;
|
||||
}
|
||||
|
||||
function isChildDocumentRoot(graph: ResolvedSemanticGraph, node: ResolvedSemanticNode): boolean {
|
||||
if (node.frameId === graph.rootFrameId) return false;
|
||||
if (semanticParentId(graph, node) !== graph.frames.get(node.frameId)?.ownerNodeId) return false;
|
||||
const role = node.vom.role?.toLowerCase() ?? "";
|
||||
return role === "rootwebarea" || role === "webarea";
|
||||
}
|
||||
|
||||
function hasMeaningfulRelations(node: ResolvedSemanticNode): boolean {
|
||||
const attrs = node.vom.attrs ?? {};
|
||||
return ["aria-controls", "aria-labelledby", "aria-owns", "aria-describedby"].some((name) =>
|
||||
attrs[name]?.trim(),
|
||||
);
|
||||
}
|
||||
|
||||
function shouldKeepNode(
|
||||
graph: ResolvedSemanticGraph,
|
||||
node: ResolvedSemanticNode,
|
||||
recovered: VomNode,
|
||||
): boolean {
|
||||
if (node.excluded || isChildDocumentRoot(graph, node)) return false;
|
||||
if (node.ax?.ignored) return false;
|
||||
const role = recovered.role?.toLowerCase() ?? "";
|
||||
if (!TRANSPARENT_ROLES.has(role)) return true;
|
||||
return Boolean(
|
||||
recovered.name ||
|
||||
recovered.text ||
|
||||
recovered.value ||
|
||||
recovered.placeholder ||
|
||||
hasMeaningfulRelations(node) ||
|
||||
(recovered.rect && ["fixed", "sticky"].includes(recovered.position)),
|
||||
);
|
||||
}
|
||||
|
||||
function redundantSourceNodes(
|
||||
graph: ResolvedSemanticGraph,
|
||||
nodes: ResolvedSemanticNode[],
|
||||
): Set<SemanticNodeId> {
|
||||
const byKey = new Map<string, ResolvedSemanticNode[]>();
|
||||
for (const node of nodes) {
|
||||
const name = node.vom.name?.trim().toLowerCase() ?? "";
|
||||
if (!name || !isVomReferenceNode({ ...node.vom, id: 0, parentId: null, referenceable: true })) {
|
||||
continue;
|
||||
}
|
||||
const key = `${node.frameId}\u0000${semanticParentId(graph, node) ?? ""}\u0000${node.vom.role?.toLowerCase() ?? ""}\u0000${name}\u0000${node.vom.sensitive === true}`;
|
||||
const matches = byKey.get(key) ?? [];
|
||||
matches.push(node);
|
||||
byKey.set(key, matches);
|
||||
}
|
||||
const duplicates = new Set<SemanticNodeId>();
|
||||
for (const matches of byKey.values()) {
|
||||
const axOnly = matches.filter((node) => node.ax && !node.dom);
|
||||
const domOnly = matches.filter((node) => node.dom && !node.ax);
|
||||
if (matches.length === 2 && axOnly.length === 1 && domOnly.length === 1) {
|
||||
duplicates.add(domOnly[0].id);
|
||||
}
|
||||
}
|
||||
return duplicates;
|
||||
}
|
||||
|
||||
function nearestKeptParent(
|
||||
graph: ResolvedSemanticGraph,
|
||||
nodeId: SemanticNodeId,
|
||||
kept: ReadonlySet<SemanticNodeId>,
|
||||
memo: Map<SemanticNodeId, SemanticNodeId | undefined>,
|
||||
): SemanticNodeId | undefined {
|
||||
if (memo.has(nodeId)) return memo.get(nodeId);
|
||||
let current = semanticParentId(graph, graph.nodes.get(nodeId) as ResolvedSemanticNode);
|
||||
const seen = new Set<SemanticNodeId>();
|
||||
const path: SemanticNodeId[] = [];
|
||||
let result: SemanticNodeId | undefined;
|
||||
while (current && !seen.has(current)) {
|
||||
if (kept.has(current)) {
|
||||
result = current;
|
||||
break;
|
||||
}
|
||||
if (memo.has(current)) {
|
||||
result = memo.get(current);
|
||||
break;
|
||||
}
|
||||
seen.add(current);
|
||||
path.push(current);
|
||||
const parent = graph.nodes.get(current);
|
||||
current = parent ? semanticParentId(graph, parent) : undefined;
|
||||
}
|
||||
memo.set(nodeId, result);
|
||||
for (const id of path) memo.set(id, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
function domAncestors(
|
||||
graph: ResolvedSemanticGraph,
|
||||
node: ResolvedSemanticNode,
|
||||
graph: StructuredSemanticGraph,
|
||||
node: StructuredSemanticNode,
|
||||
numericId: ReadonlyMap<SemanticNodeId, number>,
|
||||
): number[] {
|
||||
const ancestors: number[] = [];
|
||||
@@ -161,10 +46,7 @@ function domAncestors(
|
||||
return ancestors;
|
||||
}
|
||||
|
||||
export function projectSemanticGraph(graph: ResolvedSemanticGraph): VomScene {
|
||||
const frameOwnerIds = new Set(
|
||||
[...graph.frames.values()].flatMap((frame) => frame.ownerNodeId ?? []),
|
||||
);
|
||||
export function projectSemanticGraph(graph: StructuredSemanticGraph): VomScene {
|
||||
const connectedFrames = connectedFrameIds(graph);
|
||||
const orderedNodes = [...graph.frames.values()]
|
||||
.filter((frame) => connectedFrames.has(frame.frameId))
|
||||
@@ -172,67 +54,28 @@ export function projectSemanticGraph(graph: ResolvedSemanticGraph): VomScene {
|
||||
.flatMap((frame) =>
|
||||
(graph.nodeIdsByFrame.get(frame.frameId) ?? [])
|
||||
.map((id) => graph.nodes.get(id))
|
||||
.filter((node): node is ResolvedSemanticNode => node !== undefined)
|
||||
.filter((node): node is StructuredSemanticNode => node !== undefined && retained(node))
|
||||
.sort((a, b) => a.sourceOrder - b.sourceOrder),
|
||||
);
|
||||
|
||||
const provisionalNumericId = new Map<SemanticNodeId, number>();
|
||||
orderedNodes.forEach((node, index) => provisionalNumericId.set(node.id, index + 1));
|
||||
const provisionalNodes: VomNode[] = orderedNodes.map((node) => ({
|
||||
const numericId = new Map<SemanticNodeId, number>();
|
||||
orderedNodes.forEach((node, index) => numericId.set(node.id, index + 1));
|
||||
const nodes: VomNode[] = orderedNodes.map((node) => ({
|
||||
...node.vom,
|
||||
id: provisionalNumericId.get(node.id) as number,
|
||||
parentId: (() => {
|
||||
const parent = semanticParentId(graph, node);
|
||||
return parent ? (provisionalNumericId.get(parent) ?? null) : null;
|
||||
})(),
|
||||
id: numericId.get(node.id) as number,
|
||||
parentId: node.semanticParentId ? (numericId.get(node.semanticParentId) ?? null) : null,
|
||||
...(node.backendNodeId !== undefined ? { backendNodeId: node.backendNodeId } : {}),
|
||||
frameId: node.frameId,
|
||||
contextScopeId: graph.frames.get(node.frameId)?.contextScopeId ?? node.frameId,
|
||||
referenceable: node.referenceable,
|
||||
...(node.dom
|
||||
? {
|
||||
domParentId:
|
||||
node.domParentId === undefined ? null : (numericId.get(node.domParentId) ?? null),
|
||||
domAncestorIds: domAncestors(graph, node, numericId),
|
||||
}
|
||||
: {}),
|
||||
}));
|
||||
const recoveredById = new Map(
|
||||
applyVomInteractionRecovery(provisionalNodes).map((node) => [node.id, node]),
|
||||
);
|
||||
const duplicates = redundantSourceNodes(graph, orderedNodes);
|
||||
const kept = new Set<SemanticNodeId>();
|
||||
for (const node of orderedNodes) {
|
||||
if (duplicates.has(node.id)) continue;
|
||||
const id = provisionalNumericId.get(node.id) as number;
|
||||
const recovered = recoveredById.get(id) as VomNode;
|
||||
if ((!node.excluded && frameOwnerIds.has(node.id)) || shouldKeepNode(graph, node, recovered)) {
|
||||
kept.add(node.id);
|
||||
}
|
||||
}
|
||||
|
||||
const numericId = new Map<SemanticNodeId, number>();
|
||||
let nextId = 1;
|
||||
for (const node of orderedNodes) {
|
||||
if (!kept.has(node.id)) continue;
|
||||
numericId.set(node.id, nextId);
|
||||
nextId += 1;
|
||||
}
|
||||
|
||||
const nodes: VomNode[] = [];
|
||||
const keptParentMemo = new Map<SemanticNodeId, SemanticNodeId | undefined>();
|
||||
for (const node of orderedNodes) {
|
||||
const id = numericId.get(node.id);
|
||||
if (id === undefined) continue;
|
||||
const provisional = recoveredById.get(provisionalNumericId.get(node.id) as number) as VomNode;
|
||||
const parent = nearestKeptParent(graph, node.id, kept, keptParentMemo);
|
||||
const domParent = node.domParentId ? numericId.get(node.domParentId) : undefined;
|
||||
nodes.push({
|
||||
...provisional,
|
||||
id,
|
||||
parentId: parent ? (numericId.get(parent) ?? null) : null,
|
||||
...(node.backendNodeId !== undefined ? { backendNodeId: node.backendNodeId } : {}),
|
||||
...(node.dom
|
||||
? {
|
||||
domParentId: node.domParentId === undefined ? null : (domParent ?? null),
|
||||
domAncestorIds: domAncestors(graph, node, numericId),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
viewport: graph.viewport,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { VomNode } from "@browser-skill/vom";
|
||||
import { stableIdentifierName } from "./name-evidence";
|
||||
import type {
|
||||
ResolvedSemanticGraph,
|
||||
ResolvedSemanticNode,
|
||||
@@ -7,6 +8,7 @@ import type {
|
||||
SemanticGraphNode,
|
||||
SemanticNodeId,
|
||||
} from "./types";
|
||||
import { frameBackendKey } from "./types";
|
||||
|
||||
const FORM_TAGS = new Set(["input", "textarea", "select"]);
|
||||
const NATIVE_CONTROL_TAGS = new Set(["button", "input", "select", "textarea"]);
|
||||
@@ -57,8 +59,6 @@ function normalizeRole(value: string | undefined): string | undefined {
|
||||
}
|
||||
|
||||
function nativeRole(tag: string, attrs: Record<string, string>): string | undefined {
|
||||
const explicit = normalizeRole(attrs.role);
|
||||
if (explicit) return explicit;
|
||||
if (tag === "button") return "button";
|
||||
if (tag === "textarea") return "textbox";
|
||||
if (tag === "select") return attrs.multiple === undefined ? "combobox" : "listbox";
|
||||
@@ -90,14 +90,29 @@ function nativeRole(tag: string, attrs: Record<string, string>): string | undefi
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function resolvedRole(node: SemanticGraphNode): string | undefined {
|
||||
function domRole(tag: string, attrs: Record<string, string>): string | undefined {
|
||||
return normalizeRole(attrs.role) ?? nativeRole(tag, attrs);
|
||||
}
|
||||
|
||||
function resolvedRole(node: SemanticGraphNode): {
|
||||
value?: string;
|
||||
source: ResolvedSemanticNode["roleSource"];
|
||||
} {
|
||||
const rawAxRole = clean(axValue(node.ax?.role));
|
||||
const axRole = normalizeRole(rawAxRole);
|
||||
const domRole = nativeRole(node.dom?.tag.toLowerCase() ?? "", node.dom?.attrs ?? {});
|
||||
if (!axRole || ["generic", "none", "presentation"].includes(axRole)) {
|
||||
return domRole ?? rawAxRole;
|
||||
const attrs = node.dom?.attrs ?? {};
|
||||
const explicitDomRole = normalizeRole(attrs.role);
|
||||
const nativeDomRole = nativeRole(node.dom?.tag.toLowerCase() ?? "", attrs);
|
||||
const usableAxRole = node.ax?.ignored !== true;
|
||||
if (usableAxRole && axRole && !["generic", "none", "presentation"].includes(axRole)) {
|
||||
return { value: rawAxRole, source: "ax" };
|
||||
}
|
||||
return rawAxRole;
|
||||
if (explicitDomRole) return { value: explicitDomRole, source: "dom-explicit" };
|
||||
if (nativeDomRole) return { value: nativeDomRole, source: "dom-native" };
|
||||
if (rawAxRole) {
|
||||
return { value: rawAxRole, source: usableAxRole ? "ax" : "ax-ignored" };
|
||||
}
|
||||
return { source: "none" };
|
||||
}
|
||||
|
||||
function sensitive(node: SemanticGraphNode): boolean {
|
||||
@@ -143,14 +158,6 @@ function externalHrefHost(
|
||||
}
|
||||
}
|
||||
|
||||
function iconKeyword(value: string | undefined): string | undefined {
|
||||
const token = clean(value)
|
||||
?.split(/[#/.\s:_-]+/)
|
||||
.filter(Boolean)
|
||||
.pop();
|
||||
return token && /^[a-z][a-z0-9_-]{1,32}$/i.test(token) ? token.toLowerCase() : undefined;
|
||||
}
|
||||
|
||||
interface ResolveIndexes {
|
||||
domChildren: Map<SemanticNodeId, SemanticNodeId[]>;
|
||||
axChildren: Map<SemanticNodeId, SemanticNodeId[]>;
|
||||
@@ -214,7 +221,7 @@ function buildIndexes(graph: SemanticGraph): ResolveIndexes {
|
||||
for (const childId of domChildren.get(nodeId) ?? []) {
|
||||
const child = graph.nodes.get(childId);
|
||||
const childRole = child
|
||||
? nativeRole(child.dom?.tag.toLowerCase() ?? "", child.dom?.attrs ?? {})
|
||||
? domRole(child.dom?.tag.toLowerCase() ?? "", child.dom?.attrs ?? {})
|
||||
: undefined;
|
||||
if (childRole && INTERACTIVE_ROLES.has(childRole.toLowerCase())) continue;
|
||||
const childText = domText(childId);
|
||||
@@ -262,10 +269,8 @@ function buildIndexes(graph: SemanticGraph): ResolveIndexes {
|
||||
for (const childId of domChildren.get(nodeId) ?? []) {
|
||||
const child = graph.nodes.get(childId);
|
||||
const childRole =
|
||||
normalizeRole(axValue(child?.ax?.role)) ??
|
||||
(child
|
||||
? nativeRole(child.dom?.tag.toLowerCase() ?? "", child.dom?.attrs ?? {})
|
||||
: undefined);
|
||||
(child?.ax?.ignored === true ? undefined : normalizeRole(axValue(child?.ax?.role))) ??
|
||||
(child ? domRole(child.dom?.tag.toLowerCase() ?? "", child.dom?.attrs ?? {}) : undefined);
|
||||
if (
|
||||
(child?.dom && NATIVE_CONTROL_TAGS.has(child.dom.tag.toLowerCase())) ||
|
||||
INTERACTIVE_ROLES.has(childRole ?? "") ||
|
||||
@@ -359,7 +364,7 @@ function precedingDomText(
|
||||
for (let offset = 1; offset <= 4 && index - offset >= 0; offset += 1) {
|
||||
const sibling = graph.nodes.get(siblings[index - offset]);
|
||||
const siblingRole = sibling
|
||||
? nativeRole(sibling.dom?.tag.toLowerCase() ?? "", sibling.dom?.attrs ?? {})
|
||||
? domRole(sibling.dom?.tag.toLowerCase() ?? "", sibling.dom?.attrs ?? {})
|
||||
: undefined;
|
||||
if (siblingRole && INTERACTIVE_ROLES.has(siblingRole.toLowerCase())) break;
|
||||
const text = indexes.domText(siblings[index - offset])?.replace(/[::]\s*$/, "");
|
||||
@@ -368,49 +373,23 @@ function precedingDomText(
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function iconHint(
|
||||
nodeId: SemanticNodeId,
|
||||
graph: SemanticGraph,
|
||||
indexes: ResolveIndexes,
|
||||
depth = 0,
|
||||
): string | undefined {
|
||||
if (depth > 3) return undefined;
|
||||
for (const childId of indexes.domChildren.get(nodeId) ?? []) {
|
||||
const child = graph.nodes.get(childId);
|
||||
const attrs = child?.dom?.attrs ?? {};
|
||||
const hint =
|
||||
iconKeyword(attrs["aria-label"]) ??
|
||||
iconKeyword(attrs.title) ??
|
||||
iconKeyword(attrs.href) ??
|
||||
iconKeyword(attrs["xlink:href"]) ??
|
||||
iconKeyword(axValue(child?.ax?.name)) ??
|
||||
iconKeyword(attrs.class) ??
|
||||
iconKeyword(child?.dom?.textContent);
|
||||
if (hint) return hint;
|
||||
const nested = iconHint(childId, graph, indexes, depth + 1);
|
||||
if (nested) return nested;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function resolvedName(
|
||||
node: SemanticGraphNode,
|
||||
role: string | undefined,
|
||||
graph: SemanticGraph,
|
||||
indexes: ResolveIndexes,
|
||||
supplementalNames: ReadonlyMap<string, string>,
|
||||
identifierFallback: boolean,
|
||||
): string | undefined {
|
||||
const attrs = node.dom?.attrs ?? {};
|
||||
const formControl = FORM_TAGS.has(node.dom?.tag.toLowerCase() ?? "");
|
||||
const descendantText = indexes.domText(node.id);
|
||||
const canUseIconHint =
|
||||
!indexes.hasNativeDescendant(node.id) &&
|
||||
(INTERACTIVE_ROLES.has(role?.toLowerCase() ?? "") ||
|
||||
node.dom?.cursor === "pointer" ||
|
||||
attrs.onclick !== undefined ||
|
||||
attrs.tabindex !== undefined);
|
||||
const interactive =
|
||||
INTERACTIVE_ROLES.has(role?.toLowerCase() ?? "") ||
|
||||
NATIVE_CONTROL_TAGS.has(node.dom?.tag.toLowerCase() ?? "");
|
||||
const candidates = [
|
||||
axValue(node.ax?.name),
|
||||
indexes.axText(node.id),
|
||||
node.ax?.ignored === true ? undefined : axValue(node.ax?.name),
|
||||
node.ax?.ignored === true ? undefined : indexes.axText(node.id),
|
||||
ariaLabelledText(node, indexes),
|
||||
clean(attrs["aria-label"]),
|
||||
nativeLabelText(node, graph, indexes),
|
||||
@@ -420,7 +399,10 @@ function resolvedName(
|
||||
formControl ? precedingAxText(node, graph, indexes) : undefined,
|
||||
formControl ? precedingDomText(node, graph, indexes) : undefined,
|
||||
formControl ? clean(node.dom?.formPlaceholder ?? attrs.placeholder) : undefined,
|
||||
canUseIconHint ? iconHint(node.id, graph, indexes) : undefined,
|
||||
!interactive || node.backendNodeId === undefined
|
||||
? undefined
|
||||
: clean(supplementalNames.get(frameBackendKey(node.frameId, node.backendNodeId))),
|
||||
identifierFallback && interactive ? stableIdentifierName(attrs) : undefined,
|
||||
];
|
||||
let name = candidates.find((candidate) => candidate !== undefined);
|
||||
const hasPopup = axProperty(node.ax, "hasPopup");
|
||||
@@ -462,12 +444,21 @@ function nearbyText(
|
||||
return clean(texts.join(" "));
|
||||
}
|
||||
|
||||
export function resolveSemanticGraph(graph: SemanticGraph): ResolvedSemanticGraph {
|
||||
export function resolveSemanticGraph(
|
||||
graph: SemanticGraph,
|
||||
options: {
|
||||
supplementalNames?: ReadonlyMap<string, string>;
|
||||
identifierFallback?: boolean;
|
||||
} = {},
|
||||
): ResolvedSemanticGraph {
|
||||
const indexes = buildIndexes(graph);
|
||||
const supplementalNames = options.supplementalNames ?? new Map<string, string>();
|
||||
const identifierFallback = options.identifierFallback ?? true;
|
||||
const nodes = new Map<SemanticNodeId, ResolvedSemanticNode>();
|
||||
for (const node of graph.nodes.values()) {
|
||||
const role = resolvedRole(node);
|
||||
const name = resolvedName(node, role, graph, indexes);
|
||||
const roleResolution = resolvedRole(node);
|
||||
const role = roleResolution.value;
|
||||
const name = resolvedName(node, role, graph, indexes, supplementalNames, identifierFallback);
|
||||
const isSensitive = sensitive(node);
|
||||
const sourceValue = node.dom?.formValue ?? axValue(node.ax?.value);
|
||||
const value = isSensitive ? undefined : sourceValue;
|
||||
@@ -505,9 +496,9 @@ export function resolveSemanticGraph(graph: SemanticGraph): ResolvedSemanticGrap
|
||||
insideNative: insideNative(node, graph),
|
||||
...(role?.toLowerCase() === "link" ? { href: externalHrefHost(attrs.href, frame?.url) } : {}),
|
||||
};
|
||||
const selected = axProperty(node.ax, "selected");
|
||||
const expanded = axProperty(node.ax, "expanded");
|
||||
const controls = axProperty(node.ax, "controls");
|
||||
const selected = node.ax?.ignored === true ? undefined : axProperty(node.ax, "selected");
|
||||
const expanded = node.ax?.ignored === true ? undefined : axProperty(node.ax, "expanded");
|
||||
const controls = node.ax?.ignored === true ? undefined : axProperty(node.ax, "controls");
|
||||
if (selected === "true" && !vom.attrs?.["aria-selected"]) {
|
||||
vom.attrs = { ...(vom.attrs ?? {}), "aria-selected": "true" };
|
||||
}
|
||||
@@ -521,6 +512,7 @@ export function resolveSemanticGraph(graph: SemanticGraph): ResolvedSemanticGrap
|
||||
...node,
|
||||
vom,
|
||||
referenceable: node.backendNodeId !== undefined,
|
||||
roleSource: roleResolution.source,
|
||||
});
|
||||
}
|
||||
return { ...graph, nodes };
|
||||
|
||||
@@ -0,0 +1,436 @@
|
||||
import {
|
||||
applyVomInteractionRecovery,
|
||||
isVomReferenceNode,
|
||||
isVomStructuralRole,
|
||||
type VomNode,
|
||||
} from "@browser-skill/vom";
|
||||
import type {
|
||||
ResolvedSemanticGraph,
|
||||
ResolvedSemanticNode,
|
||||
SemanticNodeId,
|
||||
StructureDecision,
|
||||
StructuredSemanticGraph,
|
||||
StructuredSemanticNode,
|
||||
} from "./types";
|
||||
|
||||
const TRANSPARENT_ROLES = new Set(["", "generic", "none", "presentation", "inlinetextbox"]);
|
||||
const ROOT_ROLES = new Set(["rootwebarea", "webarea"]);
|
||||
const CONTENT_BOUNDARY_ROLES = new Set([
|
||||
"article",
|
||||
"dialog",
|
||||
"document",
|
||||
"feed",
|
||||
"form",
|
||||
"grid",
|
||||
"heading",
|
||||
"iframe",
|
||||
"list",
|
||||
"main",
|
||||
"navigation",
|
||||
"paragraph",
|
||||
"region",
|
||||
"section",
|
||||
"table",
|
||||
"tree",
|
||||
]);
|
||||
|
||||
function hasMeaningfulRelations(node: ResolvedSemanticNode): boolean {
|
||||
const attrs = node.vom.attrs ?? {};
|
||||
return ["aria-controls", "aria-labelledby", "aria-owns", "aria-describedby"].some((name) =>
|
||||
attrs[name]?.trim(),
|
||||
);
|
||||
}
|
||||
|
||||
function explicitStructural(node: ResolvedSemanticNode): boolean {
|
||||
return (
|
||||
isVomStructuralRole(node.vom.role) &&
|
||||
node.roleSource !== "none" &&
|
||||
node.roleSource !== "ax-ignored"
|
||||
);
|
||||
}
|
||||
|
||||
function initialParentId(
|
||||
graph: ResolvedSemanticGraph,
|
||||
node: ResolvedSemanticNode,
|
||||
): SemanticNodeId | undefined {
|
||||
let parentId = node.axParentId ?? node.domParentId;
|
||||
if (node.axParentId && node.domParentId && node.axParentId !== node.domParentId) {
|
||||
const axParent = graph.nodes.get(node.axParentId);
|
||||
const axRole = axParent?.vom.role?.toLowerCase() ?? "";
|
||||
const meaningfulAxParent =
|
||||
axParent !== undefined &&
|
||||
!ROOT_ROLES.has(axRole) &&
|
||||
(explicitStructural(axParent) ||
|
||||
!TRANSPARENT_ROLES.has(axRole) ||
|
||||
Boolean(axParent.vom.name || axParent.vom.text || hasMeaningfulRelations(axParent)));
|
||||
parentId = meaningfulAxParent ? node.axParentId : node.domParentId;
|
||||
}
|
||||
if (parentId) return parentId;
|
||||
if (node.frameId === graph.rootFrameId) return undefined;
|
||||
return graph.frames.get(node.frameId)?.ownerNodeId;
|
||||
}
|
||||
|
||||
function orderedNodes(graph: ResolvedSemanticGraph): ResolvedSemanticNode[] {
|
||||
return [...graph.frames.values()]
|
||||
.sort((a, b) => a.order - b.order)
|
||||
.flatMap((frame) =>
|
||||
(graph.nodeIdsByFrame.get(frame.frameId) ?? [])
|
||||
.map((id) => graph.nodes.get(id))
|
||||
.filter((node): node is ResolvedSemanticNode => node !== undefined)
|
||||
.sort((a, b) => a.sourceOrder - b.sourceOrder),
|
||||
);
|
||||
}
|
||||
|
||||
function recoveredNodes(
|
||||
graph: ResolvedSemanticGraph,
|
||||
nodes: ResolvedSemanticNode[],
|
||||
): Map<SemanticNodeId, VomNode> {
|
||||
const numericId = new Map(nodes.map((node, index) => [node.id, index + 1]));
|
||||
const semanticId = new Map([...numericId].map(([id, numeric]) => [numeric, id]));
|
||||
const provisional = nodes.map((node) => ({
|
||||
...node.vom,
|
||||
id: numericId.get(node.id) as number,
|
||||
parentId: (() => {
|
||||
const parent = initialParentId(graph, node);
|
||||
return parent ? (numericId.get(parent) ?? null) : null;
|
||||
})(),
|
||||
...(node.backendNodeId !== undefined ? { backendNodeId: node.backendNodeId } : {}),
|
||||
frameId: node.frameId,
|
||||
contextScopeId: graph.frames.get(node.frameId)?.contextScopeId ?? node.frameId,
|
||||
referenceable: node.referenceable,
|
||||
}));
|
||||
return new Map(
|
||||
applyVomInteractionRecovery(provisional).flatMap((node) => {
|
||||
const id = semanticId.get(node.id);
|
||||
return id ? [[id, node] as const] : [];
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function redundantSourceNodes(
|
||||
graph: ResolvedSemanticGraph,
|
||||
nodes: ResolvedSemanticNode[],
|
||||
recovered: ReadonlyMap<SemanticNodeId, VomNode>,
|
||||
): Set<SemanticNodeId> {
|
||||
const byKey = new Map<string, ResolvedSemanticNode[]>();
|
||||
for (const node of nodes) {
|
||||
const output = recovered.get(node.id) as VomNode;
|
||||
const name = output.name?.trim().toLowerCase() ?? "";
|
||||
if (!name || !isVomReferenceNode({ ...output, referenceable: true })) continue;
|
||||
const key = `${node.frameId}\u0000${initialParentId(graph, node) ?? ""}\u0000${output.role?.toLowerCase() ?? ""}\u0000${name}\u0000${output.sensitive === true}`;
|
||||
const matches = byKey.get(key) ?? [];
|
||||
matches.push(node);
|
||||
byKey.set(key, matches);
|
||||
}
|
||||
const redundant = new Set<SemanticNodeId>();
|
||||
for (const matches of byKey.values()) {
|
||||
const axOnly = matches.filter((node) => node.ax && !node.dom);
|
||||
const domOnly = matches.filter((node) => node.dom && !node.ax);
|
||||
if (matches.length === 2 && axOnly.length === 1 && domOnly.length === 1) {
|
||||
redundant.add(domOnly[0].id);
|
||||
}
|
||||
}
|
||||
return redundant;
|
||||
}
|
||||
|
||||
function baseDecision(
|
||||
graph: ResolvedSemanticGraph,
|
||||
node: ResolvedSemanticNode,
|
||||
output: VomNode,
|
||||
frameOwnerIds: ReadonlySet<SemanticNodeId>,
|
||||
redundant: ReadonlySet<SemanticNodeId>,
|
||||
): StructureDecision {
|
||||
if (node.excluded) return { kind: "excluded", reason: "excluded" };
|
||||
if (redundant.has(node.id)) return { kind: "excluded", reason: "redundant-source" };
|
||||
if (frameOwnerIds.has(node.id)) return { kind: "keep", reason: "frame-owner" };
|
||||
if (node.frameId !== graph.rootFrameId && ROOT_ROLES.has(output.role?.toLowerCase() ?? "")) {
|
||||
return { kind: "transparent", reason: "child-document-root" };
|
||||
}
|
||||
if (node.ax?.ignored === true && !node.dom) {
|
||||
return { kind: "transparent", reason: "ax-ignored" };
|
||||
}
|
||||
const role = output.role?.toLowerCase() ?? "";
|
||||
if (!TRANSPARENT_ROLES.has(role)) return { kind: "keep", reason: "semantic" };
|
||||
if (
|
||||
output.name ||
|
||||
output.text ||
|
||||
output.value ||
|
||||
output.placeholder ||
|
||||
hasMeaningfulRelations(node) ||
|
||||
(output.rect && ["fixed", "sticky"].includes(output.position))
|
||||
) {
|
||||
return { kind: "keep", reason: "semantic" };
|
||||
}
|
||||
return { kind: "transparent", reason: "wrapper" };
|
||||
}
|
||||
|
||||
function buildChildren(
|
||||
nodes: ResolvedSemanticNode[],
|
||||
relation: "axParentId" | "domParentId",
|
||||
): Map<SemanticNodeId, SemanticNodeId[]> {
|
||||
const children = new Map<SemanticNodeId, SemanticNodeId[]>();
|
||||
for (const node of nodes) {
|
||||
const parentId = node[relation];
|
||||
if (!parentId) continue;
|
||||
const siblings = children.get(parentId) ?? [];
|
||||
siblings.push(node.id);
|
||||
children.set(parentId, siblings);
|
||||
}
|
||||
return children;
|
||||
}
|
||||
|
||||
function explicitDomAncestors(
|
||||
graph: ResolvedSemanticGraph,
|
||||
nodes: ResolvedSemanticNode[],
|
||||
domChildren: ReadonlyMap<SemanticNodeId, SemanticNodeId[]>,
|
||||
): Set<SemanticNodeId> {
|
||||
const inside = new Set<SemanticNodeId>();
|
||||
const roots = nodes.filter((node) => !node.domParentId);
|
||||
const queue = roots.map((node) => node.id);
|
||||
for (let index = 0; index < queue.length; index += 1) {
|
||||
const parentId = queue[index];
|
||||
const parent = graph.nodes.get(parentId);
|
||||
const parentIsBoundary =
|
||||
parent !== undefined &&
|
||||
explicitStructural(parent) &&
|
||||
!ROOT_ROLES.has(parent.vom.role?.toLowerCase() ?? "");
|
||||
for (const childId of domChildren.get(parentId) ?? []) {
|
||||
if (inside.has(parentId) || parentIsBoundary) inside.add(childId);
|
||||
queue.push(childId);
|
||||
}
|
||||
}
|
||||
return inside;
|
||||
}
|
||||
|
||||
function reliableGroupCandidates(
|
||||
graph: ResolvedSemanticGraph,
|
||||
nodes: ResolvedSemanticNode[],
|
||||
recovered: ReadonlyMap<SemanticNodeId, VomNode>,
|
||||
decisions: ReadonlyMap<SemanticNodeId, StructureDecision>,
|
||||
domChildren: ReadonlyMap<SemanticNodeId, SemanticNodeId[]>,
|
||||
): Set<SemanticNodeId> {
|
||||
const insideExplicitStructure = explicitDomAncestors(graph, nodes, domChildren);
|
||||
const remainingChildren = new Map(
|
||||
nodes.map((node) => [node.id, domChildren.get(node.id)?.length ?? 0]),
|
||||
);
|
||||
const summaries = new Map<SemanticNodeId, { blocked: boolean; interactions: number }>();
|
||||
const queue = nodes
|
||||
.filter((node) => (remainingChildren.get(node.id) ?? 0) === 0)
|
||||
.map((node) => node.id);
|
||||
for (let index = 0; index < queue.length; index += 1) {
|
||||
const nodeId = queue[index];
|
||||
const node = graph.nodes.get(nodeId);
|
||||
const output = recovered.get(nodeId);
|
||||
if (!node || !output) continue;
|
||||
const role = output.role?.toLowerCase() ?? "";
|
||||
const excluded = decisions.get(nodeId)?.kind === "excluded";
|
||||
const blocked = excluded || explicitStructural(node) || CONTENT_BOUNDARY_ROLES.has(role);
|
||||
const summary = excluded
|
||||
? { blocked: true, interactions: 0 }
|
||||
: isVomReferenceNode(output)
|
||||
? { blocked: false, interactions: 1 }
|
||||
: blocked
|
||||
? { blocked: true, interactions: 0 }
|
||||
: (domChildren.get(nodeId) ?? []).reduce(
|
||||
(combined, childId) => {
|
||||
const child = summaries.get(childId);
|
||||
return {
|
||||
blocked: combined.blocked || !child || child.blocked,
|
||||
interactions: Math.min(2, combined.interactions + (child?.interactions ?? 0)),
|
||||
};
|
||||
},
|
||||
{ blocked: false, interactions: 0 },
|
||||
);
|
||||
summaries.set(nodeId, summary);
|
||||
const parentId = node.domParentId;
|
||||
if (!parentId) continue;
|
||||
const remaining = (remainingChildren.get(parentId) ?? 1) - 1;
|
||||
remainingChildren.set(parentId, remaining);
|
||||
if (remaining === 0) queue.push(parentId);
|
||||
}
|
||||
|
||||
const candidates = new Set<SemanticNodeId>();
|
||||
for (const node of nodes) {
|
||||
const output = recovered.get(node.id) as VomNode;
|
||||
const role = output.role?.toLowerCase() ?? "";
|
||||
if (
|
||||
!node.dom ||
|
||||
node.dom.tag.toLowerCase() !== "div" ||
|
||||
!TRANSPARENT_ROLES.has(role) ||
|
||||
insideExplicitStructure.has(node.id) ||
|
||||
decisions.get(node.id)?.kind === "excluded" ||
|
||||
node.dom.attrs.hidden !== undefined ||
|
||||
(node.dom.attrs["aria-hidden"] ?? "").toLowerCase() === "true" ||
|
||||
node.dom.attrs.inert !== undefined
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const branches = (domChildren.get(node.id) ?? []).map((id) => summaries.get(id));
|
||||
const blocked = branches.some((summary) => !summary || summary.blocked);
|
||||
const contributingBranches = branches.filter(
|
||||
(summary) => (summary?.interactions ?? 0) > 0,
|
||||
).length;
|
||||
const interactions = branches.reduce(
|
||||
(count, summary) => Math.min(2, count + (summary?.interactions ?? 0)),
|
||||
0,
|
||||
);
|
||||
if (!blocked && contributingBranches >= 2 && interactions >= 2) {
|
||||
candidates.add(node.id);
|
||||
}
|
||||
}
|
||||
return candidates;
|
||||
}
|
||||
|
||||
function leafCandidates(
|
||||
nodes: ResolvedSemanticNode[],
|
||||
candidates: ReadonlySet<SemanticNodeId>,
|
||||
domChildren: ReadonlyMap<SemanticNodeId, SemanticNodeId[]>,
|
||||
): Set<SemanticNodeId> {
|
||||
const parentById = new Map(
|
||||
nodes.flatMap((node) => (node.domParentId ? ([[node.id, node.domParentId]] as const) : [])),
|
||||
);
|
||||
const childCount = new Map(nodes.map((node) => [node.id, domChildren.get(node.id)?.length ?? 0]));
|
||||
const containsCandidate = new Set<SemanticNodeId>();
|
||||
const queue = nodes.filter((node) => (childCount.get(node.id) ?? 0) === 0).map((node) => node.id);
|
||||
const selected = new Set<SemanticNodeId>();
|
||||
for (let index = 0; index < queue.length; index += 1) {
|
||||
const nodeId = queue[index];
|
||||
if (candidates.has(nodeId) && !containsCandidate.has(nodeId)) selected.add(nodeId);
|
||||
const parentId = parentById.get(nodeId);
|
||||
if (!parentId) continue;
|
||||
if (candidates.has(nodeId) || containsCandidate.has(nodeId)) containsCandidate.add(parentId);
|
||||
const remaining = (childCount.get(parentId) ?? 1) - 1;
|
||||
childCount.set(parentId, remaining);
|
||||
if (remaining === 0) queue.push(parentId);
|
||||
}
|
||||
return selected;
|
||||
}
|
||||
|
||||
function nearestKept(
|
||||
graph: ResolvedSemanticGraph,
|
||||
startId: SemanticNodeId | undefined,
|
||||
relation: "axParentId" | "domParentId",
|
||||
kept: ReadonlySet<SemanticNodeId>,
|
||||
memo: Map<SemanticNodeId, SemanticNodeId | undefined>,
|
||||
): SemanticNodeId | undefined {
|
||||
let currentId = startId;
|
||||
const path: SemanticNodeId[] = [];
|
||||
const seen = new Set<SemanticNodeId>();
|
||||
let result: SemanticNodeId | undefined;
|
||||
while (currentId && !seen.has(currentId)) {
|
||||
if (kept.has(currentId)) {
|
||||
result = currentId;
|
||||
break;
|
||||
}
|
||||
if (memo.has(currentId)) {
|
||||
result = memo.get(currentId);
|
||||
break;
|
||||
}
|
||||
seen.add(currentId);
|
||||
path.push(currentId);
|
||||
currentId = graph.nodes.get(currentId)?.[relation];
|
||||
}
|
||||
for (const id of path) memo.set(id, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
function finalParentId(
|
||||
graph: ResolvedSemanticGraph,
|
||||
node: ResolvedSemanticNode,
|
||||
kept: ReadonlySet<SemanticNodeId>,
|
||||
decisions: ReadonlyMap<SemanticNodeId, StructureDecision>,
|
||||
axParentMemo: Map<SemanticNodeId, SemanticNodeId | undefined>,
|
||||
domParentMemo: Map<SemanticNodeId, SemanticNodeId | undefined>,
|
||||
): SemanticNodeId | undefined {
|
||||
const axParent = nearestKept(graph, node.axParentId, "axParentId", kept, axParentMemo);
|
||||
const domParent = nearestKept(graph, node.domParentId, "domParentId", kept, domParentMemo);
|
||||
const axNode = axParent ? graph.nodes.get(axParent) : undefined;
|
||||
const domNode = domParent ? graph.nodes.get(domParent) : undefined;
|
||||
const axIsRoot = ROOT_ROLES.has(axNode?.vom.role?.toLowerCase() ?? "");
|
||||
const axIsExplicit = axNode ? explicitStructural(axNode) && !axIsRoot : false;
|
||||
const domIsStructural =
|
||||
domNode !== undefined &&
|
||||
(explicitStructural(domNode) || decisions.get(domNode.id)?.kind === "inferred-group");
|
||||
if (axIsExplicit) return axParent;
|
||||
if (domIsStructural) return domParent;
|
||||
if (axParent && !axIsRoot) return axParent;
|
||||
if (domParent) return domParent;
|
||||
if (axParent) return axParent;
|
||||
if (node.frameId === graph.rootFrameId) return undefined;
|
||||
return graph.frames.get(node.frameId)?.ownerNodeId;
|
||||
}
|
||||
|
||||
function recoveredVom(node: VomNode, inferredGroup: boolean): ResolvedSemanticNode["vom"] {
|
||||
const {
|
||||
id: _id,
|
||||
parentId: _parentId,
|
||||
backendNodeId: _backendNodeId,
|
||||
frameId: _frameId,
|
||||
contextScopeId: _contextScopeId,
|
||||
referenceable: _referenceable,
|
||||
...vom
|
||||
} = node;
|
||||
if (!inferredGroup) return vom;
|
||||
const { name: _name, text: _text, nearbyText: _nearbyText, ...groupVom } = vom;
|
||||
return { ...groupVom, role: "group" };
|
||||
}
|
||||
|
||||
export function normalizeSemanticStructure(graph: ResolvedSemanticGraph): StructuredSemanticGraph {
|
||||
const nodes = orderedNodes(graph);
|
||||
const recovered = recoveredNodes(graph, nodes);
|
||||
const redundant = redundantSourceNodes(graph, nodes, recovered);
|
||||
const frameOwnerIds = new Set(
|
||||
[...graph.frames.values()].flatMap((frame) => frame.ownerNodeId ?? []),
|
||||
);
|
||||
const decisions = new Map<SemanticNodeId, StructureDecision>();
|
||||
for (const node of nodes) {
|
||||
decisions.set(
|
||||
node.id,
|
||||
baseDecision(graph, node, recovered.get(node.id) as VomNode, frameOwnerIds, redundant),
|
||||
);
|
||||
}
|
||||
|
||||
const domChildren = buildChildren(nodes, "domParentId");
|
||||
const candidates = reliableGroupCandidates(graph, nodes, recovered, decisions, domChildren);
|
||||
for (const nodeId of leafCandidates(nodes, candidates, domChildren)) {
|
||||
decisions.set(nodeId, {
|
||||
kind: "inferred-group",
|
||||
reason: "independent-interaction-branches",
|
||||
});
|
||||
}
|
||||
|
||||
const kept = new Set(
|
||||
nodes
|
||||
.filter((node) => {
|
||||
const kind = decisions.get(node.id)?.kind;
|
||||
return kind === "keep" || kind === "inferred-group";
|
||||
})
|
||||
.map((node) => node.id),
|
||||
);
|
||||
const axParentMemo = new Map<SemanticNodeId, SemanticNodeId | undefined>();
|
||||
const domParentMemo = new Map<SemanticNodeId, SemanticNodeId | undefined>();
|
||||
const structuredNodes = new Map<SemanticNodeId, StructuredSemanticNode>();
|
||||
for (const node of nodes) {
|
||||
const structure = decisions.get(node.id) as StructureDecision;
|
||||
const inferredGroup = structure.kind === "inferred-group";
|
||||
structuredNodes.set(node.id, {
|
||||
...node,
|
||||
vom: recoveredVom(recovered.get(node.id) as VomNode, inferredGroup),
|
||||
structure,
|
||||
...(kept.has(node.id)
|
||||
? {
|
||||
semanticParentId: finalParentId(
|
||||
graph,
|
||||
node,
|
||||
kept,
|
||||
decisions,
|
||||
axParentMemo,
|
||||
domParentMemo,
|
||||
),
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
return { ...graph, nodes: structuredNodes };
|
||||
}
|
||||
@@ -54,12 +54,28 @@ export interface SemanticGraph {
|
||||
export interface ResolvedSemanticNode extends SemanticGraphNode {
|
||||
vom: Omit<VomNode, "id" | "parentId" | "backendNodeId" | "frameId" | "contextScopeId">;
|
||||
referenceable: boolean;
|
||||
roleSource: "ax" | "ax-ignored" | "dom-explicit" | "dom-native" | "none";
|
||||
}
|
||||
|
||||
export interface ResolvedSemanticGraph extends Omit<SemanticGraph, "nodes"> {
|
||||
nodes: Map<SemanticNodeId, ResolvedSemanticNode>;
|
||||
}
|
||||
|
||||
export type StructureDecision =
|
||||
| { kind: "keep"; reason: "semantic" | "frame-owner" }
|
||||
| { kind: "inferred-group"; reason: "independent-interaction-branches" }
|
||||
| { kind: "transparent"; reason: "wrapper" | "child-document-root" | "ax-ignored" }
|
||||
| { kind: "excluded"; reason: "excluded" | "redundant-source" };
|
||||
|
||||
export interface StructuredSemanticNode extends ResolvedSemanticNode {
|
||||
structure: StructureDecision;
|
||||
semanticParentId?: SemanticNodeId;
|
||||
}
|
||||
|
||||
export interface StructuredSemanticGraph extends Omit<ResolvedSemanticGraph, "nodes"> {
|
||||
nodes: Map<SemanticNodeId, StructuredSemanticNode>;
|
||||
}
|
||||
|
||||
export function frameBackendKey(frameId: string, backendNodeId: number): string {
|
||||
return `${frameId}\u0000${backendNodeId}`;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user