diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index ed290d9..aa1a6a5 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -59,7 +59,15 @@ import { projectRecordSafeObservation, } from "./vom/record-safe-observation"; import type { FrameDocument } from "./vom/frame-document"; -import { buildSemanticVomScene, type SemanticAxNode } from "./vom/semantic-graph"; +import { probeTooltipNames } from "./vom/name-enrichment"; +import { + buildSemanticGraph, + buildSemanticVomScene, + normalizeSemanticStructure, + projectSemanticGraph, + resolveSemanticGraph, + type SemanticAxNode, +} from "./vom/semantic-graph"; // --------------------------------------------------------------------------- // Shared helpers (legacy aliases — observation.ts kept exporting these @@ -641,6 +649,7 @@ function findSurfaceNodeByPoint( export interface BuildVomSceneOptions { pageUrl?: string; + supplementalNames?: ReadonlyMap; } export type VomFrameDocument = CapturedFrameDocument; @@ -800,13 +809,14 @@ export function buildVomScene( export function buildFrameVomScene( documents: VomFrameDocument[], captured: CapturedViewModel, - _options: BuildVomSceneOptions = {}, + options: BuildVomSceneOptions = {}, ): VomScene { const scene = buildSemanticVomScene({ documents: documents as FrameDocument[], viewport: captured.viewport, rootFrameId: captured.rootFrameId, excludedBackendNodeIds: captured.excludedBackendNodeIds, + supplementalNames: options.supplementalNames, }); const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId); const signals = capturedOnlySignals(rootDocument?.domNodes ?? captured.nodes); @@ -1027,7 +1037,25 @@ export async function captureVomObservation( documents.length === 1 && captured.iframeNodes.size > 0 ? legacyFrameDocuments(documents[0].axNodes, captured, url, documents[0].target) : documents; - const scene = buildFrameVomScene(normalizedDocuments, captured, { pageUrl: url }); + const semanticGraph = buildSemanticGraph({ + documents: normalizedDocuments as FrameDocument[], + viewport: captured.viewport, + rootFrameId: captured.rootFrameId, + excludedBackendNodeIds: captured.excludedBackendNodeIds, + }); + const staticSemantics = resolveSemanticGraph(semanticGraph, { identifierFallback: false }); + const tooltipNames = options.conditionalSurfaceProbe + ? await probeTooltipNames(cdp, tabId, normalizedDocuments, staticSemantics, { + signal: options.signal, + hoverProbeBypassOverlay: options.hoverProbeBypassOverlay, + }) + : new Map(); + throwIfAborted(options.signal, "observation"); + const scene = projectSemanticGraph( + normalizeSemanticStructure( + resolveSemanticGraph(semanticGraph, { supplementalNames: tooltipNames }), + ), + ); const rendered = renderVom(scene, { maxDepth: options.maxDepth, maxTokens: options.maxTokens, diff --git a/apps/extension/src/tools/vom/__tests__/name-enrichment.test.ts b/apps/extension/src/tools/vom/__tests__/name-enrichment.test.ts new file mode 100644 index 0000000..5165ff8 --- /dev/null +++ b/apps/extension/src/tools/vom/__tests__/name-enrichment.test.ts @@ -0,0 +1,161 @@ +import { describe, expect, it, vi } from "vitest"; +import type { CdpTarget } from "@/browser-driver/frame-graph"; +import type { CdpRunner } from "../../shared"; +import type { CapturedNode } from "../capture"; +import type { FrameDocument, FrameOwnedAxNode } from "../frame-document"; +import { probeTooltipNames } from "../name-enrichment"; +import { buildSemanticGraph, resolveSemanticGraph } from "../semantic-graph"; +import { stableIdentifierName } from "../semantic-graph/name-evidence"; +import { frameBackendKey } from "../semantic-graph/types"; + +function document( + target: CdpTarget, + backendNodeId: number, + accessibleName?: string, + frameId = "child", + x = 10, +): FrameDocument { + const node: CapturedNode = { + backendNodeId, + parentBackendNodeId: null, + frameId, + tag: "button", + attrs: { + id: "toolInsertRecord", + "data-test-id": "toolInsertRecord", + ...(accessibleName ? { "aria-label": accessibleName } : {}), + }, + rect: { x, y: 20, w: 40, h: 30 }, + localRect: { x: 2, y: 3, w: 40, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + cursor: "pointer", + }; + return { + frameId, + parentFrameId: "main", + contextScopeId: "child", + target, + axNodes: [], + domNodes: [node], + url: "https://example.com/frame", + }; +} + +function semantics(documents: FrameDocument[]) { + return resolveSemanticGraph( + buildSemanticGraph({ + documents, + viewport: { width: 800, height: 600 }, + rootFrameId: documents[0]?.frameId, + }), + { identifierFallback: false }, + ); +} + +describe("tooltip name enrichment", () => { + it("uses the owning frame target and upgrades an identifier name from a tooltip", async () => { + const target = { tabId: 7, sessionId: "oopif-session" }; + let hovered = false; + const cdp: CdpRunner = { + send: vi.fn(async (_tabId, method, params) => { + if (method === "Input.dispatchMouseEvent") { + hovered = (params as { x?: number }).x === 30; + } + return {}; + }) as CdpRunner["send"], + sendToTarget: vi.fn(async (_target, method) => { + if (method === "Page.createIsolatedWorld") return { executionContextId: 11 }; + if (method === "DOM.resolveNode") return { object: { objectId: "button-501" } }; + if (method === "Runtime.callFunctionOn") { + return { result: { value: hovered ? ["插入行"] : [] } }; + } + return {}; + }) as CdpRunner["sendToTarget"], + }; + + const documents = [document(target, 501)]; + const graph = semantics(documents); + const names = await probeTooltipNames(cdp, 7, documents, graph); + + expect(names.get(frameBackendKey("child", 501))).toBe("插入行"); + expect(cdp.sendToTarget).toHaveBeenCalledWith( + target, + "Page.createIsolatedWorld", + expect.objectContaining({ frameId: "child" }), + ); + + const sendsBeforeCache = vi.mocked(cdp.send).mock.calls.length; + const targetSendsBeforeCache = vi.mocked( + cdp.sendToTarget as NonNullable, + ).mock.calls.length; + const cached = await probeTooltipNames(cdp, 7, documents, graph); + expect(cached.get(frameBackendKey("child", 501))).toBe("插入行"); + expect(cdp.send).toHaveBeenCalledTimes(sendsBeforeCache); + expect(cdp.sendToTarget).toHaveBeenCalledTimes(targetSendsBeforeCache); + }); + + it("does not probe controls that already have a higher-confidence name", async () => { + const cdp: CdpRunner = { + send: vi.fn(async () => ({})) as CdpRunner["send"], + sendToTarget: vi.fn(async () => ({ executionContextId: 12 })) as CdpRunner["sendToTarget"], + }; + + const documents = [document({ tabId: 8 }, 601, "Add a new record")]; + const names = await probeTooltipNames(cdp, 8, documents, semantics(documents)); + + expect(names.size).toBe(0); + expect(cdp.send).not.toHaveBeenCalled(); + expect(cdp.sendToTarget).not.toHaveBeenCalled(); + }); + + it("keeps tooltip evidence isolated across sibling frame sessions", async () => { + let hoveredX = -10; + const cdp: CdpRunner = { + send: vi.fn(async (_tabId, method, params) => { + if (method === "Input.dispatchMouseEvent") { + hoveredX = (params as { x: number }).x; + } + return {}; + }) as CdpRunner["send"], + sendToTarget: vi.fn(async (target, method, params) => { + if (method === "Page.createIsolatedWorld") { + return { executionContextId: target.sessionId === "left-session" ? 21 : 22 }; + } + if (method === "DOM.resolveNode") { + return { object: { objectId: target.sessionId === "left-session" ? "left" : "right" } }; + } + if (method === "Runtime.callFunctionOn") { + const objectId = (params as { objectId: string }).objectId; + const value = + objectId === "left" && hoveredX === 30 + ? ["Left action"] + : objectId === "right" && hoveredX === 130 + ? ["Right action"] + : []; + return { result: { value } }; + } + return {}; + }) as CdpRunner["sendToTarget"], + }; + const documents = [ + document({ tabId: 9, sessionId: "left-session" }, 701, undefined, "left", 10), + document({ tabId: 9, sessionId: "right-session" }, 701, undefined, "right", 110), + ]; + + const names = await probeTooltipNames(cdp, 9, documents, semantics(documents)); + + expect(names.get(frameBackendKey("left", 701))).toBe("Left action"); + expect(names.get(frameBackendKey("right", 701))).toBe("Right action"); + }); +}); + +describe("stable identifier evidence", () => { + it("accepts readable action identifiers and rejects framework-generated ids", () => { + expect(stableIdentifierName({ id: "toolInsertRecord" })).toBe("Insert record"); + expect(stableIdentifierName({ id: "reactivateAccount" })).toBe("Reactivate account"); + expect(stableIdentifierName({ id: "headlessui-menu-button-r1" })).toBeUndefined(); + expect(stableIdentifierName({ id: "7f15a3bc91" })).toBeUndefined(); + }); +}); diff --git a/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts b/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts index eb9d656..b5f95c1 100644 --- a/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts +++ b/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts @@ -141,6 +141,64 @@ describe("semantic VOM graph", () => { ]); }); + it("uses stable identifiers only as a final name fallback", () => { + const base = document( + "main", + [ + { + nodeId: "root", + backendDOMNodeId: 1, + role: { type: "role", value: "RootWebArea" }, + childIds: ["fallback", "labelled", "container"], + }, + { + nodeId: "fallback", + parentId: "root", + backendDOMNodeId: 2, + role: { type: "role", value: "button" }, + }, + { + nodeId: "labelled", + parentId: "root", + backendDOMNodeId: 3, + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Localized name" }, + }, + { + nodeId: "container", + parentId: "root", + backendDOMNodeId: 4, + role: { type: "role", value: "generic" }, + }, + ], + [ + dom(1, null, "body"), + dom(2, 1, "button", { "data-test-id": "toolInsertRecord" }), + dom(3, 1, "button", { id: "toolIgnoredFallback" }), + dom(4, 1, "div", { id: "toolbarContainer" }), + ], + ); + const staticScene = buildSemanticVomScene({ + viewport: { width: 800, height: 600 }, + rootFrameId: "main", + documents: [base], + }); + const enrichedScene = buildSemanticVomScene({ + viewport: { width: 800, height: 600 }, + rootFrameId: "main", + documents: [base], + supplementalNames: new Map([["main\u00002", "插入行"]]), + }); + + expect(staticScene.nodes.find((node) => node.backendNodeId === 2)?.name).toBe("Insert record"); + expect(staticScene.nodes.find((node) => node.backendNodeId === 3)?.name).toBe("Localized name"); + expect(staticScene.nodes.find((node) => node.backendNodeId === 4)?.name).toBeUndefined(); + expect(enrichedScene.nodes.find((node) => node.backendNodeId === 2)?.name).toBe("插入行"); + expect(enrichedScene.nodes.find((node) => node.backendNodeId === 3)?.name).toBe( + "Localized name", + ); + }); + it("isolates equal backend node ids in sibling frames", () => { const scene = buildSemanticVomScene({ viewport: { width: 1000, height: 800 }, @@ -212,4 +270,279 @@ describe("semantic VOM graph", () => { ["frame-b", 5], ]); }); + + it("lets explicit DOM structure override ignored AX metadata", () => { + const scene = buildSemanticVomScene({ + viewport: { width: 800, height: 600 }, + rootFrameId: "main", + documents: [ + document( + "main", + [ + { + nodeId: "root", + backendDOMNodeId: 1, + role: { type: "role", value: "RootWebArea" }, + childIds: ["toolbar"], + }, + { + nodeId: "toolbar", + parentId: "root", + backendDOMNodeId: 2, + ignored: true, + role: { type: "role", value: "generic" }, + childIds: ["wrapper"], + }, + { + nodeId: "wrapper", + parentId: "toolbar", + backendDOMNodeId: 5, + role: { type: "role", value: "generic" }, + childIds: ["bold", "italic"], + }, + { + nodeId: "bold", + parentId: "wrapper", + backendDOMNodeId: 3, + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Bold" }, + }, + { + nodeId: "italic", + parentId: "wrapper", + backendDOMNodeId: 4, + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Italic" }, + }, + ], + [ + dom(1, null, "body"), + dom(2, 1, "div", { role: "toolbar" }), + dom(5, 2, "div"), + dom(3, 5, "button"), + dom(4, 5, "button"), + ], + ), + ], + }); + + const toolbar = scene.nodes.find((node) => node.backendNodeId === 2); + const buttons = scene.nodes.filter((node) => [3, 4].includes(node.backendNodeId ?? -1)); + expect(toolbar).toEqual(expect.objectContaining({ role: "toolbar" })); + expect(toolbar && isVomReferenceNode(toolbar)).toBe(false); + expect(scene.nodes.some((node) => node.role === "group")).toBe(false); + expect(buttons.map((node) => node.parentId)).toEqual([toolbar?.id, toolbar?.id]); + }); + + it("infers an unnamed group only from independent interaction branches", () => { + const scene = buildSemanticVomScene({ + viewport: { width: 800, height: 600 }, + rootFrameId: "main", + documents: [ + document( + "main", + [ + { + nodeId: "root", + backendDOMNodeId: 1, + role: { type: "role", value: "RootWebArea" }, + childIds: ["wrapper"], + }, + { + nodeId: "wrapper", + parentId: "root", + backendDOMNodeId: 2, + role: { type: "role", value: "generic" }, + childIds: ["first", "second"], + }, + { + nodeId: "first", + parentId: "wrapper", + backendDOMNodeId: 3, + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "First" }, + }, + { + nodeId: "second", + parentId: "wrapper", + backendDOMNodeId: 4, + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Second" }, + }, + ], + [dom(1, null, "body"), dom(2, 1, "div"), dom(3, 2, "button"), dom(4, 2, "button")], + ), + ], + }); + + const group = scene.nodes.find((node) => node.backendNodeId === 2); + const buttons = scene.nodes.filter((node) => [3, 4].includes(node.backendNodeId ?? -1)); + expect(group).toEqual(expect.objectContaining({ role: "group" })); + expect(group?.name).toBeUndefined(); + expect(group && isVomReferenceNode(group)).toBe(false); + expect(buttons.map((node) => node.parentId)).toEqual([group?.id, group?.id]); + }); + + it("keeps uncertain single-branch wrappers flat", () => { + const scene = buildSemanticVomScene({ + viewport: { width: 800, height: 600 }, + rootFrameId: "main", + documents: [ + document( + "main", + [ + { + nodeId: "root", + backendDOMNodeId: 1, + role: { type: "role", value: "RootWebArea" }, + childIds: ["wrapper"], + }, + { + nodeId: "wrapper", + parentId: "root", + backendDOMNodeId: 2, + role: { type: "role", value: "generic" }, + childIds: ["button"], + }, + { + nodeId: "button", + parentId: "wrapper", + backendDOMNodeId: 3, + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Only action" }, + }, + ], + [dom(1, null, "body"), dom(2, 1, "div"), dom(3, 2, "button")], + ), + ], + }); + + expect(scene.nodes.some((node) => node.role === "group")).toBe(false); + expect(scene.nodes.find((node) => node.backendNodeId === 3)?.parentId).toBe( + scene.nodes.find((node) => node.backendNodeId === 1)?.id, + ); + }); + + it("does not infer across mixed content boundaries", () => { + const scene = buildSemanticVomScene({ + viewport: { width: 800, height: 600 }, + rootFrameId: "main", + documents: [ + document( + "main", + [ + { + nodeId: "root", + backendDOMNodeId: 1, + role: { type: "role", value: "RootWebArea" }, + childIds: ["wrapper"], + }, + { + nodeId: "wrapper", + parentId: "root", + backendDOMNodeId: 2, + role: { type: "role", value: "generic" }, + childIds: ["first", "second", "paragraph"], + }, + { + nodeId: "first", + parentId: "wrapper", + backendDOMNodeId: 3, + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "First" }, + }, + { + nodeId: "second", + parentId: "wrapper", + backendDOMNodeId: 4, + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Second" }, + }, + { + nodeId: "paragraph", + parentId: "wrapper", + backendDOMNodeId: 5, + role: { type: "role", value: "paragraph" }, + }, + ], + [ + dom(1, null, "body"), + dom(2, 1, "div"), + dom(3, 2, "button"), + dom(4, 2, "button"), + dom(5, 2, "p", {}, "Description"), + ], + ), + ], + }); + + expect(scene.nodes.some((node) => node.role === "group")).toBe(false); + }); + + it("keeps only the innermost reliable inferred group", () => { + const scene = buildSemanticVomScene({ + viewport: { width: 800, height: 600 }, + rootFrameId: "main", + documents: [ + document( + "main", + [ + { + nodeId: "root", + backendDOMNodeId: 1, + role: { type: "role", value: "RootWebArea" }, + childIds: ["outer"], + }, + { + nodeId: "outer", + parentId: "root", + backendDOMNodeId: 2, + role: { type: "role", value: "generic" }, + childIds: ["inner", "third"], + }, + { + nodeId: "inner", + parentId: "outer", + backendDOMNodeId: 3, + role: { type: "role", value: "generic" }, + childIds: ["first", "second"], + }, + { + nodeId: "first", + parentId: "inner", + backendDOMNodeId: 4, + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "First" }, + }, + { + nodeId: "second", + parentId: "inner", + backendDOMNodeId: 5, + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Second" }, + }, + { + nodeId: "third", + parentId: "outer", + backendDOMNodeId: 6, + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Third" }, + }, + ], + [ + dom(1, null, "body"), + dom(2, 1, "div"), + dom(3, 2, "div"), + dom(4, 3, "button"), + dom(5, 3, "button"), + dom(6, 2, "button"), + ], + ), + ], + }); + + const groups = scene.nodes.filter((node) => node.role === "group"); + expect(groups.map((node) => node.backendNodeId)).toEqual([3]); + expect(scene.nodes.find((node) => node.backendNodeId === 6)?.parentId).not.toBe(groups[0]?.id); + }); }); diff --git a/apps/extension/src/tools/vom/capture.ts b/apps/extension/src/tools/vom/capture.ts index d713d90..c221d8f 100644 --- a/apps/extension/src/tools/vom/capture.ts +++ b/apps/extension/src/tools/vom/capture.ts @@ -9,6 +9,7 @@ import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy"; import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge"; import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry"; import type { CdpRunner } from "../shared"; +import { clearHover, waitForHover } from "./hover-perception"; const REQUESTED_STYLES = ["position", "pointer-events", "cursor"] as const; const STYLE_COL = Object.fromEntries( @@ -460,36 +461,6 @@ function hoverStateExpression(): string { })()`; } -async function wait(ms: number, signal?: AbortSignal): Promise { - throwIfAborted(signal); - await new Promise((resolve, reject) => { - const timer = setTimeout(() => { - signal?.removeEventListener("abort", onAbort); - resolve(); - }, ms); - const onAbort = () => { - clearTimeout(timer); - signal?.removeEventListener("abort", onAbort); - reject(captureAbortError()); - }; - signal?.addEventListener("abort", onAbort, { once: true }); - }); -} - -async function clearHover(cdp: CdpRunner, tabId: number): Promise { - await cdp - .send(tabId, "Input.dispatchMouseEvent", { - type: "mouseMoved", - x: -10, - y: -10, - }) - .catch(() => - cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 }).catch(() => { - // best effort - }), - ); -} - function capturedText(node: CapturedNode): string | undefined { const value = node.attrs["aria-label"] ?? node.attrs.title ?? node.attrs.alt ?? node.textContent ?? ""; @@ -669,7 +640,7 @@ async function probeHoverSurfaces( try { await clearHover(cdp, tabId); throwIfAborted(options.signal); - await wait(HOVER_SETTLE_MS, options.signal); + await waitForHover(HOVER_SETTLE_MS, options.signal); const baselineReply = await cdp.send(tabId, "Runtime.evaluate", { expression: hoverStateExpression(), returnByValue: true, @@ -684,7 +655,7 @@ async function probeHoverSurfaces( y: candidate.y, }); throwIfAborted(options.signal); - await wait(HOVER_SETTLE_MS, options.signal); + await waitForHover(HOVER_SETTLE_MS, options.signal); const collected = await cdp.send(tabId, "Runtime.evaluate", { expression: hoverStateExpression(), returnByValue: true, diff --git a/apps/extension/src/tools/vom/hover-perception.ts b/apps/extension/src/tools/vom/hover-perception.ts new file mode 100644 index 0000000..253f792 --- /dev/null +++ b/apps/extension/src/tools/vom/hover-perception.ts @@ -0,0 +1,37 @@ +import type { CdpRunner } from "../shared"; + +function abortError(): Error { + const error = new Error("hover perception aborted"); + error.name = "AbortError"; + return error; +} + +export async function waitForHover(ms: number, signal?: AbortSignal): Promise { + if (signal?.aborted) throw abortError(); + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + signal?.removeEventListener("abort", onAbort); + resolve(); + }, ms); + const onAbort = () => { + clearTimeout(timer); + signal?.removeEventListener("abort", onAbort); + reject(abortError()); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + }); +} + +export async function clearHover(cdp: CdpRunner, tabId: number): Promise { + await cdp + .send(tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: -10, + y: -10, + }) + .catch(() => + cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 }).catch(() => { + // Best effort cleanup. + }), + ); +} diff --git a/apps/extension/src/tools/vom/name-enrichment.ts b/apps/extension/src/tools/vom/name-enrichment.ts new file mode 100644 index 0000000..c94eaf2 --- /dev/null +++ b/apps/extension/src/tools/vom/name-enrichment.ts @@ -0,0 +1,294 @@ +import { isVomReferenceNode } from "@browser-skill/vom"; +import { type CdpRunner, sendToCdpTarget } from "../shared"; +import type { FrameDocument, FrameOwnedAxNode } from "./frame-document"; +import { clearHover, waitForHover } from "./hover-perception"; +import { stableIdentifierName } from "./semantic-graph/name-evidence"; +import { frameBackendKey, type ResolvedSemanticGraph } from "./semantic-graph/types"; + +const MAX_TOOLTIP_CANDIDATES = 8; +const MAX_TOOLTIP_PROBE_MS = 2_400; +const TOOLTIP_SETTLE_MS = 250; +const CACHE_TTL_MS = 5 * 60_000; +const MAX_CACHE_ENTRIES = 500; + +interface TooltipCacheEntry { + name: string; + expiresAt: number; +} + +interface TooltipCandidate { + document: FrameDocument; + backendNodeId: number; + x: number; + y: number; + stableIdentifier: boolean; +} + +export interface TooltipNameProbeOptions { + signal?: AbortSignal; + hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise; +} + +const tooltipCache = new Map(); + +function cacheKey( + document: FrameDocument, + backendNodeId: number, +): string { + return `${document.target.tabId}\u0000${document.frameId}\u0000${document.url ?? ""}\u0000${backendNodeId}`; +} + +function cachedName( + document: FrameDocument, + backendNodeId: number, +): string | undefined { + const key = cacheKey(document, backendNodeId); + const entry = tooltipCache.get(key); + if (!entry) return undefined; + if (entry.expiresAt <= Date.now()) { + tooltipCache.delete(key); + return undefined; + } + return entry.name; +} + +function storeName( + document: FrameDocument, + backendNodeId: number, + name: string, +): void { + if (tooltipCache.size >= MAX_CACHE_ENTRIES) { + const oldest = tooltipCache.keys().next().value; + if (oldest !== undefined) tooltipCache.delete(oldest); + } + tooltipCache.set(cacheKey(document, backendNodeId), { + name, + expiresAt: Date.now() + CACHE_TTL_MS, + }); +} + +function tooltipCollectorFunction(): string { + return `function() { + const ownRoot = this && typeof this.getRootNode === "function" ? this.getRootNode() : null; + const roots = ownRoot && ownRoot !== document ? [ownRoot, document] : [document]; + const items = []; + const seen = new Set(); + for (const root of roots) { + for (const el of Array.from(root.querySelectorAll("[role='tooltip']"))) { + const style = getComputedStyle(el); + const rect = el.getBoundingClientRect(); + if (rect.width <= 0 || rect.height <= 0) continue; + if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") continue; + const text = String(el.getAttribute("aria-label") || el.getAttribute("title") || el.textContent || "") + .replace(/\\s+/g, " ") + .trim(); + if (!text || text.length > 160 || seen.has(text)) continue; + seen.add(text); + items.push(text); + } + } + return items.slice(0, 20); + }`; +} + +function cleanTooltipDiff(before: string[], after: string[]): string | undefined { + const existing = new Set(before.map((value) => value.toLowerCase())); + const added = [...new Set(after.map((value) => value.replace(/\s+/g, " ").trim()))].filter( + (value) => value && !existing.has(value.toLowerCase()), + ); + return added.length === 1 ? added[0] : undefined; +} + +function candidates( + documents: FrameDocument[], + graph: ResolvedSemanticGraph, +): TooltipCandidate[] { + const documentByFrame = new Map(documents.map((document) => [document.frameId, document])); + const byFrame = new Map[]>(); + for (const node of graph.nodes.values()) { + if ( + node.vom.name || + !node.referenceable || + node.backendNodeId === undefined || + !isVomReferenceNode({ + ...node.vom, + id: 0, + parentId: null, + backendNodeId: node.backendNodeId, + frameId: node.frameId, + referenceable: true, + }) + ) { + continue; + } + const document = documentByFrame.get(node.frameId); + const rect = node.dom?.rect; + if (!document || !rect || rect.w <= 0 || rect.h <= 0) continue; + if ( + rect.x + rect.w <= 0 || + rect.y + rect.h <= 0 || + rect.x >= graph.viewport.width || + rect.y >= graph.viewport.height + ) { + continue; + } + if ( + node.vom.disabled || + node.vom.inert || + node.vom.pointerEvents === "none" || + node.vom.sensitive + ) { + continue; + } + const frameCandidates = byFrame.get(node.frameId) ?? []; + frameCandidates.push({ + document, + backendNodeId: node.backendNodeId, + x: (Math.max(0, rect.x) + Math.min(graph.viewport.width, rect.x + rect.w)) / 2, + y: (Math.max(0, rect.y) + Math.min(graph.viewport.height, rect.y + rect.h)) / 2, + stableIdentifier: stableIdentifierName(node.dom?.attrs ?? {}) !== undefined, + }); + byFrame.set(node.frameId, frameCandidates); + } + + const out: TooltipCandidate[] = []; + for (const stableIdentifier of [true, false]) { + const queues = documents.map((document) => + (byFrame.get(document.frameId) ?? []).filter( + (candidate) => candidate.stableIdentifier === stableIdentifier, + ), + ); + for (let index = 0; queues.some((queue) => index < queue.length); index += 1) { + for (const queue of queues) { + const candidate = queue[index]; + if (candidate) out.push(candidate); + } + } + } + return out; +} + +async function executionContextId( + cdp: CdpRunner, + document: FrameDocument, +): Promise { + const result = await sendToCdpTarget<{ executionContextId?: number }>( + cdp, + document.target, + "Page.createIsolatedWorld", + { + frameId: document.frameId, + worldName: "bsk-vom-tooltip", + grantUniveralAccess: false, + }, + ); + return result.executionContextId; +} + +async function resolvedObjectId( + cdp: CdpRunner, + document: FrameDocument, + contextId: number, + backendNodeId: number, +): Promise { + const result = await sendToCdpTarget<{ + object?: { objectId?: string }; + }>(cdp, document.target, "DOM.resolveNode", { + backendNodeId, + executionContextId: contextId, + }); + return result.object?.objectId; +} + +async function tooltipTexts( + cdp: CdpRunner, + document: FrameDocument, + objectId: string, +): Promise { + const result = await sendToCdpTarget<{ + result?: { value?: string[] }; + }>(cdp, document.target, "Runtime.callFunctionOn", { + functionDeclaration: tooltipCollectorFunction(), + objectId, + returnByValue: true, + }); + return result.result?.value ?? []; +} + +async function releaseObject( + cdp: CdpRunner, + document: FrameDocument, + objectId: string, +): Promise { + await sendToCdpTarget(cdp, document.target, "Runtime.releaseObject", { objectId }).catch( + () => undefined, + ); +} + +export async function probeTooltipNames( + cdp: CdpRunner, + tabId: number, + documents: FrameDocument[], + graph: ResolvedSemanticGraph, + options: TooltipNameProbeOptions = {}, +): Promise> { + const names = new Map(); + const pending: TooltipCandidate[] = []; + for (const candidate of candidates(documents, graph)) { + const cached = cachedName(candidate.document, candidate.backendNodeId); + if (cached) { + names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), cached); + } else if (pending.length < MAX_TOOLTIP_CANDIDATES) { + pending.push(candidate); + } + } + if (pending.length === 0) return names; + + const started = Date.now(); + const contexts = new Map(); + await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined); + try { + for (const candidate of pending) { + if (options.signal?.aborted) throw new DOMException("observation aborted", "AbortError"); + if (Date.now() - started >= MAX_TOOLTIP_PROBE_MS) break; + let objectId: string | undefined; + try { + let contextId = contexts.get(candidate.document.frameId); + if (contextId === undefined) { + contextId = await executionContextId(cdp, candidate.document); + if (contextId === undefined) continue; + contexts.set(candidate.document.frameId, contextId); + } + objectId = await resolvedObjectId( + cdp, + candidate.document, + contextId, + candidate.backendNodeId, + ); + if (!objectId) continue; + await clearHover(cdp, tabId); + const before = await tooltipTexts(cdp, candidate.document, objectId); + await cdp.send(tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: candidate.x, + y: candidate.y, + }); + await waitForHover(TOOLTIP_SETTLE_MS, options.signal); + const after = await tooltipTexts(cdp, candidate.document, objectId); + const name = cleanTooltipDiff(before, after); + if (!name) continue; + names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), name); + storeName(candidate.document, candidate.backendNodeId, name); + } catch (error) { + if (error instanceof Error && error.name === "AbortError") throw error; + } finally { + if (objectId) await releaseObject(cdp, candidate.document, objectId); + await clearHover(cdp, tabId); + } + } + } finally { + await clearHover(cdp, tabId); + await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined); + } + return names; +} diff --git a/apps/extension/src/tools/vom/semantic-graph/index.ts b/apps/extension/src/tools/vom/semantic-graph/index.ts index 211cad1..7e13a00 100644 --- a/apps/extension/src/tools/vom/semantic-graph/index.ts +++ b/apps/extension/src/tools/vom/semantic-graph/index.ts @@ -1,25 +1,58 @@ import type { Viewport, VomScene } from "@browser-skill/vom"; import type { FrameDocument } from "../frame-document"; -import { buildSemanticGraph } from "./build"; -import { projectSemanticGraph } from "./project"; -import { resolveSemanticGraph } from "./resolve"; -import type { SemanticAxNode } from "./types"; +import { buildSemanticGraph as buildGraph } from "./build"; +import { projectSemanticGraph as projectGraph } from "./project"; +import { resolveSemanticGraph as resolveGraph } from "./resolve"; +import { normalizeSemanticStructure as normalizeStructure } from "./structure"; +import type { + ResolvedSemanticGraph, + SemanticAxNode, + SemanticGraph, + StructuredSemanticGraph, +} from "./types"; export type { SemanticAxNode } from "./types"; -export function buildSemanticVomScene(input: { +export interface SemanticVomInput { documents: FrameDocument[]; viewport: Viewport; rootFrameId?: string; excludedBackendNodeIds?: ReadonlySet; -}): VomScene { +} + +export function buildSemanticGraph(input: SemanticVomInput): SemanticGraph { + return buildGraph(input); +} + +export function resolveSemanticGraph( + graph: SemanticGraph, + options: { + supplementalNames?: ReadonlyMap; + identifierFallback?: boolean; + } = {}, +): ResolvedSemanticGraph { + return resolveGraph(graph, options); +} + +export function normalizeSemanticStructure(graph: ResolvedSemanticGraph): StructuredSemanticGraph { + return normalizeStructure(graph); +} + +export function projectSemanticGraph(graph: StructuredSemanticGraph): VomScene { + return projectGraph(graph); +} + +export function buildSemanticVomScene( + input: SemanticVomInput & { + supplementalNames?: ReadonlyMap; + identifierFallback?: boolean; + }, +): VomScene { return projectSemanticGraph( - resolveSemanticGraph( - buildSemanticGraph({ - documents: input.documents, - viewport: input.viewport, - rootFrameId: input.rootFrameId, - excludedBackendNodeIds: input.excludedBackendNodeIds, + normalizeSemanticStructure( + resolveSemanticGraph(buildSemanticGraph(input), { + supplementalNames: input.supplementalNames, + identifierFallback: input.identifierFallback, }), ), ); diff --git a/apps/extension/src/tools/vom/semantic-graph/name-evidence.ts b/apps/extension/src/tools/vom/semantic-graph/name-evidence.ts new file mode 100644 index 0000000..6dad4bc --- /dev/null +++ b/apps/extension/src/tools/vom/semantic-graph/name-evidence.ts @@ -0,0 +1,34 @@ +const IDENTIFIER_ATTRS = ["data-testid", "data-test-id", "id"] as const; +const GENERIC_TOKENS = new Set(["btn", "button", "control", "icon", "item", "test", "tool"]); +const GENERATED_PREFIXES = new Set(["ember", "headlessui", "mui", "radix", "react", "rc"]); + +function identifierTokens(value: string): string[] { + return value + .replace(/([a-z0-9])([A-Z])/g, "$1 $2") + .split(/[^a-zA-Z0-9]+/) + .map((token) => token.toLowerCase()) + .filter(Boolean); +} + +function readableIdentifier(value: string | undefined): string | undefined { + const raw = value?.trim(); + if (!raw || raw.length < 3 || raw.length > 80) return undefined; + if (/^[a-f0-9]{8,}$/i.test(raw)) return undefined; + + const tokens = identifierTokens(raw); + if (GENERATED_PREFIXES.has(tokens[0])) return undefined; + while (tokens.length > 1 && GENERIC_TOKENS.has(tokens[0])) tokens.shift(); + while (tokens.length > 1 && GENERIC_TOKENS.has(tokens[tokens.length - 1])) tokens.pop(); + if (tokens.length === 0 || !tokens.some((token) => /[a-z]{3,}/.test(token))) return undefined; + return tokens + .map((token, index) => (index === 0 ? `${token[0].toUpperCase()}${token.slice(1)}` : token)) + .join(" "); +} + +export function stableIdentifierName(attrs: Readonly>): string | undefined { + for (const name of IDENTIFIER_ATTRS) { + const label = readableIdentifier(attrs[name]); + if (label) return label; + } + return undefined; +} diff --git a/apps/extension/src/tools/vom/semantic-graph/project.ts b/apps/extension/src/tools/vom/semantic-graph/project.ts index fe5ef27..37e40dc 100644 --- a/apps/extension/src/tools/vom/semantic-graph/project.ts +++ b/apps/extension/src/tools/vom/semantic-graph/project.ts @@ -1,14 +1,11 @@ -import { - applyVomInteractionRecovery, - isVomReferenceNode, - type VomNode, - type VomScene, -} from "@browser-skill/vom"; -import type { ResolvedSemanticGraph, ResolvedSemanticNode, SemanticNodeId } from "./types"; +import type { VomNode, VomScene } from "@browser-skill/vom"; +import type { SemanticNodeId, StructuredSemanticGraph, StructuredSemanticNode } from "./types"; -const TRANSPARENT_ROLES = new Set(["", "generic", "none", "presentation", "inlinetextbox"]); +function retained(node: StructuredSemanticNode): boolean { + return node.structure.kind === "keep" || node.structure.kind === "inferred-group"; +} -function connectedFrameIds(graph: ResolvedSemanticGraph): Set { +function connectedFrameIds(graph: StructuredSemanticGraph): Set { const connected = new Set([graph.rootFrameId]); const childrenByParent = new Map(); for (const frame of graph.frames.values()) { @@ -24,7 +21,7 @@ function connectedFrameIds(graph: ResolvedSemanticGraph): Set { const frame = graph.frames.get(frameId); if (!frame?.ownerNodeId) continue; const owner = graph.nodes.get(frame.ownerNodeId); - if (!owner || owner.excluded) continue; + if (!owner || !retained(owner)) continue; connected.add(frame.frameId); queue.push(frame.frameId); } @@ -32,121 +29,9 @@ function connectedFrameIds(graph: ResolvedSemanticGraph): Set { return connected; } -function semanticParentId( - graph: ResolvedSemanticGraph, - node: ResolvedSemanticNode, -): SemanticNodeId | undefined { - let localParent = node.axParentId ?? node.domParentId; - if (node.axParentId && node.domParentId && node.axParentId !== node.domParentId) { - const axParent = graph.nodes.get(node.axParentId); - const axParentRole = axParent?.vom.role?.toLowerCase() ?? ""; - const axParentIsRoot = axParentRole === "rootwebarea" || axParentRole === "webarea"; - const axParentIsStructural = - !TRANSPARENT_ROLES.has(axParentRole) || - Boolean( - axParent?.vom.name || - axParent?.vom.text || - (axParent ? hasMeaningfulRelations(axParent) : false), - ); - localParent = !axParentIsRoot && axParentIsStructural ? node.axParentId : node.domParentId; - } - if (localParent) return localParent; - if (node.frameId === graph.rootFrameId) return undefined; - return graph.frames.get(node.frameId)?.ownerNodeId; -} - -function isChildDocumentRoot(graph: ResolvedSemanticGraph, node: ResolvedSemanticNode): boolean { - if (node.frameId === graph.rootFrameId) return false; - if (semanticParentId(graph, node) !== graph.frames.get(node.frameId)?.ownerNodeId) return false; - const role = node.vom.role?.toLowerCase() ?? ""; - return role === "rootwebarea" || role === "webarea"; -} - -function hasMeaningfulRelations(node: ResolvedSemanticNode): boolean { - const attrs = node.vom.attrs ?? {}; - return ["aria-controls", "aria-labelledby", "aria-owns", "aria-describedby"].some((name) => - attrs[name]?.trim(), - ); -} - -function shouldKeepNode( - graph: ResolvedSemanticGraph, - node: ResolvedSemanticNode, - recovered: VomNode, -): boolean { - if (node.excluded || isChildDocumentRoot(graph, node)) return false; - if (node.ax?.ignored) return false; - const role = recovered.role?.toLowerCase() ?? ""; - if (!TRANSPARENT_ROLES.has(role)) return true; - return Boolean( - recovered.name || - recovered.text || - recovered.value || - recovered.placeholder || - hasMeaningfulRelations(node) || - (recovered.rect && ["fixed", "sticky"].includes(recovered.position)), - ); -} - -function redundantSourceNodes( - graph: ResolvedSemanticGraph, - nodes: ResolvedSemanticNode[], -): Set { - const byKey = new Map(); - for (const node of nodes) { - const name = node.vom.name?.trim().toLowerCase() ?? ""; - if (!name || !isVomReferenceNode({ ...node.vom, id: 0, parentId: null, referenceable: true })) { - continue; - } - const key = `${node.frameId}\u0000${semanticParentId(graph, node) ?? ""}\u0000${node.vom.role?.toLowerCase() ?? ""}\u0000${name}\u0000${node.vom.sensitive === true}`; - const matches = byKey.get(key) ?? []; - matches.push(node); - byKey.set(key, matches); - } - const duplicates = new Set(); - for (const matches of byKey.values()) { - const axOnly = matches.filter((node) => node.ax && !node.dom); - const domOnly = matches.filter((node) => node.dom && !node.ax); - if (matches.length === 2 && axOnly.length === 1 && domOnly.length === 1) { - duplicates.add(domOnly[0].id); - } - } - return duplicates; -} - -function nearestKeptParent( - graph: ResolvedSemanticGraph, - nodeId: SemanticNodeId, - kept: ReadonlySet, - memo: Map, -): SemanticNodeId | undefined { - if (memo.has(nodeId)) return memo.get(nodeId); - let current = semanticParentId(graph, graph.nodes.get(nodeId) as ResolvedSemanticNode); - const seen = new Set(); - const path: SemanticNodeId[] = []; - let result: SemanticNodeId | undefined; - while (current && !seen.has(current)) { - if (kept.has(current)) { - result = current; - break; - } - if (memo.has(current)) { - result = memo.get(current); - break; - } - seen.add(current); - path.push(current); - const parent = graph.nodes.get(current); - current = parent ? semanticParentId(graph, parent) : undefined; - } - memo.set(nodeId, result); - for (const id of path) memo.set(id, result); - return result; -} - function domAncestors( - graph: ResolvedSemanticGraph, - node: ResolvedSemanticNode, + graph: StructuredSemanticGraph, + node: StructuredSemanticNode, numericId: ReadonlyMap, ): number[] { const ancestors: number[] = []; @@ -161,10 +46,7 @@ function domAncestors( return ancestors; } -export function projectSemanticGraph(graph: ResolvedSemanticGraph): VomScene { - const frameOwnerIds = new Set( - [...graph.frames.values()].flatMap((frame) => frame.ownerNodeId ?? []), - ); +export function projectSemanticGraph(graph: StructuredSemanticGraph): VomScene { const connectedFrames = connectedFrameIds(graph); const orderedNodes = [...graph.frames.values()] .filter((frame) => connectedFrames.has(frame.frameId)) @@ -172,67 +54,28 @@ export function projectSemanticGraph(graph: ResolvedSemanticGraph): VomScene { .flatMap((frame) => (graph.nodeIdsByFrame.get(frame.frameId) ?? []) .map((id) => graph.nodes.get(id)) - .filter((node): node is ResolvedSemanticNode => node !== undefined) + .filter((node): node is StructuredSemanticNode => node !== undefined && retained(node)) .sort((a, b) => a.sourceOrder - b.sourceOrder), ); - const provisionalNumericId = new Map(); - orderedNodes.forEach((node, index) => provisionalNumericId.set(node.id, index + 1)); - const provisionalNodes: VomNode[] = orderedNodes.map((node) => ({ + const numericId = new Map(); + orderedNodes.forEach((node, index) => numericId.set(node.id, index + 1)); + const nodes: VomNode[] = orderedNodes.map((node) => ({ ...node.vom, - id: provisionalNumericId.get(node.id) as number, - parentId: (() => { - const parent = semanticParentId(graph, node); - return parent ? (provisionalNumericId.get(parent) ?? null) : null; - })(), + id: numericId.get(node.id) as number, + parentId: node.semanticParentId ? (numericId.get(node.semanticParentId) ?? null) : null, ...(node.backendNodeId !== undefined ? { backendNodeId: node.backendNodeId } : {}), frameId: node.frameId, contextScopeId: graph.frames.get(node.frameId)?.contextScopeId ?? node.frameId, referenceable: node.referenceable, + ...(node.dom + ? { + domParentId: + node.domParentId === undefined ? null : (numericId.get(node.domParentId) ?? null), + domAncestorIds: domAncestors(graph, node, numericId), + } + : {}), })); - const recoveredById = new Map( - applyVomInteractionRecovery(provisionalNodes).map((node) => [node.id, node]), - ); - const duplicates = redundantSourceNodes(graph, orderedNodes); - const kept = new Set(); - for (const node of orderedNodes) { - if (duplicates.has(node.id)) continue; - const id = provisionalNumericId.get(node.id) as number; - const recovered = recoveredById.get(id) as VomNode; - if ((!node.excluded && frameOwnerIds.has(node.id)) || shouldKeepNode(graph, node, recovered)) { - kept.add(node.id); - } - } - - const numericId = new Map(); - let nextId = 1; - for (const node of orderedNodes) { - if (!kept.has(node.id)) continue; - numericId.set(node.id, nextId); - nextId += 1; - } - - const nodes: VomNode[] = []; - const keptParentMemo = new Map(); - for (const node of orderedNodes) { - const id = numericId.get(node.id); - if (id === undefined) continue; - const provisional = recoveredById.get(provisionalNumericId.get(node.id) as number) as VomNode; - const parent = nearestKeptParent(graph, node.id, kept, keptParentMemo); - const domParent = node.domParentId ? numericId.get(node.domParentId) : undefined; - nodes.push({ - ...provisional, - id, - parentId: parent ? (numericId.get(parent) ?? null) : null, - ...(node.backendNodeId !== undefined ? { backendNodeId: node.backendNodeId } : {}), - ...(node.dom - ? { - domParentId: node.domParentId === undefined ? null : (domParent ?? null), - domAncestorIds: domAncestors(graph, node, numericId), - } - : {}), - }); - } return { viewport: graph.viewport, diff --git a/apps/extension/src/tools/vom/semantic-graph/resolve.ts b/apps/extension/src/tools/vom/semantic-graph/resolve.ts index 9a3b761..03281cb 100644 --- a/apps/extension/src/tools/vom/semantic-graph/resolve.ts +++ b/apps/extension/src/tools/vom/semantic-graph/resolve.ts @@ -1,4 +1,5 @@ import type { VomNode } from "@browser-skill/vom"; +import { stableIdentifierName } from "./name-evidence"; import type { ResolvedSemanticGraph, ResolvedSemanticNode, @@ -7,6 +8,7 @@ import type { SemanticGraphNode, SemanticNodeId, } from "./types"; +import { frameBackendKey } from "./types"; const FORM_TAGS = new Set(["input", "textarea", "select"]); const NATIVE_CONTROL_TAGS = new Set(["button", "input", "select", "textarea"]); @@ -57,8 +59,6 @@ function normalizeRole(value: string | undefined): string | undefined { } function nativeRole(tag: string, attrs: Record): string | undefined { - const explicit = normalizeRole(attrs.role); - if (explicit) return explicit; if (tag === "button") return "button"; if (tag === "textarea") return "textbox"; if (tag === "select") return attrs.multiple === undefined ? "combobox" : "listbox"; @@ -90,14 +90,29 @@ function nativeRole(tag: string, attrs: Record): string | undefi return undefined; } -function resolvedRole(node: SemanticGraphNode): string | undefined { +function domRole(tag: string, attrs: Record): string | undefined { + return normalizeRole(attrs.role) ?? nativeRole(tag, attrs); +} + +function resolvedRole(node: SemanticGraphNode): { + value?: string; + source: ResolvedSemanticNode["roleSource"]; +} { const rawAxRole = clean(axValue(node.ax?.role)); const axRole = normalizeRole(rawAxRole); - const domRole = nativeRole(node.dom?.tag.toLowerCase() ?? "", node.dom?.attrs ?? {}); - if (!axRole || ["generic", "none", "presentation"].includes(axRole)) { - return domRole ?? rawAxRole; + const attrs = node.dom?.attrs ?? {}; + const explicitDomRole = normalizeRole(attrs.role); + const nativeDomRole = nativeRole(node.dom?.tag.toLowerCase() ?? "", attrs); + const usableAxRole = node.ax?.ignored !== true; + if (usableAxRole && axRole && !["generic", "none", "presentation"].includes(axRole)) { + return { value: rawAxRole, source: "ax" }; } - return rawAxRole; + if (explicitDomRole) return { value: explicitDomRole, source: "dom-explicit" }; + if (nativeDomRole) return { value: nativeDomRole, source: "dom-native" }; + if (rawAxRole) { + return { value: rawAxRole, source: usableAxRole ? "ax" : "ax-ignored" }; + } + return { source: "none" }; } function sensitive(node: SemanticGraphNode): boolean { @@ -143,14 +158,6 @@ function externalHrefHost( } } -function iconKeyword(value: string | undefined): string | undefined { - const token = clean(value) - ?.split(/[#/.\s:_-]+/) - .filter(Boolean) - .pop(); - return token && /^[a-z][a-z0-9_-]{1,32}$/i.test(token) ? token.toLowerCase() : undefined; -} - interface ResolveIndexes { domChildren: Map; axChildren: Map; @@ -214,7 +221,7 @@ function buildIndexes(graph: SemanticGraph): ResolveIndexes { for (const childId of domChildren.get(nodeId) ?? []) { const child = graph.nodes.get(childId); const childRole = child - ? nativeRole(child.dom?.tag.toLowerCase() ?? "", child.dom?.attrs ?? {}) + ? domRole(child.dom?.tag.toLowerCase() ?? "", child.dom?.attrs ?? {}) : undefined; if (childRole && INTERACTIVE_ROLES.has(childRole.toLowerCase())) continue; const childText = domText(childId); @@ -262,10 +269,8 @@ function buildIndexes(graph: SemanticGraph): ResolveIndexes { for (const childId of domChildren.get(nodeId) ?? []) { const child = graph.nodes.get(childId); const childRole = - normalizeRole(axValue(child?.ax?.role)) ?? - (child - ? nativeRole(child.dom?.tag.toLowerCase() ?? "", child.dom?.attrs ?? {}) - : undefined); + (child?.ax?.ignored === true ? undefined : normalizeRole(axValue(child?.ax?.role))) ?? + (child ? domRole(child.dom?.tag.toLowerCase() ?? "", child.dom?.attrs ?? {}) : undefined); if ( (child?.dom && NATIVE_CONTROL_TAGS.has(child.dom.tag.toLowerCase())) || INTERACTIVE_ROLES.has(childRole ?? "") || @@ -359,7 +364,7 @@ function precedingDomText( for (let offset = 1; offset <= 4 && index - offset >= 0; offset += 1) { const sibling = graph.nodes.get(siblings[index - offset]); const siblingRole = sibling - ? nativeRole(sibling.dom?.tag.toLowerCase() ?? "", sibling.dom?.attrs ?? {}) + ? domRole(sibling.dom?.tag.toLowerCase() ?? "", sibling.dom?.attrs ?? {}) : undefined; if (siblingRole && INTERACTIVE_ROLES.has(siblingRole.toLowerCase())) break; const text = indexes.domText(siblings[index - offset])?.replace(/[：:]\s*$/, ""); @@ -368,49 +373,23 @@ function precedingDomText( return undefined; } -function iconHint( - nodeId: SemanticNodeId, - graph: SemanticGraph, - indexes: ResolveIndexes, - depth = 0, -): string | undefined { - if (depth > 3) return undefined; - for (const childId of indexes.domChildren.get(nodeId) ?? []) { - const child = graph.nodes.get(childId); - const attrs = child?.dom?.attrs ?? {}; - const hint = - iconKeyword(attrs["aria-label"]) ?? - iconKeyword(attrs.title) ?? - iconKeyword(attrs.href) ?? - iconKeyword(attrs["xlink:href"]) ?? - iconKeyword(axValue(child?.ax?.name)) ?? - iconKeyword(attrs.class) ?? - iconKeyword(child?.dom?.textContent); - if (hint) return hint; - const nested = iconHint(childId, graph, indexes, depth + 1); - if (nested) return nested; - } - return undefined; -} - function resolvedName( node: SemanticGraphNode, role: string | undefined, graph: SemanticGraph, indexes: ResolveIndexes, + supplementalNames: ReadonlyMap, + identifierFallback: boolean, ): string | undefined { const attrs = node.dom?.attrs ?? {}; const formControl = FORM_TAGS.has(node.dom?.tag.toLowerCase() ?? ""); const descendantText = indexes.domText(node.id); - const canUseIconHint = - !indexes.hasNativeDescendant(node.id) && - (INTERACTIVE_ROLES.has(role?.toLowerCase() ?? "") || - node.dom?.cursor === "pointer" || - attrs.onclick !== undefined || - attrs.tabindex !== undefined); + const interactive = + INTERACTIVE_ROLES.has(role?.toLowerCase() ?? "") || + NATIVE_CONTROL_TAGS.has(node.dom?.tag.toLowerCase() ?? ""); const candidates = [ - axValue(node.ax?.name), - indexes.axText(node.id), + node.ax?.ignored === true ? undefined : axValue(node.ax?.name), + node.ax?.ignored === true ? undefined : indexes.axText(node.id), ariaLabelledText(node, indexes), clean(attrs["aria-label"]), nativeLabelText(node, graph, indexes), @@ -420,7 +399,10 @@ function resolvedName( formControl ? precedingAxText(node, graph, indexes) : undefined, formControl ? precedingDomText(node, graph, indexes) : undefined, formControl ? clean(node.dom?.formPlaceholder ?? attrs.placeholder) : undefined, - canUseIconHint ? iconHint(node.id, graph, indexes) : undefined, + !interactive || node.backendNodeId === undefined + ? undefined + : clean(supplementalNames.get(frameBackendKey(node.frameId, node.backendNodeId))), + identifierFallback && interactive ? stableIdentifierName(attrs) : undefined, ]; let name = candidates.find((candidate) => candidate !== undefined); const hasPopup = axProperty(node.ax, "hasPopup"); @@ -462,12 +444,21 @@ function nearbyText( return clean(texts.join(" ")); } -export function resolveSemanticGraph(graph: SemanticGraph): ResolvedSemanticGraph { +export function resolveSemanticGraph( + graph: SemanticGraph, + options: { + supplementalNames?: ReadonlyMap; + identifierFallback?: boolean; + } = {}, +): ResolvedSemanticGraph { const indexes = buildIndexes(graph); + const supplementalNames = options.supplementalNames ?? new Map(); + const identifierFallback = options.identifierFallback ?? true; const nodes = new Map(); for (const node of graph.nodes.values()) { - const role = resolvedRole(node); - const name = resolvedName(node, role, graph, indexes); + const roleResolution = resolvedRole(node); + const role = roleResolution.value; + const name = resolvedName(node, role, graph, indexes, supplementalNames, identifierFallback); const isSensitive = sensitive(node); const sourceValue = node.dom?.formValue ?? axValue(node.ax?.value); const value = isSensitive ? undefined : sourceValue; @@ -505,9 +496,9 @@ export function resolveSemanticGraph(graph: SemanticGraph): ResolvedSemanticGrap insideNative: insideNative(node, graph), ...(role?.toLowerCase() === "link" ? { href: externalHrefHost(attrs.href, frame?.url) } : {}), }; - const selected = axProperty(node.ax, "selected"); - const expanded = axProperty(node.ax, "expanded"); - const controls = axProperty(node.ax, "controls"); + const selected = node.ax?.ignored === true ? undefined : axProperty(node.ax, "selected"); + const expanded = node.ax?.ignored === true ? undefined : axProperty(node.ax, "expanded"); + const controls = node.ax?.ignored === true ? undefined : axProperty(node.ax, "controls"); if (selected === "true" && !vom.attrs?.["aria-selected"]) { vom.attrs = { ...(vom.attrs ?? {}), "aria-selected": "true" }; } @@ -521,6 +512,7 @@ export function resolveSemanticGraph(graph: SemanticGraph): ResolvedSemanticGrap ...node, vom, referenceable: node.backendNodeId !== undefined, + roleSource: roleResolution.source, }); } return { ...graph, nodes }; diff --git a/apps/extension/src/tools/vom/semantic-graph/structure.ts b/apps/extension/src/tools/vom/semantic-graph/structure.ts new file mode 100644 index 0000000..9b5b1a6 --- /dev/null +++ b/apps/extension/src/tools/vom/semantic-graph/structure.ts @@ -0,0 +1,436 @@ +import { + applyVomInteractionRecovery, + isVomReferenceNode, + isVomStructuralRole, + type VomNode, +} from "@browser-skill/vom"; +import type { + ResolvedSemanticGraph, + ResolvedSemanticNode, + SemanticNodeId, + StructureDecision, + StructuredSemanticGraph, + StructuredSemanticNode, +} from "./types"; + +const TRANSPARENT_ROLES = new Set(["", "generic", "none", "presentation", "inlinetextbox"]); +const ROOT_ROLES = new Set(["rootwebarea", "webarea"]); +const CONTENT_BOUNDARY_ROLES = new Set([ + "article", + "dialog", + "document", + "feed", + "form", + "grid", + "heading", + "iframe", + "list", + "main", + "navigation", + "paragraph", + "region", + "section", + "table", + "tree", +]); + +function hasMeaningfulRelations(node: ResolvedSemanticNode): boolean { + const attrs = node.vom.attrs ?? {}; + return ["aria-controls", "aria-labelledby", "aria-owns", "aria-describedby"].some((name) => + attrs[name]?.trim(), + ); +} + +function explicitStructural(node: ResolvedSemanticNode): boolean { + return ( + isVomStructuralRole(node.vom.role) && + node.roleSource !== "none" && + node.roleSource !== "ax-ignored" + ); +} + +function initialParentId( + graph: ResolvedSemanticGraph, + node: ResolvedSemanticNode, +): SemanticNodeId | undefined { + let parentId = node.axParentId ?? node.domParentId; + if (node.axParentId && node.domParentId && node.axParentId !== node.domParentId) { + const axParent = graph.nodes.get(node.axParentId); + const axRole = axParent?.vom.role?.toLowerCase() ?? ""; + const meaningfulAxParent = + axParent !== undefined && + !ROOT_ROLES.has(axRole) && + (explicitStructural(axParent) || + !TRANSPARENT_ROLES.has(axRole) || + Boolean(axParent.vom.name || axParent.vom.text || hasMeaningfulRelations(axParent))); + parentId = meaningfulAxParent ? node.axParentId : node.domParentId; + } + if (parentId) return parentId; + if (node.frameId === graph.rootFrameId) return undefined; + return graph.frames.get(node.frameId)?.ownerNodeId; +} + +function orderedNodes(graph: ResolvedSemanticGraph): ResolvedSemanticNode[] { + return [...graph.frames.values()] + .sort((a, b) => a.order - b.order) + .flatMap((frame) => + (graph.nodeIdsByFrame.get(frame.frameId) ?? []) + .map((id) => graph.nodes.get(id)) + .filter((node): node is ResolvedSemanticNode => node !== undefined) + .sort((a, b) => a.sourceOrder - b.sourceOrder), + ); +} + +function recoveredNodes( + graph: ResolvedSemanticGraph, + nodes: ResolvedSemanticNode[], +): Map { + const numericId = new Map(nodes.map((node, index) => [node.id, index + 1])); + const semanticId = new Map([...numericId].map(([id, numeric]) => [numeric, id])); + const provisional = nodes.map((node) => ({ + ...node.vom, + id: numericId.get(node.id) as number, + parentId: (() => { + const parent = initialParentId(graph, node); + return parent ? (numericId.get(parent) ?? null) : null; + })(), + ...(node.backendNodeId !== undefined ? { backendNodeId: node.backendNodeId } : {}), + frameId: node.frameId, + contextScopeId: graph.frames.get(node.frameId)?.contextScopeId ?? node.frameId, + referenceable: node.referenceable, + })); + return new Map( + applyVomInteractionRecovery(provisional).flatMap((node) => { + const id = semanticId.get(node.id); + return id ? [[id, node] as const] : []; + }), + ); +} + +function redundantSourceNodes( + graph: ResolvedSemanticGraph, + nodes: ResolvedSemanticNode[], + recovered: ReadonlyMap, +): Set { + const byKey = new Map(); + for (const node of nodes) { + const output = recovered.get(node.id) as VomNode; + const name = output.name?.trim().toLowerCase() ?? ""; + if (!name || !isVomReferenceNode({ ...output, referenceable: true })) continue; + const key = `${node.frameId}\u0000${initialParentId(graph, node) ?? ""}\u0000${output.role?.toLowerCase() ?? ""}\u0000${name}\u0000${output.sensitive === true}`; + const matches = byKey.get(key) ?? []; + matches.push(node); + byKey.set(key, matches); + } + const redundant = new Set(); + for (const matches of byKey.values()) { + const axOnly = matches.filter((node) => node.ax && !node.dom); + const domOnly = matches.filter((node) => node.dom && !node.ax); + if (matches.length === 2 && axOnly.length === 1 && domOnly.length === 1) { + redundant.add(domOnly[0].id); + } + } + return redundant; +} + +function baseDecision( + graph: ResolvedSemanticGraph, + node: ResolvedSemanticNode, + output: VomNode, + frameOwnerIds: ReadonlySet, + redundant: ReadonlySet, +): StructureDecision { + if (node.excluded) return { kind: "excluded", reason: "excluded" }; + if (redundant.has(node.id)) return { kind: "excluded", reason: "redundant-source" }; + if (frameOwnerIds.has(node.id)) return { kind: "keep", reason: "frame-owner" }; + if (node.frameId !== graph.rootFrameId && ROOT_ROLES.has(output.role?.toLowerCase() ?? "")) { + return { kind: "transparent", reason: "child-document-root" }; + } + if (node.ax?.ignored === true && !node.dom) { + return { kind: "transparent", reason: "ax-ignored" }; + } + const role = output.role?.toLowerCase() ?? ""; + if (!TRANSPARENT_ROLES.has(role)) return { kind: "keep", reason: "semantic" }; + if ( + output.name || + output.text || + output.value || + output.placeholder || + hasMeaningfulRelations(node) || + (output.rect && ["fixed", "sticky"].includes(output.position)) + ) { + return { kind: "keep", reason: "semantic" }; + } + return { kind: "transparent", reason: "wrapper" }; +} + +function buildChildren( + nodes: ResolvedSemanticNode[], + relation: "axParentId" | "domParentId", +): Map { + const children = new Map(); + for (const node of nodes) { + const parentId = node[relation]; + if (!parentId) continue; + const siblings = children.get(parentId) ?? []; + siblings.push(node.id); + children.set(parentId, siblings); + } + return children; +} + +function explicitDomAncestors( + graph: ResolvedSemanticGraph, + nodes: ResolvedSemanticNode[], + domChildren: ReadonlyMap, +): Set { + const inside = new Set(); + const roots = nodes.filter((node) => !node.domParentId); + const queue = roots.map((node) => node.id); + for (let index = 0; index < queue.length; index += 1) { + const parentId = queue[index]; + const parent = graph.nodes.get(parentId); + const parentIsBoundary = + parent !== undefined && + explicitStructural(parent) && + !ROOT_ROLES.has(parent.vom.role?.toLowerCase() ?? ""); + for (const childId of domChildren.get(parentId) ?? []) { + if (inside.has(parentId) || parentIsBoundary) inside.add(childId); + queue.push(childId); + } + } + return inside; +} + +function reliableGroupCandidates( + graph: ResolvedSemanticGraph, + nodes: ResolvedSemanticNode[], + recovered: ReadonlyMap, + decisions: ReadonlyMap, + domChildren: ReadonlyMap, +): Set { + const insideExplicitStructure = explicitDomAncestors(graph, nodes, domChildren); + const remainingChildren = new Map( + nodes.map((node) => [node.id, domChildren.get(node.id)?.length ?? 0]), + ); + const summaries = new Map(); + const queue = nodes + .filter((node) => (remainingChildren.get(node.id) ?? 0) === 0) + .map((node) => node.id); + for (let index = 0; index < queue.length; index += 1) { + const nodeId = queue[index]; + const node = graph.nodes.get(nodeId); + const output = recovered.get(nodeId); + if (!node || !output) continue; + const role = output.role?.toLowerCase() ?? ""; + const excluded = decisions.get(nodeId)?.kind === "excluded"; + const blocked = excluded || explicitStructural(node) || CONTENT_BOUNDARY_ROLES.has(role); + const summary = excluded + ? { blocked: true, interactions: 0 } + : isVomReferenceNode(output) + ? { blocked: false, interactions: 1 } + : blocked + ? { blocked: true, interactions: 0 } + : (domChildren.get(nodeId) ?? []).reduce( + (combined, childId) => { + const child = summaries.get(childId); + return { + blocked: combined.blocked || !child || child.blocked, + interactions: Math.min(2, combined.interactions + (child?.interactions ?? 0)), + }; + }, + { blocked: false, interactions: 0 }, + ); + summaries.set(nodeId, summary); + const parentId = node.domParentId; + if (!parentId) continue; + const remaining = (remainingChildren.get(parentId) ?? 1) - 1; + remainingChildren.set(parentId, remaining); + if (remaining === 0) queue.push(parentId); + } + + const candidates = new Set(); + for (const node of nodes) { + const output = recovered.get(node.id) as VomNode; + const role = output.role?.toLowerCase() ?? ""; + if ( + !node.dom || + node.dom.tag.toLowerCase() !== "div" || + !TRANSPARENT_ROLES.has(role) || + insideExplicitStructure.has(node.id) || + decisions.get(node.id)?.kind === "excluded" || + node.dom.attrs.hidden !== undefined || + (node.dom.attrs["aria-hidden"] ?? "").toLowerCase() === "true" || + node.dom.attrs.inert !== undefined + ) { + continue; + } + + const branches = (domChildren.get(node.id) ?? []).map((id) => summaries.get(id)); + const blocked = branches.some((summary) => !summary || summary.blocked); + const contributingBranches = branches.filter( + (summary) => (summary?.interactions ?? 0) > 0, + ).length; + const interactions = branches.reduce( + (count, summary) => Math.min(2, count + (summary?.interactions ?? 0)), + 0, + ); + if (!blocked && contributingBranches >= 2 && interactions >= 2) { + candidates.add(node.id); + } + } + return candidates; +} + +function leafCandidates( + nodes: ResolvedSemanticNode[], + candidates: ReadonlySet, + domChildren: ReadonlyMap, +): Set { + const parentById = new Map( + nodes.flatMap((node) => (node.domParentId ? ([[node.id, node.domParentId]] as const) : [])), + ); + const childCount = new Map(nodes.map((node) => [node.id, domChildren.get(node.id)?.length ?? 0])); + const containsCandidate = new Set(); + const queue = nodes.filter((node) => (childCount.get(node.id) ?? 0) === 0).map((node) => node.id); + const selected = new Set(); + for (let index = 0; index < queue.length; index += 1) { + const nodeId = queue[index]; + if (candidates.has(nodeId) && !containsCandidate.has(nodeId)) selected.add(nodeId); + const parentId = parentById.get(nodeId); + if (!parentId) continue; + if (candidates.has(nodeId) || containsCandidate.has(nodeId)) containsCandidate.add(parentId); + const remaining = (childCount.get(parentId) ?? 1) - 1; + childCount.set(parentId, remaining); + if (remaining === 0) queue.push(parentId); + } + return selected; +} + +function nearestKept( + graph: ResolvedSemanticGraph, + startId: SemanticNodeId | undefined, + relation: "axParentId" | "domParentId", + kept: ReadonlySet, + memo: Map, +): SemanticNodeId | undefined { + let currentId = startId; + const path: SemanticNodeId[] = []; + const seen = new Set(); + let result: SemanticNodeId | undefined; + while (currentId && !seen.has(currentId)) { + if (kept.has(currentId)) { + result = currentId; + break; + } + if (memo.has(currentId)) { + result = memo.get(currentId); + break; + } + seen.add(currentId); + path.push(currentId); + currentId = graph.nodes.get(currentId)?.[relation]; + } + for (const id of path) memo.set(id, result); + return result; +} + +function finalParentId( + graph: ResolvedSemanticGraph, + node: ResolvedSemanticNode, + kept: ReadonlySet, + decisions: ReadonlyMap, + axParentMemo: Map, + domParentMemo: Map, +): SemanticNodeId | undefined { + const axParent = nearestKept(graph, node.axParentId, "axParentId", kept, axParentMemo); + const domParent = nearestKept(graph, node.domParentId, "domParentId", kept, domParentMemo); + const axNode = axParent ? graph.nodes.get(axParent) : undefined; + const domNode = domParent ? graph.nodes.get(domParent) : undefined; + const axIsRoot = ROOT_ROLES.has(axNode?.vom.role?.toLowerCase() ?? ""); + const axIsExplicit = axNode ? explicitStructural(axNode) && !axIsRoot : false; + const domIsStructural = + domNode !== undefined && + (explicitStructural(domNode) || decisions.get(domNode.id)?.kind === "inferred-group"); + if (axIsExplicit) return axParent; + if (domIsStructural) return domParent; + if (axParent && !axIsRoot) return axParent; + if (domParent) return domParent; + if (axParent) return axParent; + if (node.frameId === graph.rootFrameId) return undefined; + return graph.frames.get(node.frameId)?.ownerNodeId; +} + +function recoveredVom(node: VomNode, inferredGroup: boolean): ResolvedSemanticNode["vom"] { + const { + id: _id, + parentId: _parentId, + backendNodeId: _backendNodeId, + frameId: _frameId, + contextScopeId: _contextScopeId, + referenceable: _referenceable, + ...vom + } = node; + if (!inferredGroup) return vom; + const { name: _name, text: _text, nearbyText: _nearbyText, ...groupVom } = vom; + return { ...groupVom, role: "group" }; +} + +export function normalizeSemanticStructure(graph: ResolvedSemanticGraph): StructuredSemanticGraph { + const nodes = orderedNodes(graph); + const recovered = recoveredNodes(graph, nodes); + const redundant = redundantSourceNodes(graph, nodes, recovered); + const frameOwnerIds = new Set( + [...graph.frames.values()].flatMap((frame) => frame.ownerNodeId ?? []), + ); + const decisions = new Map(); + for (const node of nodes) { + decisions.set( + node.id, + baseDecision(graph, node, recovered.get(node.id) as VomNode, frameOwnerIds, redundant), + ); + } + + const domChildren = buildChildren(nodes, "domParentId"); + const candidates = reliableGroupCandidates(graph, nodes, recovered, decisions, domChildren); + for (const nodeId of leafCandidates(nodes, candidates, domChildren)) { + decisions.set(nodeId, { + kind: "inferred-group", + reason: "independent-interaction-branches", + }); + } + + const kept = new Set( + nodes + .filter((node) => { + const kind = decisions.get(node.id)?.kind; + return kind === "keep" || kind === "inferred-group"; + }) + .map((node) => node.id), + ); + const axParentMemo = new Map(); + const domParentMemo = new Map(); + const structuredNodes = new Map(); + for (const node of nodes) { + const structure = decisions.get(node.id) as StructureDecision; + const inferredGroup = structure.kind === "inferred-group"; + structuredNodes.set(node.id, { + ...node, + vom: recoveredVom(recovered.get(node.id) as VomNode, inferredGroup), + structure, + ...(kept.has(node.id) + ? { + semanticParentId: finalParentId( + graph, + node, + kept, + decisions, + axParentMemo, + domParentMemo, + ), + } + : {}), + }); + } + return { ...graph, nodes: structuredNodes }; +} diff --git a/apps/extension/src/tools/vom/semantic-graph/types.ts b/apps/extension/src/tools/vom/semantic-graph/types.ts index 3cb098b..bd26d76 100644 --- a/apps/extension/src/tools/vom/semantic-graph/types.ts +++ b/apps/extension/src/tools/vom/semantic-graph/types.ts @@ -54,12 +54,28 @@ export interface SemanticGraph { export interface ResolvedSemanticNode extends SemanticGraphNode { vom: Omit; referenceable: boolean; + roleSource: "ax" | "ax-ignored" | "dom-explicit" | "dom-native" | "none"; } export interface ResolvedSemanticGraph extends Omit { nodes: Map; } +export type StructureDecision = + | { kind: "keep"; reason: "semantic" | "frame-owner" } + | { kind: "inferred-group"; reason: "independent-interaction-branches" } + | { kind: "transparent"; reason: "wrapper" | "child-document-root" | "ax-ignored" } + | { kind: "excluded"; reason: "excluded" | "redundant-source" }; + +export interface StructuredSemanticNode extends ResolvedSemanticNode { + structure: StructureDecision; + semanticParentId?: SemanticNodeId; +} + +export interface StructuredSemanticGraph extends Omit { + nodes: Map; +} + export function frameBackendKey(frameId: string, backendNodeId: number): string { return `${frameId}\u0000${backendNodeId}`; } diff --git a/packages/vom/src/__tests__/render.test.ts b/packages/vom/src/__tests__/render.test.ts index d8a3d04..fa02ae5 100644 --- a/packages/vom/src/__tests__/render.test.ts +++ b/packages/vom/src/__tests__/render.test.ts @@ -333,6 +333,84 @@ describe("renderVom single-layer page", () => { expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]); }); + it("does not treat implementation CSS classes as control names", () => { + const out = renderVom( + scene([ + node({ id: 1, role: "RootWebArea", name: "Doc" }), + node({ + id: 2, + parentId: 1, + role: "generic", + cursor: "pointer", + attrs: { onclick: "handleClick()" }, + rect: { x: 20, y: 20, w: 40, h: 40 }, + }), + node({ + id: 3, + parentId: 2, + role: "img", + tag: "svg", + attrs: { class: "kocomz" }, + }), + ]), + ); + + expect(out.text).not.toContain("kocomz"); + expect(out.refs).toEqual([]); + }); + + it("recovers icon controls only from explicit semantic icon evidence", () => { + const out = renderVom( + scene([ + node({ id: 1, role: "RootWebArea", name: "Doc" }), + node({ + id: 2, + parentId: 1, + role: "generic", + cursor: "pointer", + attrs: { onclick: "handleFilter()" }, + rect: { x: 20, y: 20, w: 40, h: 40 }, + }), + node({ + id: 3, + parentId: 2, + role: "img", + tag: "svg", + attrs: { "aria-label": "Filter rows", class: "kocomz" }, + }), + ]), + ); + + expect(out.text).toContain('@e1 button "Filter rows"'); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]); + }); + + it("keeps a semantic image as evidence for its clickable wrapper", () => { + const out = renderVom( + scene([ + node({ id: 1, role: "RootWebArea", name: "Doc" }), + node({ + id: 2, + parentId: 1, + role: "generic", + cursor: "pointer", + rect: { x: 20, y: 20, w: 40, h: 40 }, + }), + node({ + id: 3, + parentId: 2, + role: "img", + name: "Account", + cursor: "pointer", + tag: "div", + }), + ]), + ); + + expect(out.text).toContain('@e1 button "Account"'); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]); + }); + it("keeps the deepest custom control when wrapper and child both look clickable", () => { const out = renderVom( scene([ diff --git a/packages/vom/src/index.ts b/packages/vom/src/index.ts index aab8507..afbbf36 100644 --- a/packages/vom/src/index.ts +++ b/packages/vom/src/index.ts @@ -1,4 +1,9 @@ -export { applyVomInteractionRecovery, isVomReferenceNode, renderVom } from "./render"; +export { + applyVomInteractionRecovery, + isVomReferenceNode, + isVomStructuralRole, + renderVom, +} from "./render"; export type { ActiveScopeBlock, BlockingLayer, diff --git a/packages/vom/src/render.ts b/packages/vom/src/render.ts index d20c785..b943dbd 100644 --- a/packages/vom/src/render.ts +++ b/packages/vom/src/render.ts @@ -75,6 +75,10 @@ const STRUCTURAL_ROLES = new Set([ "webarea", ]); +export function isVomStructuralRole(role: string | undefined): boolean { + return STRUCTURAL_ROLES.has(role?.toLowerCase() ?? ""); +} + const SENSITIVE_MASK = "•••"; const DEFAULT_MAX_DEPTH = Number.POSITIVE_INFINITY; const DEFAULT_MAX_TOKENS = Number.POSITIVE_INFINITY; @@ -193,7 +197,7 @@ function shouldRender(node: VomNode): boolean { const role = normalizedRole(node); if (SKIP_ROLES.has(role)) return false; if (INTERACTIVE_ROLES.has(role)) return true; - if (STRUCTURAL_ROLES.has(role)) return true; + if (isVomStructuralRole(role)) return true; return cleaned(node.name) !== undefined; } @@ -222,14 +226,30 @@ function isContentEditable(node: VomNode): boolean { return contentEditable === "true" || contentEditable === "plaintext-only"; } -function iconKeyword(value: string | undefined): string | undefined { - const cleanedValue = cleaned(value); - if (!cleanedValue) return undefined; - const last = cleanedValue - .split(/[#/.\s:_-]+/) +const ICON_REFERENCE_MARKERS = new Set(["glyph", "icon", "icons", "symbol"]); +const ICON_REFERENCE_DECORATORS = new Set(["fill", "filled", "line", "outline", "outlined"]); + +function iconReferenceName(value: string | undefined): string | undefined { + const reference = cleaned(value); + if (!reference) return undefined; + const fragment = reference.includes("#") ? reference.slice(reference.lastIndexOf("#") + 1) : ""; + const tokens = (fragment || reference) + ?.split(/[#/.\s:_-]+/) .filter(Boolean) - .pop(); - return last && /^[a-z][a-z0-9_-]{1,32}$/i.test(last) ? last.toLowerCase() : undefined; + .map((token) => token.toLowerCase()); + if (tokens.length === 0) return undefined; + if (fragment) { + const semanticTokens = tokens.filter( + (token) => !ICON_REFERENCE_MARKERS.has(token) && !ICON_REFERENCE_DECORATORS.has(token), + ); + return semanticTokens.length > 0 ? semanticTokens.join(" ") : undefined; + } + const marker = tokens.findLastIndex((token) => ICON_REFERENCE_MARKERS.has(token)); + if (marker < 0) return undefined; + const semanticTokens = tokens + .slice(marker + 1) + .filter((token) => !ICON_REFERENCE_DECORATORS.has(token)); + return semanticTokens.length > 0 ? semanticTokens.join(" ") : undefined; } function iconHintFromSubtree( @@ -241,12 +261,12 @@ function iconHintFromSubtree( for (const child of children.get(node.id) ?? []) { const attrs = child.attrs ?? {}; const hint = - iconKeyword(attrs["aria-label"]) ?? - iconKeyword(attrs.title) ?? - iconKeyword(attrs.href) ?? - iconKeyword(attrs["xlink:href"]) ?? - iconKeyword(attrs.class) ?? - iconKeyword(child.text); + cleaned(attrs["aria-label"]) ?? + cleaned(attrs.title) ?? + (normalizedRole(child) === "img" ? cleaned(child.name) : undefined) ?? + (["img", "svg", "use"].includes(normalizedTag(child)) + ? (iconReferenceName(attrs.href) ?? iconReferenceName(attrs["xlink:href"])) + : undefined); if (hint) return hint; const nested = iconHintFromSubtree(child, children, depth + 1); if (nested) return nested; @@ -272,6 +292,7 @@ function recoveredRole( node: VomNode, children?: Map, ): string | undefined { + const currentRole = normalizedRole(node); const explicitRole = node.attrs?.role?.toLowerCase() ?? ""; const hasHandler = hasRecoveryHandler(node); const focusable = isFocusable(node); @@ -293,7 +314,9 @@ function recoveredRole( : "button"; } if (hasHandler && hasName) return "button"; - if (node.cursor === "pointer" && hasName) return "button"; + if ((!currentRole || SKIP_ROLES.has(currentRole)) && node.cursor === "pointer" && hasName) { + return "button"; + } return undefined; }