From 5e0d26ccb270d8e418ed842be2eff8b75e23ebc5 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Wed, 22 Jul 2026 13:45:19 +0800 Subject: [PATCH] feat(vom): method update --- .../src/tools/__tests__/observation.test.ts | 75 +-- apps/extension/src/tools/observation.ts | 417 ++++++++-------- packages/vom/src/__tests__/render.test.ts | 88 ++++ packages/vom/src/render.ts | 452 +++++++++++++++++- packages/vom/src/types.ts | 10 + 5 files changed, 770 insertions(+), 272 deletions(-) diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index c9bb546..f275939 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -394,37 +394,33 @@ describe("buildVomScene", () => { ], }; - expect(buildVomScene(axNodes, captured)).toEqual({ - viewport: { width: 1000, height: 800 }, - nodes: [ - { - id: 100, - parentId: null, - role: "RootWebArea", - name: "Example", - tag: "body", - rect: { x: 0, y: 0, w: 1000, h: 800 }, - paintOrder: 0, - position: "static", - pointerEvents: "auto", - modal: false, - sensitive: false, - }, - { - id: 200, - parentId: 100, - role: "button", - name: "Submit", - tag: "button", - rect: { x: 20, y: 20, w: 120, h: 40 }, - paintOrder: 1, - position: "static", - pointerEvents: "auto", - modal: false, - sensitive: false, - }, - ], - }); + const scene = buildVomScene(axNodes, captured); + expect(scene.viewport).toEqual({ width: 1000, height: 800 }); + expect(scene.nodes[0]).toEqual( + expect.objectContaining({ + id: 100, + parentId: null, + role: "RootWebArea", + name: "Example", + tag: "body", + rect: { x: 0, y: 0, w: 1000, h: 800 }, + domParentId: null, + domAncestorIds: [], + }), + ); + expect(scene.nodes[1]).toEqual( + expect.objectContaining({ + id: 200, + parentId: 100, + role: "button", + name: "Submit", + tag: "button", + rect: { x: 20, y: 20, w: 120, h: 40 }, + domParentId: 100, + domAncestorIds: [100], + attrs: {}, + }), + ); }); it("uses the nearest backend AX ancestor as parentId", () => { @@ -916,8 +912,13 @@ describe("buildVomScene", () => { ], }; - const node = buildVomScene(axNodes, captured).nodes.find((n) => n.id === 20); - expect(node).toEqual(expect.objectContaining({ id: 20, role: "button", name: "close" })); + const scene = buildVomScene(axNodes, captured); + expect(scene.nodes.find((n) => n.id === 20)).toEqual( + expect.objectContaining({ id: 20, role: "generic", cursor: "pointer" }), + ); + const rendered = renderVom(scene); + expect(rendered.text).toContain('@e1 button "close"'); + expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]); }); it("does not promote a clickable container that wraps a real interactive control", () => { @@ -1056,10 +1057,12 @@ describe("buildVomScene", () => { const scene = buildVomScene(axNodes, captured); expect(scene.nodes.find((n) => n.id === 20)).toEqual( - expect.objectContaining({ id: 20, role: "button", name: "收藏" }), + expect.objectContaining({ id: 20, role: "generic", attrs: { "aria-label": "收藏" } }), ); - // Inner clickable must NOT also become a button. - expect(scene.nodes.find((n) => n.id === 30)?.role).not.toBe("button"); + expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic"); + const rendered = renderVom(scene); + expect(rendered.text).toContain('@e1 button "收藏"'); + expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]); }); it("marks captured dialog elements as modal without AX role or aria-modal", () => { diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index c53872f..a50c25a 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -342,43 +342,122 @@ function capturedIframeNameFor(node: CapturedNode): string | undefined { return id ? id : undefined; } -const FORM_CONTROL_TAGS = new Set(["input", "textarea", "select"]); - -/** AX roles we treat as non-semantic wrappers eligible for clickable promotion. */ -const PROMOTABLE_ROLES = new Set(["", "generic", "none", "presentation"]); - -/** - * Interactive AX roles. Mirrors `packages/vom`'s INTERACTIVE_ROLES — kept - * local so the package stays a pure projection (design §2). Used to tell an - * atomic custom control (icon button / toggle) apart from a clickable - * *container* (card/row) that wraps real controls we must not collapse. - */ -const INTERACTIVE_ROLES_LC = new Set([ - "button", - "checkbox", - "combobox", - "listbox", - "link", - "menuitem", - "menuitemcheckbox", - "menuitemradio", - "option", - "radio", - "scrollbar", - "searchbox", - "slider", - "spinbutton", - "switch", - "tab", - "textbox", - "treeitem", -]); - function cleanAttr(value: string | undefined): string | undefined { const trimmed = value?.replace(/\s+/g, " ").trim(); return trimmed ? trimmed : undefined; } +const FORM_CONTROL_TAGS = new Set(["input", "textarea", "select"]); + +const NATIVE_CONTROL_TAGS = new Set(["button", "input", "select", "textarea"]); + +function buildCapturedChildren(capturedNodes: CapturedNode[]): Map { + const children = new Map(); + for (const node of capturedNodes) { + if (node.parentBackendNodeId === null) continue; + const siblings = children.get(node.parentBackendNodeId); + if (siblings) siblings.push(node); + else children.set(node.parentBackendNodeId, [node]); + } + return children; +} + +function capturedDomAncestorIds( + node: CapturedNode, + capturedByBackendId: Map, +): number[] { + const ancestors: number[] = []; + let parentId = node.parentBackendNodeId; + let guard = 0; + while (parentId !== null && guard < capturedByBackendId.size) { + ancestors.push(parentId); + parentId = capturedByBackendId.get(parentId)?.parentBackendNodeId ?? null; + guard += 1; + } + return ancestors; +} + +function capturedHasNativeDescendant( + node: CapturedNode, + childrenByParentId: Map, +): boolean { + const stack = [...(childrenByParentId.get(node.backendNodeId) ?? [])]; + while (stack.length > 0) { + const child = stack.pop() as CapturedNode; + if (NATIVE_CONTROL_TAGS.has(normalizeTag(child.tag))) return true; + stack.push(...(childrenByParentId.get(child.backendNodeId) ?? [])); + } + return false; +} + +function capturedInsideNative( + node: CapturedNode, + capturedByBackendId: Map, +): boolean { + let parentId = node.parentBackendNodeId; + let guard = 0; + while (parentId !== null && guard < capturedByBackendId.size) { + const parent = capturedByBackendId.get(parentId); + if (!parent) break; + if (NATIVE_CONTROL_TAGS.has(normalizeTag(parent.tag))) return true; + parentId = parent.parentBackendNodeId; + guard += 1; + } + return false; +} + +function nearbyTextFor( + node: CapturedNode, + childrenByParentId: Map, +): string | undefined { + if (node.parentBackendNodeId === null) return undefined; + const siblings = childrenByParentId.get(node.parentBackendNodeId) ?? []; + const index = siblings.findIndex((sibling) => sibling.backendNodeId === node.backendNodeId); + if (index < 0) return undefined; + + const labels: string[] = []; + for (const sibling of siblings.slice(Math.max(0, index - 3), index)) { + const text = cleanAttr(sibling.textContent); + if (text) labels.push(text); + } + for (const sibling of siblings.slice(index + 1, index + 4)) { + const text = cleanAttr(sibling.textContent); + if (text) labels.push(text); + } + return labels.length > 0 ? labels.join(" ") : undefined; +} + +interface VomNodeDomSignals { + capturedByBackendId: Map; + childrenByParentId: Map; +} + +function applyCapturedSignals( + node: VomNode, + capturedNode: CapturedNode | undefined, + signals: VomNodeDomSignals, +): VomNode { + if (!capturedNode) return node; + const attrs = capturedNode.attrs; + const text = cleanAttr(capturedNode.textContent); + const nearbyText = nearbyTextFor(capturedNode, signals.childrenByParentId); + return { + ...node, + domParentId: capturedNode.parentBackendNodeId, + domAncestorIds: capturedDomAncestorIds(capturedNode, signals.capturedByBackendId), + cursor: capturedNode.cursor, + attrs, + ...(text ? { text } : {}), + ...(nearbyText ? { nearbyText } : {}), + disabled: + Object.prototype.hasOwnProperty.call(attrs, "disabled") || + (attrs["aria-disabled"] ?? "").toLowerCase() === "true", + inert: Object.prototype.hasOwnProperty.call(attrs, "inert"), + hasNativeDescendant: capturedHasNativeDescendant(capturedNode, signals.childrenByParentId), + insideNative: capturedInsideNative(capturedNode, signals.capturedByBackendId), + }; +} + /** * Name for a native form control. VOM models the *perceived* viewport * (spec §1): an empty field displays its placeholder, so when the field has @@ -400,174 +479,30 @@ function formControlName( return axName ?? ariaLabel ?? placeholder ?? title; } -/** Strip a sprite/file reference like `#close` or `icon-close.svg` to `close`. */ -function iconKeyword(href: string | undefined): string | undefined { - if (!href) return undefined; - let s = href.trim(); - const hash = s.lastIndexOf("#"); - if (hash >= 0) s = s.slice(hash + 1); - s = s.split("/").pop() ?? s; - s = s.replace(/\.(svg|png|webp|gif|jpe?g)$/i, ""); - s = s.replace(/^(icons?[-_]?|ic[-_]|svg[-_])/i, ""); - s = s.replace(/[-_]+/g, " ").trim(); - return s ? s : undefined; -} - -/** First icon-derived label found in a clickable element's captured subtree. */ -function iconHint( - root: CapturedNode, - capChildren: Map, -): string | undefined { - const queue: Array<{ node: CapturedNode; depth: number }> = ( - capChildren.get(root.backendNodeId) ?? [] - ).map((node) => ({ node, depth: 1 })); - while (queue.length > 0) { - const { node, depth } = queue.shift() as { node: CapturedNode; depth: number }; - const labelled = - cleanAttr(node.attrs["aria-label"]) ?? - cleanAttr(node.attrs.title) ?? - cleanAttr(node.attrs.alt); - if (labelled) return labelled; - if (node.tag === "use") { - const kw = iconKeyword(node.attrs["xlink:href"] ?? node.attrs.href); - if (kw) return kw; - } - if (depth < 4) { - for (const child of capChildren.get(node.backendNodeId) ?? []) { - queue.push({ node: child, depth: depth + 1 }); - } - } - } - return undefined; -} - -/** Best-effort name for a promoted clickable: own labels → text → icon. */ -function clickableName( - captured: CapturedNode, - capChildren: Map, -): string | undefined { - return ( - cleanAttr(captured.attrs["aria-label"]) ?? - cleanAttr(captured.attrs.title) ?? - cleanAttr(captured.textContent) ?? - iconHint(captured, capChildren) - ); -} - -function rectCoverage( - rect: { x: number; y: number; w: number; h: number } | null, - vw: number, - vh: number, -): number { - if (!rect || vw <= 0 || vh <= 0) return 0; - const ix = Math.max(0, Math.min(rect.x + rect.w, vw) - Math.max(rect.x, 0)); - const iy = Math.max(0, Math.min(rect.y + rect.h, vh) - Math.max(rect.y, 0)); - const overlap = ix * iy; - return overlap <= 0 ? 0 : Math.min(1, overlap / (vw * vh)); -} - -/** - * Promote custom clickable controls the AX tree drops as `generic`/unnamed - * (icon buttons, CSS checkboxes, "send code" spans …) to a referenceable - * `button` so the snapshot can act on them. Guards keep noise out: must be - * `cursor: pointer`, have a box, not be a viewport-scale overlay, and not - * wrap any real interactive descendant (so clickable cards or rows stay - * containers and their inner controls keep their own refs). - */ -function promoteClickableControls( - nodes: VomNode[], - capturedById: Map, - capturedNodes: CapturedNode[], - vw: number, - vh: number, -): void { - // Snapshot original roles + parent/child links before any mutation so the - // guards below are immune to roles we flip to "button" mid-pass. - const originalRoleLc = new Map( - nodes.map((node) => [node.id, (node.role ?? "").toLowerCase()]), - ); - const parentOf = new Map(nodes.map((node) => [node.id, node.parentId])); - const semChildren = new Map(); - for (const node of nodes) { - if (node.parentId === null) continue; - const siblings = semChildren.get(node.parentId); - if (siblings) siblings.push(node); - else semChildren.set(node.parentId, [node]); - } - - const capChildren = new Map(); - for (const cap of capturedNodes) { - if (cap.parentBackendNodeId === null) continue; - const siblings = capChildren.get(cap.parentBackendNodeId); - if (siblings) siblings.push(cap); - else capChildren.set(cap.parentBackendNodeId, [cap]); - } - - // A custom clickable: a non-semantic, pointer-cursor box small enough to be - // a control rather than a viewport-scale overlay. - const candidates = new Set(); - for (const node of nodes) { - if (!PROMOTABLE_ROLES.has(originalRoleLc.get(node.id) ?? "")) continue; - const cap = capturedById.get(node.id); - if (!cap || cap.cursor !== "pointer" || !node.rect) continue; - if (rectCoverage(node.rect, vw, vh) > 0.5) continue; - candidates.add(node.id); - } - - // wraps a *real* interactive control → it's a clickable container, leave it - // (and its inner refs) alone. - const wrapsInteractive = (id: number): boolean => { - const stack = [...(semChildren.get(id) ?? [])]; - while (stack.length > 0) { - const node = stack.pop() as VomNode; - if (INTERACTIVE_ROLES_LC.has(originalRoleLc.get(node.id) ?? "")) return true; - const kids = semChildren.get(node.id); - if (kids) stack.push(...kids); - } - return false; - }; - - // only the outermost clickable in a nested chain becomes the control, so an - // icon inside a custom button doesn't render as a button-in-a-button. - const hasCandidateAncestor = (id: number): boolean => { - let parent = parentOf.get(id) ?? null; - while (parent !== null) { - if (candidates.has(parent)) return true; - parent = parentOf.get(parent) ?? null; - } - return false; - }; - - for (const node of nodes) { - if (!candidates.has(node.id)) continue; - if (wrapsInteractive(node.id) || hasCandidateAncestor(node.id)) continue; - const cap = capturedById.get(node.id); - if (!cap) continue; - - node.role = "button"; - if (!cleanAttr(node.name)) { - const derived = clickableName(cap, capChildren); - if (derived) node.name = derived; - } - } -} - -function vomNodeFromCaptured(capturedNode: CapturedNode, parentId: number | null): VomNode { +function vomNodeFromCaptured( + capturedNode: CapturedNode, + parentId: number | null, + signals: VomNodeDomSignals, +): VomNode { const sensitive = isSensitive(capturedNode); const value = sensitive ? (capturedNode.attrs.value ?? "") : undefined; const tag = normalizeTag(capturedNode.tag); - const node: VomNode = { - id: capturedNode.backendNodeId, - parentId, - tag, - rect: capturedNode.rect, - paintOrder: capturedNode.paintOrder, - position: capturedNode.position || "static", - pointerEvents: capturedNode.pointerEvents || "auto", - modal: isModalSignal(undefined, capturedNode), - sensitive, - ...(value !== undefined ? { value } : {}), - }; + const node = applyCapturedSignals( + { + id: capturedNode.backendNodeId, + parentId, + tag, + rect: capturedNode.rect, + paintOrder: capturedNode.paintOrder, + position: capturedNode.position || "static", + pointerEvents: capturedNode.pointerEvents || "auto", + modal: isModalSignal(undefined, capturedNode), + sensitive, + ...(value !== undefined ? { value } : {}), + }, + capturedNode, + signals, + ); if (tag === "iframe") { node.role = "Iframe"; const name = capturedIframeNameFor(capturedNode); @@ -576,6 +511,56 @@ function vomNodeFromCaptured(capturedNode: CapturedNode, parentId: number | null return node; } +function axVomNode( + axNode: CdpAxNode, + capturedNode: CapturedNode | undefined, + axById: Map, + signals: VomNodeDomSignals, +): VomNode { + const role = axString(axNode.role); + const value = axValue(axNode.value); + const name = FORM_CONTROL_TAGS.has(normalizeTag(capturedNode?.tag)) + ? formControlName(capturedNode, axString(axNode.name), value) + : axString(axNode.name); + const node: VomNode = { + id: axNode.backendDOMNodeId as number, + parentId: nearestBackendParent(axNode, axById), + tag: normalizeTag(capturedNode?.tag), + rect: capturedNode?.rect ?? null, + paintOrder: capturedNode?.paintOrder ?? 0, + position: capturedNode?.position || "static", + pointerEvents: capturedNode?.pointerEvents || "auto", + modal: isModalSignal(axNode, capturedNode), + sensitive: isSensitive(capturedNode), + }; + if (role) node.role = role; + if (name) node.name = name; + if (value !== undefined) node.value = value; + return applyCapturedSignals(node, capturedNode, signals); +} + +function iframeSignals(iframeNodes: CapturedNode[]): VomNodeDomSignals { + const capturedByBackendId = new Map(); + for (const node of iframeNodes) { + capturedByBackendId.set(node.backendNodeId, node); + } + return { + capturedByBackendId, + childrenByParentId: buildCapturedChildren(iframeNodes), + }; +} + +function capturedOnlySignals(capturedNodes: CapturedNode[]): VomNodeDomSignals { + const capturedByBackendId = new Map(); + for (const node of capturedNodes) { + capturedByBackendId.set(node.backendNodeId, node); + } + return { + capturedByBackendId, + childrenByParentId: buildCapturedChildren(capturedNodes), + }; +} + function axNodeInOverlaySubtree( axNode: CdpAxNode, axById: Map, @@ -598,10 +583,8 @@ function axNodeInOverlaySubtree( } export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel): VomScene { - const capturedByBackendId = new Map(); - for (const node of captured.nodes) { - capturedByBackendId.set(node.backendNodeId, node); - } + const signals = capturedOnlySignals(captured.nodes); + const { capturedByBackendId } = signals; const axById = new Map(); for (const node of axNodes) { @@ -618,24 +601,7 @@ export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel) const capturedNode = capturedByBackendId.get(axNode.backendDOMNodeId); const role = axString(axNode.role); - const value = axValue(axNode.value); - const name = FORM_CONTROL_TAGS.has(normalizeTag(capturedNode?.tag)) - ? formControlName(capturedNode, axString(axNode.name), value) - : axString(axNode.name); - const vomNode: VomNode = { - id: axNode.backendDOMNodeId, - parentId: nearestBackendParent(axNode, axById), - tag: normalizeTag(capturedNode?.tag), - rect: capturedNode?.rect ?? null, - paintOrder: capturedNode?.paintOrder ?? 0, - position: capturedNode?.position || "static", - pointerEvents: capturedNode?.pointerEvents || "auto", - modal: isModalSignal(axNode, capturedNode), - sensitive: isSensitive(capturedNode), - }; - if (role) vomNode.role = role; - if (name) vomNode.name = name; - if (value !== undefined) vomNode.value = value; + const vomNode = axVomNode(axNode, capturedNode, axById, signals); // For link nodes, attach external hostname so the renderer can annotate it. // Relative hrefs and same-origin hrefs are omitted to avoid noise. @@ -658,10 +624,11 @@ export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel) if (seenBackendIds.has(capturedNode.backendNodeId)) continue; if (excludedBackendNodeIds.has(capturedNode.backendNodeId)) continue; seenBackendIds.add(capturedNode.backendNodeId); - nodes.push(vomNodeFromCaptured(capturedNode, capturedNode.parentBackendNodeId)); + nodes.push(vomNodeFromCaptured(capturedNode, capturedNode.parentBackendNodeId, signals)); } for (const [iframeBackendId, iframeNodes] of captured.iframeNodes) { + const signals = iframeSignals(iframeNodes); for (const iframeNode of iframeNodes) { if ( seenBackendIds.has(iframeNode.backendNodeId) || @@ -676,7 +643,7 @@ export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel) seenBackendIds.add(iframeNode.backendNodeId); const vomNode: VomNode = { - ...vomNodeFromCaptured(iframeNode, iframeBackendId), + ...vomNodeFromCaptured(iframeNode, iframeBackendId, signals), role, }; if (name) vomNode.name = name; @@ -684,14 +651,6 @@ export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel) } } - promoteClickableControls( - nodes, - capturedByBackendId, - captured.nodes, - captured.viewport.width, - captured.viewport.height, - ); - return { viewport: captured.viewport, nodes }; } diff --git a/packages/vom/src/__tests__/render.test.ts b/packages/vom/src/__tests__/render.test.ts index afe56c1..4042628 100644 --- a/packages/vom/src/__tests__/render.test.ts +++ b/packages/vom/src/__tests__/render.test.ts @@ -190,6 +190,94 @@ describe("renderVom single-layer page", () => { const renderedRefs = new Set(Array.from(out.text.matchAll(/@(e\d+)/g), (m) => m[1])); expect(out.refs.every((r) => renderedRefs.has(r.ref))).toBe(true); }); + + it("recovers custom clickable controls from DOM-only signals", () => { + const out = renderVom( + scene([ + node({ id: 1, role: "RootWebArea", name: "Doc" }), + node({ + id: 2, + parentId: 1, + role: "generic", + text: "Open settings", + cursor: "pointer", + attrs: { onclick: "handleOpen()" }, + rect: { x: 20, y: 20, w: 160, h: 40 }, + }), + ]), + ); + + expect(out.text).toContain('@e1 button "Open settings"'); + expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); + }); + + it("keeps the deepest custom control when wrapper and child both look clickable", () => { + const out = renderVom( + scene([ + node({ id: 1, role: "RootWebArea", name: "Doc" }), + node({ + id: 2, + parentId: 1, + role: "generic", + text: "Card", + cursor: "pointer", + attrs: { onclick: "handleCard()" }, + rect: { x: 20, y: 20, w: 280, h: 120 }, + }), + node({ + id: 3, + parentId: 2, + role: "generic", + text: "Details", + cursor: "pointer", + attrs: { onclick: "handleDetails()" }, + rect: { x: 40, y: 70, w: 90, h: 32 }, + }), + ]), + ); + + expect(out.text).not.toContain('@e1 button "Card"'); + expect(out.text).toContain('@e1 button "Details"'); + expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]); + }); + + it("does not recover clickable wrappers around native controls", () => { + const out = renderVom( + scene([ + node({ id: 1, role: "RootWebArea", name: "Doc" }), + node({ + id: 2, + parentId: 1, + role: "generic", + text: "Checkout", + cursor: "pointer", + attrs: { onclick: "handleCheckout()" }, + hasNativeDescendant: true, + rect: { x: 20, y: 20, w: 280, h: 120 }, + }), + node({ id: 3, parentId: 2, role: "button", name: "Pay", tag: "button" }), + ]), + ); + + expect(out.text).not.toContain('@e1 button "Checkout"'); + expect(out.text).toContain('@e1 button "Pay"'); + expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]); + }); + + it("adds context to duplicate weak action labels", () => { + const out = renderVom( + scene([ + node({ id: 1, role: "RootWebArea", name: "Admin" }), + node({ id: 2, parentId: 1, role: "section", name: "Project Alpha" }), + node({ id: 3, parentId: 2, role: "button", name: "View", tag: "button" }), + node({ id: 4, parentId: 1, role: "section", name: "Project Beta" }), + node({ id: 5, parentId: 4, role: "button", name: "View", tag: "button" }), + ]), + ); + + expect(out.text).toContain('@e1 button "View" [ctx: Project Alpha]'); + expect(out.text).toContain('@e2 button "View" [ctx: Project Beta]'); + }); }); describe("renderVom double-layer page", () => { diff --git a/packages/vom/src/render.ts b/packages/vom/src/render.ts index 376204d..1701aec 100644 --- a/packages/vom/src/render.ts +++ b/packages/vom/src/render.ts @@ -69,6 +69,41 @@ const SENSITIVE_MASK = "•••"; const DEFAULT_MAX_DEPTH = Number.POSITIVE_INFINITY; const DEFAULT_MAX_TOKENS = Number.POSITIVE_INFINITY; const MAX_VALUE_LENGTH = 200; +const MAX_CUSTOM_RECOVERY_AREA = 180_000; +const MAX_RECOVERABLE_SINGLE_INTENT_AREA = 360_000; +const MAX_HANDLE_CONTEXT_ITEMS = 3; + +const RECOVERY_HANDLER_ATTRS = ["onclick", "onmousedown", "onkeydown", "onkeyup", "onkeypress"]; +const NATIVE_TAGS = new Set(["button", "input", "select", "textarea"]); +const GRAPHIC_ONLY_TAGS = new Set(["svg", "use", "path", "g", "symbol"]); +const EXPLICIT_RECOVERY_ROLES = new Set([ + "button", + "menuitem", + "switch", + "tab", + "checkbox", + "radio", +]); +const WEAK_ACTION_LABELS = new Set([ + "add", + "apply", + "cancel", + "delete", + "edit", + "go", + "more", + "open", + "remove", + "reset", + "save", + "select", + "submit", + "suspend", + "view", + "close", + "confirm", + "continue", +]); function estimateTokens(text: string): number { return Math.ceil(text.length / 4); @@ -83,6 +118,22 @@ function normalizedRole(node: VomNode): string { return node.role?.toLowerCase() ?? ""; } +function normalizeContextKey(s: string): string { + return s.split(/\s+/).join(" ").toLowerCase(); +} + +function isProbableNavigationTarget(value: string): boolean { + const trimmed = value.trim(); + return trimmed.startsWith("http://") || trimmed.startsWith("https://"); +} + +function isUrlBearingStructuralAction(node: VomNode): boolean { + const role = normalizedRole(node); + if (!["row", "cell", "gridcell", "listitem", "rowheader"].includes(role)) return false; + const name = cleaned(node.name); + return name !== undefined && isProbableNavigationTarget(name); +} + function shouldRender(node: VomNode): boolean { if (!node.role) return false; @@ -95,7 +146,171 @@ function shouldRender(node: VomNode): boolean { } function shouldReference(node: VomNode): boolean { - return INTERACTIVE_ROLES.has(normalizedRole(node)); + return INTERACTIVE_ROLES.has(normalizedRole(node)) || isUrlBearingStructuralAction(node); +} + +function hasRecoveryHandler(node: VomNode): boolean { + const attrs = node.attrs ?? {}; + return RECOVERY_HANDLER_ATTRS.some((key) => Object.prototype.hasOwnProperty.call(attrs, key)); +} + +function isFocusable(node: VomNode): boolean { + const tabindex = node.attrs?.tabindex; + if (tabindex === undefined) return false; + const value = Number.parseInt(tabindex, 10); + return Number.isFinite(value) && value >= 0; +} + +function isContentEditable(node: VomNode): boolean { + const contentEditable = node.attrs?.contenteditable?.toLowerCase(); + return contentEditable === "true" || contentEditable === "plaintext-only"; +} + +function iconKeyword(value: string | undefined): string | undefined { + const cleanedValue = cleaned(value); + if (!cleanedValue) return undefined; + const last = cleanedValue + .split(/[#/.\s:_-]+/) + .filter(Boolean) + .pop(); + return last && /^[a-z][a-z0-9_-]{1,32}$/i.test(last) ? last.toLowerCase() : undefined; +} + +function iconHintFromSubtree( + node: VomNode, + children: Map, + depth = 0, +): string | undefined { + if (depth > 3) return undefined; + for (const child of children.get(node.id) ?? []) { + const attrs = child.attrs ?? {}; + const hint = + iconKeyword(attrs["aria-label"]) ?? + iconKeyword(attrs.title) ?? + iconKeyword(attrs.href) ?? + iconKeyword(attrs["xlink:href"]) ?? + iconKeyword(attrs.class) ?? + iconKeyword(child.text); + if (hint) return hint; + const nested = iconHintFromSubtree(child, children, depth + 1); + if (nested) return nested; + } + return undefined; +} + +function recoveryCandidateName( + node: VomNode, + children?: Map, +): string | undefined { + return ( + cleaned(node.attrs?.["aria-label"]) ?? + cleaned(node.attrs?.title) ?? + cleaned(node.name) ?? + cleaned(node.text) ?? + cleaned(node.nearbyText) ?? + (children ? iconHintFromSubtree(node, children) : undefined) + ); +} + +function recoveredRole( + node: VomNode, + children?: Map, +): string | undefined { + const explicitRole = node.attrs?.role?.toLowerCase() ?? ""; + const hasHandler = hasRecoveryHandler(node); + const focusable = isFocusable(node); + const hasName = recoveryCandidateName(node, children) !== undefined; + + if (isContentEditable(node)) return "textbox"; + if (explicitRole === "combobox" && (hasHandler || focusable)) return "combobox"; + if ( + EXPLICIT_RECOVERY_ROLES.has(explicitRole) && + hasName && + (hasHandler || focusable || node.cursor === "pointer") + ) { + return explicitRole === "switch" || + explicitRole === "checkbox" || + explicitRole === "radio" || + explicitRole === "tab" || + explicitRole === "menuitem" + ? explicitRole + : "button"; + } + if (hasHandler && hasName) return "button"; + if (node.cursor === "pointer" && hasName) return "button"; + return undefined; +} + +function rectArea(node: VomNode): number { + if (!node.rect) return 0; + return Math.max(0, node.rect.w) * Math.max(0, node.rect.h); +} + +function passesRecoveryGuards(node: VomNode, children: Map): boolean { + const area = rectArea(node); + const tag = node.tag.toLowerCase(); + + if (node.pointerEvents === "none") return false; + if (node.disabled || node.inert || node.insideNative) return false; + if (node.hasNativeDescendant && !isContentEditable(node)) return false; + + const singleIntentLargeCandidate = + area <= MAX_RECOVERABLE_SINGLE_INTENT_AREA && + recoveryCandidateName(node, children) !== undefined && + hasRecoveryHandler(node); + if (area > MAX_CUSTOM_RECOVERY_AREA && !isContentEditable(node) && !singleIntentLargeCandidate) { + return false; + } + if (NATIVE_TAGS.has(tag) || tag === "a") return false; + if (GRAPHIC_ONLY_TAGS.has(tag)) return false; + return recoveredRole(node, children) !== undefined; +} + +function buildParentMap(nodes: VomNode[]): Map { + const ids = new Set(nodes.map((node) => node.id)); + const parentMap = new Map(); + for (const node of nodes) { + parentMap.set(node.id, node.parentId !== null && ids.has(node.parentId) ? node.parentId : null); + } + return parentMap; +} + +function applyRecovery(nodes: VomNode[]): VomNode[] { + const parentMap = buildParentMap(nodes); + const children = buildChildren(nodes); + const recoveredIds = new Set(); + const recoveredRoleById = new Map(); + const recoveredNameById = new Map(); + + for (const node of nodes) { + if (shouldReference(node)) continue; + const role = recoveredRole(node, children); + if (!role || !passesRecoveryGuards(node, children)) continue; + recoveredIds.add(node.id); + recoveredRoleById.set(node.id, role); + const name = recoveryCandidateName(node, children); + if (name) recoveredNameById.set(node.id, name); + } + + for (const node of nodes) { + if (!recoveredIds.has(node.id)) continue; + let current = parentMap.get(node.id) ?? null; + let guard = 0; + while (current !== null && guard <= parentMap.size) { + recoveredIds.delete(current); + current = parentMap.get(current) ?? null; + guard += 1; + } + } + + return nodes.map((node) => { + if (!recoveredIds.has(node.id)) return node; + return { + ...node, + role: recoveredRoleById.get(node.id), + name: recoveredNameById.get(node.id) ?? node.name, + }; + }); } function buildChildren(nodes: VomNode[]): Map { @@ -112,11 +327,222 @@ function buildChildren(nodes: VomNode[]): Map { return children; } -function renderNodeLine(node: VomNode, depth: number, ref: string | undefined): string { +function duplicateReferenceNames(nodes: VomNode[]): Set { + const counts = new Map(); + for (const node of nodes) { + if (!shouldReference(node)) continue; + const name = cleaned(node.name); + if (!name) continue; + const key = normalizeContextKey(name); + counts.set(key, (counts.get(key) ?? 0) + 1); + } + return new Set([...counts].filter(([, count]) => count > 1).map(([name]) => name)); +} + +function needsHandleContext(nodeName: string, duplicateRefNames: Set): boolean { + const key = normalizeContextKey(nodeName); + if (!key) return false; + return duplicateRefNames.has(key) || WEAK_ACTION_LABELS.has(key); +} + +function contextLabel(node: VomNode, targetName: string): string | undefined { + if ( + !["heading", "article", "section", "group", "region", "row", "form"].includes( + normalizedRole(node), + ) + ) { + return undefined; + } + return weakContextText( + cleaned(node.name) ?? cleaned(node.text) ?? cleaned(node.nearbyText), + targetName, + ); +} + +function weakLabelContext(node: VomNode, targetName: string): string | undefined { + if (shouldReference(node)) return undefined; + return weakContextText( + cleaned(node.name) ?? cleaned(node.text) ?? cleaned(node.nearbyText), + targetName, + ); +} + +function weakContextText(label: string | undefined, targetName: string): string | undefined { + const clean = cleaned(label); + if (!clean) return undefined; + const key = normalizeContextKey(clean); + if ( + !key || + key === normalizeContextKey(targetName) || + key.startsWith("ctx_") || + key === "section" || + WEAK_ACTION_LABELS.has(key) + ) { + return undefined; + } + return clean; +} + +type ContextSource = "owned" | "same" | "dom"; + +interface ContextCandidate { + text: string; + source: ContextSource; + order: number; +} + +function contextSourceRank(source: ContextSource): number { + return source === "owned" ? 0 : source === "same" ? 1 : 2; +} + +function contextCategory(key: string): string { + return key.match(/^[a-z\s]+/)?.[0].trim() ?? key; +} + +function contextRank(key: string): number { + if (key.startsWith("level ")) return 0; + if (key.startsWith("group ")) return 1; + return 2; +} + +function dedupeContexts(contexts: ContextCandidate[]): string[] { + const seen = new Set(); + const seenCategories = new Set(); + const sorted = [...contexts].sort( + (a, b) => + contextSourceRank(a.source) - contextSourceRank(b.source) || + contextRank(normalizeContextKey(a.text)) - contextRank(normalizeContextKey(b.text)) || + a.order - b.order, + ); + const unique: string[] = []; + for (const ctx of sorted) { + const clean = cleaned(ctx.text); + if (!clean) continue; + const key = normalizeContextKey(clean); + const category = `${ctx.source}:${contextCategory(key)}`; + if (!key || seen.has(key) || seenCategories.has(category)) continue; + seen.add(key); + seenCategories.add(category); + unique.push(clean); + if (unique.length >= MAX_HANDLE_CONTEXT_ITEMS) break; + } + return unique; +} + +function isContextBoundary(node: VomNode): boolean { + return ["region", "form", "section", "article", "row", "listitem", "group"].includes( + normalizedRole(node), + ); +} + +function collectWeakLabelsFromSubtree( + node: VomNode, + nodeName: string, + state: RenderState, + labels: string[], +): void { + if (shouldReference(node)) return; + const label = weakLabelContext(node, nodeName); + if (label) { + labels.push(label); + return; + } + for (const child of state.children.get(node.id) ?? []) { + collectWeakLabelsFromSubtree(child, nodeName, state, labels); + if (labels.length >= MAX_HANDLE_CONTEXT_ITEMS) return; + } +} + +function collectOwnedContext( + node: VomNode, + nodeName: string, + state: RenderState, +): ContextCandidate[] { + const contexts: ContextCandidate[] = []; + let current = state.parentMap.get(node.id) ?? null; + let guard = 0; + while (current !== null && guard <= state.parentMap.size) { + const parent = state.nodesById.get(current); + if (!parent) break; + const label = contextLabel(parent, nodeName); + if (label) contexts.push({ text: label, source: "owned", order: contexts.length }); + current = state.parentMap.get(current) ?? null; + guard += 1; + } + return contexts; +} + +function collectSameContainerContext( + node: VomNode, + nodeName: string, + state: RenderState, +): ContextCandidate[] { + const labels: string[] = []; + let childId = node.id; + let parentId = state.parentMap.get(node.id) ?? null; + let guard = 0; + while (parentId !== null && guard <= state.parentMap.size) { + const siblings = state.children.get(parentId) ?? []; + for (const sibling of siblings) { + if (sibling.id === childId) break; + collectWeakLabelsFromSubtree(sibling, nodeName, state, labels); + while (labels.length > MAX_HANDLE_CONTEXT_ITEMS) labels.shift(); + } + + const parent = state.nodesById.get(parentId); + if (!parent || isContextBoundary(parent)) break; + childId = parentId; + parentId = state.parentMap.get(parentId) ?? null; + guard += 1; + } + return labels.map((text, index) => ({ text, source: "same", order: index })); +} + +function collectDomContext( + node: VomNode, + nodeName: string, + state: RenderState, +): ContextCandidate[] { + const ancestorIds = node.domAncestorIds ?? []; + if (ancestorIds.length === 0) return []; + const ancestors = new Set(ancestorIds); + const contexts: ContextCandidate[] = []; + + for (const candidate of state.nodesById.values()) { + if (candidate.id === node.id) continue; + if (isContextBoundary(candidate) && !ancestorIds.includes(candidate.id)) continue; + const parentId = candidate.domParentId; + if (parentId === undefined || parentId === null || !ancestors.has(parentId)) continue; + const label = contextLabel(candidate, nodeName); + if (!label) continue; + const rank = ancestorIds.indexOf(parentId); + contexts.push({ text: label, source: "dom", order: rank < 0 ? contexts.length : rank }); + } + + return contexts; +} + +function handleContext(node: VomNode, state: RenderState): string[] { + const name = cleaned(node.name); + if (!name || !needsHandleContext(name, state.duplicateRefNames)) return []; + return dedupeContexts([ + ...collectOwnedContext(node, name, state), + ...collectSameContainerContext(node, name, state), + ...collectDomContext(node, name, state), + ]); +} + +function renderNodeLine( + node: VomNode, + depth: number, + ref: string | undefined, + context: string[] = [], +): string { let line = `${" ".repeat(depth)}${ref ? `@${ref} ` : ""}${node.role}`; const name = cleaned(node.name); if (name) line += ` ${JSON.stringify(name)}`; + if (context.length > 0) line += ` [ctx: ${context.join(" > ")}]`; // For link nodes with an external href, annotate so the agent can // distinguish external navigation from same-origin links. @@ -139,6 +565,10 @@ interface RenderState { maxTokens: number; truncated: boolean; stopped: boolean; + children: Map; + parentMap: Map; + nodesById: Map; + duplicateRefNames: Set; } function renderTree( @@ -164,7 +594,8 @@ function renderTree( } const ref = shouldReference(node) ? `e${state.nextRef}` : undefined; - const line = renderNodeLine(node, depth, ref); + const context = ref ? handleContext(node, state) : []; + const line = renderNodeLine(node, depth, ref, context); const nextTokens = state.tokens + estimateTokens(line); if (nextTokens > state.maxTokens) { state.truncated = true; @@ -184,6 +615,7 @@ function renderTree( } function renderNodes(nodes: VomNode[], options: VomOptions, initialLines: string[]): RenderState { + const children = buildChildren(nodes); const state: RenderState = { lines: [...initialLines], refs: [], @@ -193,9 +625,13 @@ function renderNodes(nodes: VomNode[], options: VomOptions, initialLines: string maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS, truncated: false, stopped: false, + children, + parentMap: buildParentMap(nodes), + nodesById: new Map(nodes.map((node) => [node.id, node])), + duplicateRefNames: duplicateReferenceNames(nodes), }; - renderTree(buildChildren(nodes), null, 1, state); + renderTree(children, null, 1, state); return state; } @@ -247,10 +683,12 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp } export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult { - const layer = detectBlockingLayer(scene.nodes, scene.viewport); - if (layer) return renderDoubleLayer(scene, layer, options); + const nodes = applyRecovery(scene.nodes); + const renderScene = { ...scene, nodes }; + const layer = detectBlockingLayer(nodes, scene.viewport); + if (layer) return renderDoubleLayer(renderScene, layer, options); - const state = renderNodes(scene.nodes, options, [ + const state = renderNodes(nodes, options, [ "@vom 1", `@view ${scene.viewport.width}x${scene.viewport.height}`, "@layers 1 focus=L1", diff --git a/packages/vom/src/types.ts b/packages/vom/src/types.ts index 2d0371e..30d736e 100644 --- a/packages/vom/src/types.ts +++ b/packages/vom/src/types.ts @@ -26,15 +26,25 @@ export interface VomNode { name?: string; value?: string; href?: string; // hostname of external link target; omitted for same-origin links + text?: string; + nearbyText?: string; tag: string; rect: Rect | null; paintOrder: number; position: string; pointerEvents: string; + cursor?: string; + attrs?: Record; + domParentId?: number | null; + domAncestorIds?: number[]; modal?: boolean; sensitive?: boolean; + disabled?: boolean; + inert?: boolean; + hasNativeDescendant?: boolean; + insideNative?: boolean; } export interface VomScene {