diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index 643ad0b..9d9e003 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -1430,7 +1430,7 @@ describe("buildVomScene", () => { expect(renderVom(scene).text).toContain("[ยง active: Reviews (12)]"); }); - it("maps hover surface probes to matching node labels", () => { + it("maps hover surface probes to matching backend node ids", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", @@ -1451,7 +1451,7 @@ describe("buildVomScene", () => { iframeNodes: new Map(), excludedBackendNodeIds: new Set(), surfaceProbes: [ - { triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] }, + { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] }, ], nodes: [ { @@ -1483,6 +1483,195 @@ describe("buildVomScene", () => { expect(renderVom(scene).text).toContain('@e1 button "Products" [hover first: Shoes | Bags]'); }); + it("maps hover surface probes to rendered descendants in the same trigger subtree", () => { + const axNodes: CdpAxNode[] = [ + { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + backendDOMNodeId: 10, + childIds: ["2"], + }, + { + nodeId: "2", + parentId: "1", + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "image" }, + backendDOMNodeId: 21, + }, + ]; + const scene = buildVomScene(axNodes, { + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + surfaceProbes: [ + { triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] }, + ], + nodes: [ + { + backendNodeId: 10, + parentBackendNodeId: null, + tag: "body", + attrs: {}, + rect: { x: 0, y: 0, w: 1000, h: 800 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 20, + parentBackendNodeId: 10, + tag: "div", + attrs: { class: "tg-avatar" }, + rect: { x: 900, y: 10, w: 30, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 21, + parentBackendNodeId: 20, + tag: "div", + attrs: { class: "tg-avatar__inner" }, + rect: { x: 902, y: 12, w: 26, h: 26 }, + paintOrder: 2, + position: "static", + pointerEvents: "auto", + }, + ], + }); + + expect(renderVom(scene).text).toContain('@e1 button "image" [hover first: My profile]'); + }); + + it("maps hover surface probes to DOM-recovered custom controls", () => { + const axNodes: CdpAxNode[] = [ + { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + backendDOMNodeId: 10, + childIds: ["2"], + }, + { + nodeId: "2", + parentId: "1", + role: { type: "role", value: "img" }, + name: { type: "computedString", value: "image" }, + backendDOMNodeId: 21, + }, + ]; + const scene = buildVomScene(axNodes, { + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + surfaceProbes: [ + { + triggerBackendNodeId: 20, + triggerPoint: { x: 915, y: 25 }, + triggerAction: "hover", + subItems: ["My profile", "Sign out"], + }, + ], + nodes: [ + { + backendNodeId: 10, + parentBackendNodeId: null, + tag: "body", + attrs: {}, + rect: { x: 0, y: 0, w: 1000, h: 800 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + cursor: "auto", + }, + { + backendNodeId: 20, + parentBackendNodeId: 10, + tag: "div", + attrs: { class: "tg-avatar" }, + rect: { x: 900, y: 10, w: 30, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + cursor: "pointer", + }, + { + backendNodeId: 21, + parentBackendNodeId: 20, + tag: "div", + attrs: { class: "tg-avatar__inner" }, + rect: { x: 902, y: 12, w: 26, h: 26 }, + paintOrder: 2, + position: "static", + pointerEvents: "auto", + cursor: "pointer", + }, + ], + }); + + expect(scene.surfaces).toEqual([ + { triggerId: 21, triggerAction: "hover", subItems: ["My profile", "Sign out"] }, + ]); + expect(renderVom(scene).text).toContain( + '@e1 button "image" [hover first: My profile | Sign out]', + ); + }); + + it("does not attach hover probes to distant geometry matches", () => { + const axNodes: CdpAxNode[] = [ + { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + backendDOMNodeId: 10, + childIds: ["2"], + }, + { + nodeId: "2", + parentId: "1", + role: { type: "role", value: "link" }, + name: { type: "computedString", value: "0" }, + backendDOMNodeId: 42, + }, + ]; + const scene = buildVomScene(axNodes, { + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + surfaceProbes: [ + { + triggerBackendNodeId: 999, + triggerPoint: { x: 900, y: 20 }, + triggerAction: "hover", + subItems: ["My profile"], + }, + ], + nodes: [ + { + backendNodeId: 10, + parentBackendNodeId: null, + tag: "body", + attrs: {}, + rect: { x: 0, y: 0, w: 1000, h: 800 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 42, + parentBackendNodeId: 10, + tag: "a", + attrs: {}, + rect: { x: 860, y: 120, w: 40, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + ], + }); + + expect(scene.surfaces).toBeUndefined(); + expect(renderVom(scene).text).not.toContain("[hover first:"); + }); + it("enriches names and active scope signals from AX properties", () => { const axNodes: CdpAxNode[] = [ { @@ -1953,7 +2142,7 @@ describe("handleSnapshot", () => { }); const res = await handleObserve( sm, - { session_id: "aa11" }, + { session_id: "aa11", debug_surfaces: true }, { cdp: { send: send as unknown as ( @@ -1974,6 +2163,7 @@ describe("handleSnapshot", () => { ); if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`); + expect(res.debug).toEqual({ surface_probes: [] }); expect(send).toHaveBeenCalledWith( 4, "Runtime.evaluate", diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index 308db59..c382d96 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -342,6 +342,7 @@ export class ToolDispatcher { cdp: this.cdp, tabsApi: chromeTabsCaptureApi, conditionalSurfaceProbe: !this.hasHoverLatch(), + hoverProbeBypassOverlay: bypassOverlay, } : undefined, ), diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index 11ad305..59d39ff 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -5,7 +5,9 @@ import { type ActiveScopeBlock, + applyVomInteractionRecovery, type CondSurface, + isVomReferenceNode, renderVom, type VomNode, type VomScene, @@ -908,39 +910,26 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A return blocks; } -function labelForSurfaceMatch(node: VomNode): string { - return ( - cleanAttr(node.name) ?? - cleanAttr(node.text) ?? - cleanAttr(node.attrs?.["aria-label"]) ?? - cleanAttr(node.attrs?.title) ?? - "" - ); -} - function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] { const probes = captured.surfaceProbes ?? []; if (probes.length === 0) return []; const surfaces: CondSurface[] = []; - const nodesWithLabels = nodes - .map((node) => ({ node, key: normalizeProbeKey(labelForSurfaceMatch(node)) })) - .filter((entry) => entry.key.length > 0); + const recoveredNodes = applyVomInteractionRecovery(nodes); + const signals = capturedOnlySignals(captured.nodes); const used = new Set(); for (const probe of probes) { - const probeKey = normalizeProbeKey(probe.triggerLabel); - if (!probeKey || probe.subItems.length === 0) continue; - const match = - nodesWithLabels.find((entry) => entry.key === probeKey && !used.has(entry.node.id)) ?? - nodesWithLabels.find( - (entry) => - !used.has(entry.node.id) && - (probeKey.startsWith(entry.key) || entry.key.startsWith(probeKey)), - ); + if (probe.subItems.length === 0 || used.has(probe.triggerBackendNodeId)) continue; + const match = findSurfaceTriggerNode( + probe.triggerBackendNodeId, + recoveredNodes, + signals, + probe.triggerPoint, + ); if (!match) continue; - used.add(match.node.id); + used.add(match.id); surfaces.push({ - triggerId: match.node.id, + triggerId: match.id, triggerAction: probe.triggerAction, subItems: probe.subItems, }); @@ -948,6 +937,77 @@ function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel) return surfaces; } +function findSurfaceTriggerNode( + triggerBackendNodeId: number, + nodes: VomNode[], + signals: VomNodeDomSignals, + triggerPoint?: { x: number; y: number }, +): VomNode | undefined { + const renderedById = new Map(nodes.map((node) => [node.id, node])); + const exact = renderedById.get(triggerBackendNodeId); + if (exact && isSurfaceAttachableNode(exact)) return exact; + + const domDescendant = nodes.find( + (node) => + isSurfaceAttachableNode(node) && + (node.domParentId === triggerBackendNodeId || + node.domAncestorIds?.includes(triggerBackendNodeId) === true), + ); + if (domDescendant) return domDescendant; + + const queue = [...(signals.childrenByParentId.get(triggerBackendNodeId) ?? [])]; + while (queue.length > 0) { + const child = queue.shift() as CapturedNode; + const rendered = renderedById.get(child.backendNodeId); + if (rendered && isSurfaceAttachableNode(rendered)) return rendered; + queue.push(...(signals.childrenByParentId.get(child.backendNodeId) ?? [])); + } + + let current = signals.capturedByBackendId.get(triggerBackendNodeId); + while (current?.parentBackendNodeId !== null && current?.parentBackendNodeId !== undefined) { + const parent = renderedById.get(current.parentBackendNodeId); + if (parent && isSurfaceAttachableNode(parent)) return parent; + current = signals.capturedByBackendId.get(current.parentBackendNodeId); + } + + if (triggerPoint) { + return findSurfaceNodeByPoint(triggerPoint, nodes, signals); + } + + return undefined; +} + +function isSurfaceAttachableNode(node: VomNode): boolean { + return isVomReferenceNode(node); +} + +function findSurfaceNodeByPoint( + point: { x: number; y: number }, + nodes: VomNode[], + signals: VomNodeDomSignals, +): VomNode | undefined { + let best: { node: VomNode; score: number } | undefined; + for (const node of nodes) { + if (!isSurfaceAttachableNode(node)) continue; + const rect = node.rect ?? signals.capturedByBackendId.get(node.id)?.rect; + if (!rect) continue; + const contains = + point.x >= rect.x && + point.x <= rect.x + rect.w && + point.y >= rect.y && + point.y <= rect.y + rect.h; + const centerX = rect.x + rect.w / 2; + const centerY = rect.y + rect.h / 2; + const distance = Math.hypot(point.x - centerX, point.y - centerY); + if (!contains && distance > 40) continue; + const score = contains ? distance : distance + 1_000; + if (!best || score < best.score) { + best = { node, score }; + } + } + return best?.node; +} + function axNodeInOverlaySubtree( axNode: CdpAxNode, axById: Map, @@ -1073,6 +1133,7 @@ export interface SnapshotDeps { query(q: chrome.tabs.QueryInfo): Promise; }; conditionalSurfaceProbe?: boolean; + hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise; } let defaultDeps: SnapshotDeps | null = null; @@ -1218,9 +1279,13 @@ async function captureForVom( cdp: CdpRunner, tabId: number, conditionalSurfaceProbe: boolean, + hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise, ): Promise { try { - return await captureViewModel(cdp, tabId, { conditionalSurfaceProbe }); + return await captureViewModel(cdp, tabId, { + conditionalSurfaceProbe, + hoverProbeBypassOverlay, + }); } catch { return fallbackCapturedViewModel(cdp, tabId); } @@ -1261,7 +1326,12 @@ async function handleVomObservation( const axNodes = result.nodes ?? []; const effectiveConditionalSurfaceProbe = deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe; - const captured = await captureForVom(deps.cdp, target.tabId, effectiveConditionalSurfaceProbe); + const captured = await captureForVom( + deps.cdp, + target.tabId, + effectiveConditionalSurfaceProbe, + deps.hoverProbeBypassOverlay, + ); const scene = buildVomScene(axNodes, captured, { pageUrl: target.url }); const rendered = renderVom(scene, { maxDepth: params.max_depth, @@ -1278,6 +1348,19 @@ async function handleVomObservation( ref_count: rendered.refs.length, tab_id: target.tabId, truncated: rendered.truncated, + ...(toolName === "observe" && (params as ObserveParams).debug_surfaces + ? { + debug: { + surface_probes: (captured.surfaceProbes ?? []).map((probe) => ({ + trigger_backend_node_id: probe.triggerBackendNodeId, + ...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}), + trigger_action: probe.triggerAction, + sub_items: probe.subItems, + ...(probe.confidence ? { confidence: probe.confidence } : {}), + })), + }, + } + : {}), }); } catch (err) { return { diff --git a/apps/extension/src/tools/vom/__tests__/capture.test.ts b/apps/extension/src/tools/vom/__tests__/capture.test.ts index 2fe1f2e..086e216 100644 --- a/apps/extension/src/tools/vom/__tests__/capture.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture.test.ts @@ -51,6 +51,156 @@ function fakeSnapshotReply() { }; } +function hoverTriggerSnapshotReply() { + const S = [ + "html", + "body", + "button", + "class", + "user-avatar", + "position", + "static", + "pointer-events", + "auto", + "cursor", + "pointer", + ]; + const i = (s: string) => S.indexOf(s); + return { + strings: S, + documents: [ + { + nodes: { + parentIndex: [-1, 0, 1], + nodeType: [1, 1, 1], + nodeName: [i("html"), i("body"), i("button")], + backendNodeId: [10, 11, 12], + attributes: [[], [], [i("class"), i("user-avatar")]], + }, + layout: { + nodeIndex: [1, 2], + styles: [ + [i("static"), i("auto"), i("pointer")], + [i("static"), i("auto"), i("pointer")], + ], + bounds: [ + [0, 0, 1000, 800], + [940, 16, 32, 32], + ], + paintOrders: [0, 1], + }, + }, + ], + }; +} + +function twoHoverTriggerSnapshotReply() { + const S = [ + "html", + "body", + "button", + "class", + "user-avatar", + "secondary-trigger", + "position", + "static", + "pointer-events", + "auto", + "cursor", + "pointer", + ]; + const i = (s: string) => S.indexOf(s); + return { + strings: S, + documents: [ + { + nodes: { + parentIndex: [-1, 0, 1, 1], + nodeType: [1, 1, 1, 1], + nodeName: [i("html"), i("body"), i("button"), i("button")], + backendNodeId: [10, 11, 12, 13], + attributes: [ + [], + [], + [i("class"), i("user-avatar")], + [i("class"), i("secondary-trigger")], + ], + }, + layout: { + nodeIndex: [1, 2, 3], + styles: [ + [i("static"), i("auto"), i("pointer")], + [i("static"), i("auto"), i("pointer")], + [i("static"), i("auto"), i("pointer")], + ], + bounds: [ + [0, 0, 1000, 800], + [940, 16, 32, 32], + [100, 120, 32, 32], + ], + paintOrders: [0, 1, 2], + }, + }, + ], + }; +} + +function nestedHoverTriggerSnapshotReply() { + const S = [ + "html", + "body", + "div", + "img", + "class", + "tg-avatar", + "tg-avatar__inner", + "tg-avatar__image", + "position", + "static", + "pointer-events", + "auto", + "cursor", + "pointer", + ]; + const i = (s: string) => S.indexOf(s); + return { + strings: S, + documents: [ + { + nodes: { + parentIndex: [-1, 0, 1, 2, 3], + nodeType: [1, 1, 1, 1, 1], + nodeName: [i("html"), i("body"), i("div"), i("div"), i("img")], + backendNodeId: [10, 11, 12, 13, 14], + attributes: [ + [], + [], + [i("class"), i("tg-avatar")], + [i("class"), i("tg-avatar__inner")], + [i("class"), i("tg-avatar__image")], + ], + }, + layout: { + nodeIndex: [1, 2, 3, 4], + styles: [ + [i("static"), i("auto"), i("pointer")], + [i("static"), i("auto"), i("pointer")], + [i("static"), i("auto"), i("pointer")], + [i("static"), i("auto"), i("pointer")], + ], + bounds: [ + [0, 0, 1000, 800], + [940, 16, 32, 32], + [942, 18, 28, 28], + [944, 20, 24, 24], + ], + paintOrders: [0, 1, 2, 3], + }, + }, + ], + }; +} + function makeCdp(snapshot: unknown) { return { send: vi.fn(async (_tab: number, method: string) => { @@ -126,11 +276,11 @@ describe("captureViewModel", () => { }); it("runs hover surface probes when explicitly enabled", async () => { - let runtimeCalls = 0; + let hoverStateCalls = 0; const cdp = { send: vi.fn(async (_tabId: number, method: string, params?: object) => { if (method === "DOMSnapshot.enable") return {}; - if (method === "DOMSnapshot.captureSnapshot") return fakeSnapshotReply(); + if (method === "DOMSnapshot.captureSnapshot") return hoverTriggerSnapshotReply(); if (method === "Page.getLayoutMetrics") { return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, @@ -141,22 +291,23 @@ describe("captureViewModel", () => { if (expression.includes("input,textarea,select")) { return { result: { value: { controls: [], childFrames: [] } } }; } - runtimeCalls += 1; - return runtimeCalls === 1 - ? { - result: { - value: [ - { - triggerSel: ".menu", - affectedSub: " .items", - label: "Products", - x: 50, - y: 20, - }, - ], - }, - } - : { result: { value: ["Shoes", "Bags"] } }; + if (expression.includes("document.styleSheets")) { + return { result: { value: [] } }; + } + if (expression.includes("querySelectorAll(selectors)")) { + hoverStateCalls += 1; + return hoverStateCalls === 1 + ? { result: { value: [] } } + : { + result: { + value: [ + { text: "My profile", role: "", tag: "a", x: 900, y: 60 }, + { text: "Sign out", role: "", tag: "a", x: 900, y: 90 }, + ], + }, + }; + } + throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`); } if (method === "Input.dispatchMouseEvent") return {}; throw new Error(`unexpected ${method}`); @@ -166,12 +317,18 @@ describe("captureViewModel", () => { const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true }); expect(result.surfaceProbes).toEqual([ - { triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] }, + { + triggerBackendNodeId: 12, + triggerPoint: { x: 956, y: 32 }, + triggerAction: "hover", + subItems: ["My profile", "Sign out"], + confidence: "high", + }, ]); expect(cdp.send).toHaveBeenCalledWith( 4, "Input.dispatchMouseEvent", - expect.objectContaining({ type: "mouseMoved", x: 50, y: 20 }), + expect.objectContaining({ type: "mouseMoved", x: 956, y: 32 }), ); expect(cdp.send).toHaveBeenCalledWith( 4, @@ -180,6 +337,91 @@ describe("captureViewModel", () => { ); }); + it("uses a fresh baseline for each hover candidate", async () => { + let hoverStateCalls = 0; + const cdp = { + send: vi.fn(async (_tabId: number, method: string, params?: object) => { + if (method === "DOMSnapshot.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") return twoHoverTriggerSnapshotReply(); + if (method === "Page.getLayoutMetrics") { + return { + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, + }; + } + if (method === "Runtime.evaluate") { + const expression = (params as { expression?: string } | undefined)?.expression ?? ""; + if (expression.includes("input,textarea,select")) { + return { result: { value: { controls: [], childFrames: [] } } }; + } + if (expression.includes("document.styleSheets")) { + return { result: { value: [] } }; + } + if (expression.includes("querySelectorAll(selectors)")) { + hoverStateCalls += 1; + const visibleMenu = [ + { text: "My profile", role: "", tag: "a", x: 900, y: 60 }, + { text: "Sign out", role: "", tag: "a", x: 900, y: 90 }, + ]; + return hoverStateCalls === 1 + ? { result: { value: [] } } + : { result: { value: visibleMenu } }; + } + throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`); + } + if (method === "Input.dispatchMouseEvent") return {}; + throw new Error(`unexpected ${method}`); + }) as unknown as (tabId: number, method: string, params?: object) => Promise, + }; + + const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true }); + + expect(result.surfaceProbes).toEqual([ + expect.objectContaining({ + triggerBackendNodeId: 12, + subItems: ["My profile", "Sign out"], + }), + ]); + expect(hoverStateCalls).toBeGreaterThanOrEqual(4); + }); + + it("deduplicates nested hover candidates for one visual trigger", async () => { + let hoverMoves = 0; + const cdp = { + send: vi.fn(async (_tabId: number, method: string, params?: object) => { + if (method === "DOMSnapshot.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") return nestedHoverTriggerSnapshotReply(); + if (method === "Page.getLayoutMetrics") { + return { + cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, + }; + } + if (method === "Runtime.evaluate") { + const expression = (params as { expression?: string } | undefined)?.expression ?? ""; + if (expression.includes("input,textarea,select")) { + return { result: { value: { controls: [], childFrames: [] } } }; + } + if (expression.includes("document.styleSheets")) { + return { result: { value: [] } }; + } + if (expression.includes("querySelectorAll(selectors)")) { + return { result: { value: [] } }; + } + throw new Error(`unexpected Runtime.evaluate: ${expression.slice(0, 80)}`); + } + if (method === "Input.dispatchMouseEvent") { + const point = params as { x?: number; y?: number }; + if (point.x !== -10 && point.x !== 0) hoverMoves += 1; + return {}; + } + throw new Error(`unexpected ${method}`); + }) as unknown as (tabId: number, method: string, params?: object) => Promise, + }; + + await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true }); + + expect(hoverMoves).toBe(1); + }); + it("excludes the agent's own overlay shadow host and its inlined shadow subtree", async () => { // DOMSnapshot inlines an open shadow root's content as descendants of the // host. The agent's WXT overlay host carries a fixed full-viewport diff --git a/apps/extension/src/tools/vom/capture.ts b/apps/extension/src/tools/vom/capture.ts index d8358f5..44aba56 100644 --- a/apps/extension/src/tools/vom/capture.ts +++ b/apps/extension/src/tools/vom/capture.ts @@ -45,9 +45,11 @@ export interface CapturedNode { export type CapturedIframeNodes = Map; export interface CapturedSurfaceProbe { - triggerLabel: string; + triggerBackendNodeId: number; + triggerPoint?: { x: number; y: number }; triggerAction: "hover" | "focus" | string; subItems: string[]; + confidence?: "high" | "medium" | "low"; } export interface CapturedViewModel { @@ -61,6 +63,7 @@ export interface CapturedViewModel { export interface CaptureViewModelOptions { conditionalSurfaceProbe?: boolean; + hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise; } /** Sparse array format Chrome uses for infrequently-set per-node fields. */ @@ -225,11 +228,12 @@ interface RuntimeEvaluateReply { } interface HoverCandidate { - triggerSel: string; - affectedSub: string; - label: string; + backendNodeId: number; + label?: string; x: number; y: number; + score: number; + reasons: string[]; } interface CdpDomNode { @@ -322,17 +326,19 @@ function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set(reply: RuntimeEvaluateReply): T | undefined { return reply.result?.value as T | undefined; } -function hoverCssScanExpression(): string { +function hoverCssTriggerScanExpression(): string { return `(() => { const visibilityProps = ["display", "visibility", "opacity", "maxHeight", "height", "overflow"]; - const pairs = []; + const centres = []; const seenRules = new Set(); for (const sheet of Array.from(document.styleSheets)) { let rules; @@ -348,90 +354,73 @@ function hoverCssScanExpression(): string { const hoverIndex = part.indexOf(":hover"); if (hoverIndex < 0) continue; const triggerSel = part.slice(0, hoverIndex).trim(); - const affectedSub = part.slice(hoverIndex + 6).trim(); if (!triggerSel) continue; - const key = triggerSel + "||" + affectedSub; - if (seenRules.has(key)) continue; - seenRules.add(key); - pairs.push({ triggerSel, affectedSub }); + if (seenRules.has(triggerSel)) continue; + seenRules.add(triggerSel); + let elements; + try { elements = Array.from(document.querySelectorAll(triggerSel)); } catch { continue; } + for (const el of elements) { + if (!(el instanceof HTMLElement)) continue; + const rect = el.getBoundingClientRect(); + const style = getComputedStyle(el); + if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue; + centres.push({ x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 }); + if (centres.length >= 24) return centres; + } } } } - - const candidates = []; - const seenLabels = new Set(); - const isUnsafeTrigger = (el) => { - if (!(el instanceof HTMLElement)) return true; - const tag = el.tagName.toLowerCase(); - if (["input", "textarea", "select", "option"].includes(tag)) return true; - if (el.isContentEditable) return true; - if (el.hasAttribute("disabled") || el.hasAttribute("inert")) return true; - if ((el.getAttribute("aria-disabled") || "").toLowerCase() === "true") return true; - return false; - }; - for (const pair of pairs) { - let elements; - try { elements = Array.from(document.querySelectorAll(pair.triggerSel)); } catch { continue; } - for (const el of elements) { - if (!(el instanceof Element)) continue; - if (isUnsafeTrigger(el)) continue; - const rect = el.getBoundingClientRect(); - const style = getComputedStyle(el); - if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none" || style.pointerEvents === "none") continue; - const label = (el.textContent || "").replace(/\\s+/g, " ").trim(); - if (!label || seenLabels.has(label)) continue; - seenLabels.add(label); - candidates.push({ - triggerSel: pair.triggerSel, - affectedSub: pair.affectedSub, - label, - x: rect.left + rect.width / 2, - y: rect.top + rect.height / 2, - }); - if (candidates.length >= ${MAX_HOVER_TRIGGERS}) return candidates; - } - } - return candidates; + return centres; })()`; } -function hoverCollectExpression(candidate: HoverCandidate): string { - return `(() => { - const triggerSel = ${JSON.stringify(candidate.triggerSel)}; - const rawAffectedSub = ${JSON.stringify(candidate.affectedSub)}; - const x = ${JSON.stringify(candidate.x)}; - const y = ${JSON.stringify(candidate.y)}; - const hit = document.elementFromPoint(x, y); - const trigger = hit instanceof Element ? hit.closest(triggerSel) : null; - if (!trigger) return []; +interface HoverRuntimeItem { + text: string; + role: string; + tag: string; + x: number; + y: number; +} - const affectedSub = rawAffectedSub.replace(/^[>+~]\\s*/, "").trim(); - let targets = []; - if (!affectedSub) { - targets = [trigger]; - } else { - try { targets = Array.from(trigger.querySelectorAll(affectedSub)); } catch { targets = []; } - } +function hoverStateExpression(): string { + return `(() => { const items = []; const seen = new Set(); - const push = (value) => { - const text = String(value || "").replace(/\\s+/g, " ").trim(); + const selectors = [ + "a", + "button", + "[role='menuitem']", + "[role='menuitemcheckbox']", + "[role='menuitemradio']", + "[role='option']", + "[role='tab']", + "[role='link']", + "[role='button']" + ].join(","); + const push = (el) => { + if (!(el instanceof HTMLElement)) return; + const style = getComputedStyle(el); + const rect = el.getBoundingClientRect(); + if (rect.width <= 0 || rect.height <= 0) return; + if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") return; + const text = String( + el.getAttribute("aria-label") || + el.getAttribute("title") || + el.textContent || + "" + ).replace(/\\s+/g, " ").trim(); if (!text || seen.has(text)) return; seen.add(text); - items.push(text); + items.push({ + text, + role: (el.getAttribute("role") || "").toLowerCase(), + tag: el.tagName.toLowerCase(), + x: rect.left, + y: rect.top, + }); }; - for (const target of targets) { - if (!(target instanceof Element)) continue; - const style = getComputedStyle(target); - if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") continue; - const clickable = target.querySelectorAll('a, button, [role="menuitem"], [role="option"]'); - if (clickable.length > 0) { - for (const child of Array.from(clickable)) push(child.textContent); - } else { - push(target.textContent); - } - } - return items.slice(0, 12); + for (const el of Array.from(document.querySelectorAll(selectors))) push(el); + return items.slice(0, 400); })()`; } @@ -453,45 +442,313 @@ async function clearHover(cdp: CdpRunner, tabId: number): Promise { ); } -async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise { +function isVisibleProbeNode(node: CapturedNode): boolean { + const rect = node.rect; + return ( + !!rect && + rect.w > 0 && + rect.h > 0 && + node.pointerEvents !== "none" && + node.attrs.hidden === undefined && + node.attrs.inert === undefined && + (node.attrs["aria-hidden"] ?? "").toLowerCase() !== "true" + ); +} + +function isUnsafeHoverTrigger(node: CapturedNode): boolean { + const tag = node.tag.toLowerCase(); + if (["input", "textarea", "select", "option"].includes(tag)) return true; + if ((node.attrs.contenteditable ?? "").toLowerCase() === "true") return true; + if (node.attrs.disabled !== undefined || node.attrs.inert !== undefined) return true; + return (node.attrs["aria-disabled"] ?? "").toLowerCase() === "true"; +} + +function capturedText(node: CapturedNode): string | undefined { + const value = + node.attrs["aria-label"] ?? node.attrs.title ?? node.attrs.alt ?? node.textContent ?? ""; + const clean = value.replace(/\s+/g, " ").trim(); + return clean || undefined; +} + +function hasGraphicDescendant( + node: CapturedNode, + childrenByParentId: Map, + depth = 0, +): boolean { + if (depth > 3) return false; + for (const child of childrenByParentId.get(node.backendNodeId) ?? []) { + const tag = child.tag.toLowerCase(); + if (["img", "svg", "use", "path", "i"].includes(tag)) return true; + if (hasGraphicDescendant(child, childrenByParentId, depth + 1)) return true; + } + return false; +} + +function hasHoverPopupSignal(node: CapturedNode): boolean { + const attrs = node.attrs; + const haystack = [ + attrs.id, + attrs.class, + attrs["data-testid"], + attrs["data-test"], + attrs["data-cy"], + attrs["aria-haspopup"], + attrs["aria-controls"], + ] + .filter(Boolean) + .join(" ") + .toLowerCase(); + return /(menu|dropdown|popover|popup|avatar|profile|account|user)/.test(haystack); +} + +function roleOf(node: CapturedNode): string { + return (node.attrs.role ?? "").toLowerCase(); +} + +function hasDirectInteractiveSignal(node: CapturedNode): boolean { + const tag = node.tag.toLowerCase(); + const role = roleOf(node); + return ( + ["button", "a", "summary"].includes(tag) || + ["button", "link", "menuitem", "tab", "combobox"].includes(role) || + node.attrs.tabindex !== undefined || + node.cursor === "pointer" || + node.attrs.onclick !== undefined || + node.attrs.onmouseenter !== undefined || + node.attrs.onmouseover !== undefined || + node.attrs["aria-haspopup"] !== undefined || + node.attrs["aria-expanded"] === "false" + ); +} + +function scoreHoverCandidate( + node: CapturedNode, + childrenByParentId: Map, + cssHoverPoints: Array<{ x: number; y: number }>, +): HoverCandidate | null { + if (!isVisibleProbeNode(node) || isUnsafeHoverTrigger(node)) { + return null; + } + const rect = node.rect; + if (!rect) return null; + const area = rect.w * rect.h; + if (area <= 0 || area > 160_000) return null; + + const reasons: string[] = []; + let score = 0; + const tag = node.tag.toLowerCase(); + const role = roleOf(node); + const label = capturedText(node); + const graphic = hasGraphicDescendant(node, childrenByParentId); + const directInteractive = hasDirectInteractiveSignal(node); + const popupSignal = hasHoverPopupSignal(node); + const compactTopbarIcon = !label && graphic && rect.y < 120 && rect.w <= 96 && rect.h <= 96; + if (!directInteractive && !popupSignal && !compactTopbarIcon) return null; + + if (node.attrs["aria-haspopup"] !== undefined) { + score += 80; + reasons.push("aria-haspopup"); + } + if ((node.attrs["aria-expanded"] ?? "").toLowerCase() === "false") { + score += 55; + reasons.push("collapsed"); + } + if (popupSignal) { + score += 45; + reasons.push("popup-signal"); + } + if (tag === "button" || role === "button") { + score += 30; + reasons.push("button"); + } + if (node.cursor === "pointer") { + score += 25; + reasons.push("pointer"); + } + if (!label && graphic) { + score += 40; + reasons.push("icon-only"); + } + if (rect.y < 120) { + score += 25; + reasons.push("topbar"); + } + if (rect.w <= 80 && rect.h <= 80) { + score += 20; + reasons.push("compact"); + } + if ( + cssHoverPoints.some( + (point) => + point.x >= rect.x && + point.x <= rect.x + rect.w && + point.y >= rect.y && + point.y <= rect.y + rect.h, + ) + ) { + score += 40; + reasons.push("css-hover"); + } + + if (score < 45) return null; + return { + backendNodeId: node.backendNodeId, + label, + x: rect.x + rect.w / 2, + y: rect.y + rect.h / 2, + score, + reasons, + }; +} + +function buildHoverCandidates( + nodes: CapturedNode[], + cssHoverPoints: Array<{ x: number; y: number }>, +): HoverCandidate[] { + const childrenByParentId = new Map(); + const parentByBackendId = new Map(); + for (const node of nodes) { + parentByBackendId.set(node.backendNodeId, node.parentBackendNodeId); + if (node.parentBackendNodeId === null) continue; + const children = childrenByParentId.get(node.parentBackendNodeId) ?? []; + children.push(node); + childrenByParentId.set(node.parentBackendNodeId, children); + } + + const candidates = nodes + .map((node) => scoreHoverCandidate(node, childrenByParentId, cssHoverPoints)) + .filter((candidate): candidate is HoverCandidate => candidate !== null) + .sort((a, b) => b.score - a.score); + + const deduped: HoverCandidate[] = []; + const seen = new Set(); + for (const candidate of candidates) { + if (seen.has(candidate.backendNodeId)) continue; + if (deduped.some((existing) => sameHoverCluster(existing, candidate, parentByBackendId))) { + continue; + } + seen.add(candidate.backendNodeId); + deduped.push(candidate); + if (deduped.length >= MAX_HOVER_TRIGGERS) break; + } + return deduped; +} + +function sameHoverCluster( + a: HoverCandidate, + b: HoverCandidate, + parentByBackendId: Map, +): boolean { + if (Math.hypot(a.x - b.x, a.y - b.y) <= 8) return true; + return ( + isBackendAncestor(a.backendNodeId, b.backendNodeId, parentByBackendId) || + isBackendAncestor(b.backendNodeId, a.backendNodeId, parentByBackendId) + ); +} + +function isBackendAncestor( + ancestorId: number, + nodeId: number, + parentByBackendId: Map, +): boolean { + let parentId = parentByBackendId.get(nodeId); + let guard = 0; + while (parentId !== null && parentId !== undefined && guard < parentByBackendId.size) { + if (parentId === ancestorId) return true; + parentId = parentByBackendId.get(parentId); + guard += 1; + } + return false; +} + +function diffHoverItems(before: HoverRuntimeItem[], after: HoverRuntimeItem[]): string[] { + const beforeKeys = new Set(before.map((item) => item.text.toLowerCase())); + const out: string[] = []; + const seen = new Set(); + for (const item of after) { + const text = item.text.replace(/\s+/g, " ").trim(); + const key = text.toLowerCase(); + if (!text || beforeKeys.has(key) || seen.has(key)) continue; + seen.add(key); + out.push(text); + if (out.length >= MAX_HOVER_SUB_ITEMS) break; + } + return out; +} + +function confidenceForHover( + candidate: HoverCandidate, + subItems: string[], +): "high" | "medium" | "low" { + if (subItems.length >= 2 && candidate.score >= 80) return "high"; + if (subItems.length >= 2 || candidate.score >= 80) return "medium"; + return "low"; +} + +async function probeHoverSurfaces( + cdp: CdpRunner, + tabId: number, + nodes: CapturedNode[], + options: CaptureViewModelOptions, +): Promise { const started = Date.now(); try { - const scan = await cdp.send(tabId, "Runtime.evaluate", { - expression: hoverCssScanExpression(), + const cssScan = await cdp.send(tabId, "Runtime.evaluate", { + expression: hoverCssTriggerScanExpression(), returnByValue: true, }); - const candidates = runtimeValue(scan) ?? []; + const cssHoverPoints = runtimeValue>(cssScan) ?? []; + const candidates = buildHoverCandidates(nodes, cssHoverPoints); + if (candidates.length === 0) return []; + const results: CapturedSurfaceProbe[] = []; - const seen = new Set(); - for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) { - if (Date.now() - started > MAX_HOVER_PROBE_MS) break; - if (!candidate.label || seen.has(candidate.label)) continue; - try { - await cdp.send(tabId, "Input.dispatchMouseEvent", { - type: "mouseMoved", - x: candidate.x, - y: candidate.y, - }); - await wait(HOVER_SETTLE_MS); - const collected = await cdp.send(tabId, "Runtime.evaluate", { - expression: hoverCollectExpression(candidate), - returnByValue: true, - }); - const subItems = (runtimeValue(collected) ?? []) - .map((item) => item.replace(/\s+/g, " ").trim()) - .filter(Boolean); - if (subItems.length === 0) continue; - seen.add(candidate.label); - results.push({ - triggerLabel: candidate.label, - triggerAction: "hover", - subItems, - }); - } catch { - continue; - } finally { - await clearHover(cdp, tabId); + const seen = new Set(); + await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined); + try { + for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) { + if (Date.now() - started > MAX_HOVER_PROBE_MS) break; + if (results.length >= MAX_HOVER_SURFACES) break; + if (seen.has(candidate.backendNodeId)) continue; + try { + await clearHover(cdp, tabId); + await wait(HOVER_SETTLE_MS); + const baselineReply = await cdp.send(tabId, "Runtime.evaluate", { + expression: hoverStateExpression(), + returnByValue: true, + }); + const baselineItems = runtimeValue(baselineReply) ?? []; + + await cdp.send(tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: candidate.x, + y: candidate.y, + }); + await wait(HOVER_SETTLE_MS); + const collected = await cdp.send(tabId, "Runtime.evaluate", { + expression: hoverStateExpression(), + returnByValue: true, + }); + const subItems = diffHoverItems( + baselineItems, + runtimeValue(collected) ?? [], + ); + if (subItems.length === 0) continue; + seen.add(candidate.backendNodeId); + results.push({ + triggerBackendNodeId: candidate.backendNodeId, + triggerPoint: { x: candidate.x, y: candidate.y }, + triggerAction: "hover", + subItems, + confidence: confidenceForHover(candidate, subItems), + }); + } catch { + continue; + } finally { + await clearHover(cdp, tabId); + } } + } finally { + await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined); } return results; } catch (err) { @@ -797,7 +1054,9 @@ export async function captureViewModel( await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()]); - const surfaceProbes = options.conditionalSurfaceProbe ? await probeHoverSurfaces(cdp, tabId) : []; + const surfaceProbes = options.conditionalSurfaceProbe + ? await probeHoverSurfaces(cdp, tabId, nodes, options) + : []; return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds }; } diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index c3ebdff..4533d7d 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -314,8 +314,21 @@ export interface SnapshotResult { dialogs?: JavaScriptDialogInfo[]; } -export type ObserveParams = SnapshotParams; -export type ObserveResult = SnapshotResult; +export interface ObserveParams extends SnapshotParams { + debug_surfaces?: boolean; +} + +export interface ObserveResult extends SnapshotResult { + debug?: { + surface_probes?: Array<{ + trigger_backend_node_id: number; + trigger_point?: { x: number; y: number }; + trigger_action: string; + sub_items: string[]; + confidence?: string; + }>; + }; +} export interface GetHtmlParams { session_id: string; diff --git a/crates/bsk-cli/src/cli/observe.rs b/crates/bsk-cli/src/cli/observe.rs index 72efa28..bae34fe 100644 --- a/crates/bsk-cli/src/cli/observe.rs +++ b/crates/bsk-cli/src/cli/observe.rs @@ -29,6 +29,10 @@ pub struct ObserveArgs { /// Soft cap on rendered tokens (~4 chars/token). #[arg(long = "max-tokens")] pub max_tokens: Option, + + /// Include conditional surface probe diagnostics in JSON output. + #[arg(long = "debug-surfaces")] + pub debug_surfaces: bool, } pub fn dispatch(args: ObserveArgs, format: Format) -> Result<(), CliError> { @@ -42,6 +46,7 @@ fn run(sock: PathBuf, args: ObserveArgs, format: Format) -> Result<(), CliError> tab_id: args.tab_id, max_depth: args.max_depth, max_tokens: args.max_tokens, + debug_surfaces: args.debug_surfaces, }; let reply: ObserveResult = call(sock, params)?; match format { diff --git a/crates/bsk-cli/tests/tools_ipc.rs b/crates/bsk-cli/tests/tools_ipc.rs index 63714a0..d40bb9c 100644 --- a/crates/bsk-cli/tests/tools_ipc.rs +++ b/crates/bsk-cli/tests/tools_ipc.rs @@ -478,6 +478,7 @@ async fn observe_returns_semantic_text_and_ref_count() { tab_id: 13, truncated: false, dialogs: vec![], + debug: None, }) .unwrap(), ) @@ -492,6 +493,7 @@ async fn observe_returns_semantic_text_and_ref_count() { tab_id: None, max_depth: None, max_tokens: None, + debug_surfaces: false, }, ) .await diff --git a/crates/bsk-protocol/schema/tool_observe_params.json b/crates/bsk-protocol/schema/tool_observe_params.json index 2da54a9..10e57a0 100644 --- a/crates/bsk-protocol/schema/tool_observe_params.json +++ b/crates/bsk-protocol/schema/tool_observe_params.json @@ -7,6 +7,11 @@ "session_id" ], "properties": { + "debug_surfaces": { + "description": "Include implementation diagnostics for conditional surface probes.", + "default": false, + "type": "boolean" + }, "max_depth": { "description": "Cap the depth of the rendered VOM tree.", "type": [ diff --git a/crates/bsk-protocol/schema/tool_observe_result.json b/crates/bsk-protocol/schema/tool_observe_result.json index 2b2d9fd..22ee410 100644 --- a/crates/bsk-protocol/schema/tool_observe_result.json +++ b/crates/bsk-protocol/schema/tool_observe_result.json @@ -8,6 +8,16 @@ "text" ], "properties": { + "debug": { + "anyOf": [ + { + "$ref": "#/definitions/ObserveDebug" + }, + { + "type": "null" + } + ] + }, "dialogs": { "type": "array", "items": { @@ -103,6 +113,73 @@ "prompt", "beforeunload" ] + }, + "ObserveDebug": { + "type": "object", + "properties": { + "surface_probes": { + "type": "array", + "items": { + "$ref": "#/definitions/SurfaceProbeDebug" + } + } + } + }, + "SurfaceProbeDebug": { + "type": "object", + "required": [ + "sub_items", + "trigger_action", + "trigger_backend_node_id" + ], + "properties": { + "confidence": { + "type": [ + "string", + "null" + ] + }, + "sub_items": { + "type": "array", + "items": { + "type": "string" + } + }, + "trigger_action": { + "type": "string" + }, + "trigger_backend_node_id": { + "type": "integer", + "format": "int64" + }, + "trigger_point": { + "anyOf": [ + { + "$ref": "#/definitions/SurfaceProbePoint" + }, + { + "type": "null" + } + ] + } + } + }, + "SurfaceProbePoint": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number", + "format": "double" + }, + "y": { + "type": "number", + "format": "double" + } + } } } } diff --git a/crates/bsk-protocol/src/tools/observation.rs b/crates/bsk-protocol/src/tools/observation.rs index 715ddc9..0103ea5 100644 --- a/crates/bsk-protocol/src/tools/observation.rs +++ b/crates/bsk-protocol/src/tools/observation.rs @@ -74,6 +74,32 @@ pub struct ObserveParams { /// best-effort heuristic based on character count). #[serde(default, skip_serializing_if = "Option::is_none")] pub max_tokens: Option, + /// Include implementation diagnostics for conditional surface probes. + #[serde(default)] + pub debug_surfaces: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct SurfaceProbePoint { + pub x: f64, + pub y: f64, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct SurfaceProbeDebug { + pub trigger_backend_node_id: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trigger_point: Option, + pub trigger_action: String, + pub sub_items: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub confidence: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ObserveDebug { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub surface_probes: Vec, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] @@ -92,6 +118,8 @@ pub struct ObserveResult { pub truncated: bool, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub dialogs: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub debug: Option, } // --------------------------------------------------------------------------- @@ -280,6 +308,7 @@ mod tests { tab_id: 42, truncated: false, dialogs: Vec::new(), + debug: None, }; let v = serde_json::to_value(&r).unwrap(); assert_eq!(v.get("ref_count").and_then(|v| v.as_u64()), Some(1)); diff --git a/packages/vom/src/index.ts b/packages/vom/src/index.ts index 45fe4ac..745eb00 100644 --- a/packages/vom/src/index.ts +++ b/packages/vom/src/index.ts @@ -1,4 +1,4 @@ -export { renderVom } from "./render"; +export { applyVomInteractionRecovery, isVomReferenceNode, renderVom } from "./render"; export type { ActiveScopeBlock, BlockingLayer, diff --git a/packages/vom/src/render.ts b/packages/vom/src/render.ts index 742b58a..0926fee 100644 --- a/packages/vom/src/render.ts +++ b/packages/vom/src/render.ts @@ -197,7 +197,7 @@ function shouldRender(node: VomNode): boolean { return cleaned(node.name) !== undefined; } -function shouldReference(node: VomNode): boolean { +export function isVomReferenceNode(node: VomNode): boolean { return INTERACTIVE_ROLES.has(normalizedRole(node)) || isUrlBearingStructuralAction(node); } @@ -342,7 +342,7 @@ function isAncestorOf( return false; } -function applyRecovery(nodes: VomNode[]): VomNode[] { +export function applyVomInteractionRecovery(nodes: VomNode[]): VomNode[] { const parentMap = buildParentMap(nodes); const children = buildChildren(nodes); const recoveredIds = new Set(); @@ -350,7 +350,7 @@ function applyRecovery(nodes: VomNode[]): VomNode[] { const recoveredNameById = new Map(); for (const node of nodes) { - if (shouldReference(node)) continue; + if (isVomReferenceNode(node)) continue; const role = recoveredRole(node, children); if (!role || !passesRecoveryGuards(node, children)) continue; recoveredIds.add(node.id); @@ -397,7 +397,7 @@ function buildChildren(nodes: VomNode[]): Map { function duplicateReferenceNames(nodes: VomNode[]): Set { const counts = new Map(); for (const node of nodes) { - if (!shouldReference(node)) continue; + if (!isVomReferenceNode(node)) continue; const name = cleaned(node.name); if (!name) continue; const key = normalizeContextKey(name); @@ -427,7 +427,7 @@ function contextLabel(node: VomNode, targetName: string): string | undefined { } function weakLabelContext(node: VomNode, targetName: string): string | undefined { - if (shouldReference(node)) return undefined; + if (isVomReferenceNode(node)) return undefined; return weakContextText( cleaned(node.name) ?? cleaned(node.text) ?? cleaned(node.nearbyText), targetName, @@ -509,7 +509,7 @@ function collectWeakLabelsFromSubtree( state: RenderState, labels: string[], ): void { - if (shouldReference(node)) return; + if (isVomReferenceNode(node)) return; const label = weakLabelContext(node, nodeName); if (label) { labels.push(label); @@ -666,7 +666,7 @@ function descendantTextCoveredByName(node: VomNode, state: RenderState): boolean const texts: string[] = []; while (stack.length > 0) { const child = stack.pop() as VomNode; - if (shouldReference(child)) return false; + if (isVomReferenceNode(child)) return false; const text = cleaned(child.name) ?? cleaned(child.text) ?? cleaned(child.value); if (text) texts.push(text); stack.push(...(state.children.get(child.id) ?? [])); @@ -750,7 +750,7 @@ function renderTree( continue; } - const ref = shouldReference(node) ? `e${state.nextRef}` : undefined; + const ref = isVomReferenceNode(node) ? `e${state.nextRef}` : undefined; const context = ref ? handleContext(node, state) : []; const surface = state.surfaceMap.get(node.id); const line = renderNodeLine(node, depth, ref, context, surface); @@ -915,7 +915,7 @@ function applyActiveRegionPolicy(nodes: VomNode[], scene: VomScene): VomNode[] { const blockedRoots = new Set(); for (const target of nodes) { - if (!shouldReference(target)) continue; + if (!isVomReferenceNode(target)) continue; const blocked = candidates.some((candidate) => isBlockedByRegion(target, candidate.node, parentMap, candidate.viewportCoverage), ); @@ -970,7 +970,7 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp } export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult { - const nodes = applyRecovery(scene.nodes); + const nodes = applyVomInteractionRecovery(scene.nodes); const renderScene = { ...scene, nodes }; const layer = detectBlockingLayer(nodes, scene.viewport); if (layer) return renderDoubleLayer(renderScene, layer, options);