diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index f275939..5c385f6 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -570,6 +570,82 @@ describe("buildVomScene", () => { ]); }); + it("dedupes iframe fallback controls already exposed through AX", () => { + const axNodes: CdpAxNode[] = [ + { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + backendDOMNodeId: 10, + childIds: ["2"], + }, + { + nodeId: "2", + parentId: "1", + role: { type: "role", value: "Iframe" }, + backendDOMNodeId: 20, + childIds: ["3"], + }, + { + nodeId: "3", + parentId: "2", + role: { type: "role", value: "textbox" }, + name: { type: "x", value: "输入密码" }, + backendDOMNodeId: 202, + properties: [{ name: "inputType", value: { value: "password" } }], + }, + ]; + const captured: CapturedViewModel = { + viewport: { width: 1000, height: 800 }, + nodes: [ + { + backendNodeId: 10, + parentBackendNodeId: null, + tag: "body", + attrs: {}, + rect: { x: 0, y: 0, w: 1000, h: 800 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 20, + parentBackendNodeId: 10, + tag: "iframe", + attrs: {}, + rect: { x: 100, y: 100, w: 400, h: 300 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + ], + iframeNodes: new Map([ + [ + 20, + [ + { + backendNodeId: 102, + parentBackendNodeId: null, + tag: "input", + attrs: { type: "password", placeholder: "输入密码" }, + rect: { x: 0, y: 50, w: 300, h: 40 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + ], + ], + ]), + excludedBackendNodeIds: new Set(), + }; + + const passwordNodes = buildVomScene(axNodes, captured).nodes.filter( + (node) => node.role === "textbox" && node.name === "输入密码", + ); + + expect(passwordNodes).toHaveLength(1); + expect(passwordNodes[0]).toEqual(expect.objectContaining({ id: 202, sensitive: true })); + }); + it("keeps unnamed iframe controls but skips unnamed iframe links", () => { const scene = buildVomScene( [ @@ -843,6 +919,66 @@ describe("buildVomScene", () => { ); }); + it("uses nearby preceding text as a form control label", () => { + const axNodes: CdpAxNode[] = [ + { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + backendDOMNodeId: 1, + childIds: ["2"], + }, + { + nodeId: "2", + parentId: "1", + role: { type: "role", value: "textbox" }, + backendDOMNodeId: 20, + }, + ]; + const captured: CapturedViewModel = { + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + nodes: [ + { + backendNodeId: 1, + parentBackendNodeId: null, + tag: "div", + attrs: {}, + rect: { x: 0, y: 0, w: 500, h: 80 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 10, + parentBackendNodeId: 1, + tag: "span", + attrs: {}, + textContent: "验证码:", + rect: { x: 0, y: 0, w: 80, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 20, + parentBackendNodeId: 1, + tag: "input", + attrs: { type: "text" }, + rect: { x: 90, y: 0, w: 120, h: 30 }, + paintOrder: 2, + position: "static", + pointerEvents: "auto", + cursor: "text", + }, + ], + }; + + expect(buildVomScene(axNodes, captured).nodes.find((node) => node.id === 20)).toEqual( + expect.objectContaining({ role: "textbox", name: "验证码" }), + ); + }); + it("promotes a cursor:pointer icon control to a button named from its sprite", () => { // A close button built as
// — the AX tree exposes it as `generic` with no name, so it would be dropped. @@ -1065,6 +1201,284 @@ describe("buildVomScene", () => { expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]); }); + it("builds active scope blocks from active aria-controls relationships", () => { + const axNodes: CdpAxNode[] = [ + { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + backendDOMNodeId: 10, + childIds: ["2"], + }, + { + nodeId: "2", + parentId: "1", + role: { type: "role", value: "tab" }, + name: { type: "computedString", value: "Reviews (12)" }, + backendDOMNodeId: 20, + }, + ]; + const captured: CapturedViewModel = { + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + nodes: [ + { + backendNodeId: 10, + parentBackendNodeId: null, + tag: "body", + attrs: {}, + rect: { x: 0, y: 0, w: 1000, h: 800 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 20, + parentBackendNodeId: 10, + tag: "button", + attrs: { role: "tab", "aria-selected": "true", "aria-controls": "reviews-panel" }, + rect: { x: 20, y: 20, w: 120, h: 40 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 30, + parentBackendNodeId: 10, + tag: "div", + attrs: { id: "reviews-panel" }, + rect: null, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + textContent: "Jane - ear cups are small", + }, + { + backendNodeId: 31, + parentBackendNodeId: 30, + tag: "p", + attrs: {}, + rect: null, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + textContent: "Bob - great sound", + }, + ], + }; + + const scene = buildVomScene(axNodes, captured); + expect(scene.activeScopeBlocks).toEqual([ + { + triggerId: 20, + label: "Reviews (12)", + lines: ["Jane - ear cups are small", "Bob - great sound"], + }, + ]); + expect(renderVom(scene).text).toContain("[§ active: Reviews (12)]"); + }); + + it("maps hover surface probes to matching node labels", () => { + const axNodes: CdpAxNode[] = [ + { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + backendDOMNodeId: 10, + childIds: ["2"], + }, + { + nodeId: "2", + parentId: "1", + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Products" }, + backendDOMNodeId: 20, + }, + ]; + const scene = buildVomScene(axNodes, { + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + surfaceProbes: [ + { triggerLabel: "Products", triggerAction: "hover", subItems: ["Shoes", "Bags"] }, + ], + nodes: [ + { + backendNodeId: 10, + parentBackendNodeId: null, + tag: "body", + attrs: {}, + rect: { x: 0, y: 0, w: 1000, h: 800 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 20, + parentBackendNodeId: 10, + tag: "button", + attrs: {}, + rect: { x: 20, y: 20, w: 120, h: 40 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + ], + }); + + expect(scene.surfaces).toEqual([ + { triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] }, + ]); + expect(renderVom(scene).text).toContain('@e1 button "Products" [→ Shoes | Bags]'); + }); + + it("enriches names and active scope signals from AX properties", () => { + const axNodes: CdpAxNode[] = [ + { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + backendDOMNodeId: 10, + childIds: ["2", "3"], + }, + { + nodeId: "2", + parentId: "1", + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Products" }, + backendDOMNodeId: 20, + properties: [ + { name: "hasPopup", value: { value: "menu" } }, + { name: "expanded", value: { value: "true" } }, + { name: "controls", value: { value: "products-panel" } }, + ], + }, + { + nodeId: "3", + parentId: "1", + role: { type: "role", value: "textbox" }, + name: { type: "computedString", value: "Card number" }, + backendDOMNodeId: 30, + properties: [{ name: "inputType", value: { value: "credit-card" } }], + }, + ]; + const scene = buildVomScene(axNodes, { + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + nodes: [ + { + backendNodeId: 10, + parentBackendNodeId: null, + tag: "body", + attrs: {}, + rect: { x: 0, y: 0, w: 1000, h: 800 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 20, + parentBackendNodeId: 10, + tag: "button", + attrs: {}, + rect: { x: 20, y: 20, w: 120, h: 40 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 25, + parentBackendNodeId: 10, + tag: "div", + attrs: { id: "products-panel" }, + rect: null, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + textContent: "Shoes Bags", + }, + { + backendNodeId: 30, + parentBackendNodeId: 10, + tag: "input", + attrs: {}, + rect: { x: 20, y: 80, w: 200, h: 40 }, + paintOrder: 2, + position: "static", + pointerEvents: "auto", + }, + ], + }); + + expect(scene.nodes.find((node) => node.id === 20)).toEqual( + expect.objectContaining({ + name: "Products [expanded]", + attrs: expect.objectContaining({ + "aria-expanded": "true", + "aria-controls": "products-panel", + }), + }), + ); + expect(scene.activeScopeBlocks).toEqual([ + { triggerId: 20, label: "Products [expanded]", lines: ["Shoes Bags"] }, + ]); + expect(scene.nodes.find((node) => node.id === 30)?.sensitive).toBe(true); + }); + + it("aggregates AX virtual text into unnamed structural nodes", () => { + const scene = buildVomScene( + [ + { + nodeId: "1", + role: { type: "role", value: "RootWebArea" }, + backendDOMNodeId: 10, + childIds: ["2"], + }, + { + nodeId: "2", + parentId: "1", + role: { type: "role", value: "paragraph" }, + backendDOMNodeId: 20, + childIds: ["3"], + }, + { + nodeId: "3", + parentId: "2", + role: { type: "role", value: "InlineTextBox" }, + name: { type: "computedString", value: "Inline only text" }, + }, + ], + { + viewport: { width: 1000, height: 800 }, + iframeNodes: new Map(), + excludedBackendNodeIds: new Set(), + nodes: [ + { + backendNodeId: 10, + parentBackendNodeId: null, + tag: "body", + attrs: {}, + rect: { x: 0, y: 0, w: 1000, h: 800 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + { + backendNodeId: 20, + parentBackendNodeId: 10, + tag: "p", + attrs: {}, + rect: { x: 20, y: 20, w: 400, h: 40 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + }, + ], + }, + ); + + expect(scene.nodes.find((node) => node.id === 20)?.name).toBe("Inline only text"); + }); + it("marks captured dialog elements as modal without AX role or aria-modal", () => { const scene = buildVomScene([], { viewport: { width: 640, height: 480 }, @@ -1533,7 +1947,8 @@ describe("handleSnapshot", () => { expect(res.text).toContain("L1 modal cover=100%"); // Iframe content must appear in L1 — both inputs rendered expect(res.text).toMatch(/textbox "请输入手机号"/); - expect(res.text).toMatch(/textbox "密码" ="•••"/); + expect(res.text).toMatch(/textbox "密码"/); + expect(res.text).not.toMatch(/textbox "密码" ="•••"/); // Ref-store must include the iframe's input backendNodeIds const ctx = sm["sessions"].get("aa11")!; const phoneRef = res.text.match(/@(e\d+) textbox "请输入手机号"/)?.[1]; diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index a50c25a..aeace1b 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -3,7 +3,13 @@ // tab (defaulting to the Agent Window's active tab when omitted) and // returns a payload that mirrors the bsk-protocol Rust structs. -import { renderVom, type VomNode, type VomScene } from "@browser-skill/vom"; +import { + type ActiveScopeBlock, + type CondSurface, + renderVom, + type VomNode, + type VomScene, +} from "@browser-skill/vom"; import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; import type { SessionManager } from "@/session-manager/manager"; import type { @@ -234,6 +240,7 @@ export interface CdpAxNode { name?: { type: string; value?: string }; description?: { value?: string }; value?: { value?: string | number | boolean }; + properties?: Array<{ name?: string; value?: { value?: string | number | boolean } }>; childIds?: string[]; } @@ -268,6 +275,120 @@ function isSensitive(capturedNode: CapturedNode | undefined): boolean { return (attrs.type ?? "").toLowerCase() === "password"; } +interface AxNodeSignals { + hasPopup: boolean; + expanded: boolean; + selected: boolean; + controls: string; + sensitive: boolean; + aggregatedText?: string; +} + +const AX_TEXT_AGGREGATE_ROLES = new Set([ + "paragraph", + "listitem", + "term", + "definition", + "cell", + "gridcell", + "caption", + "figcaption", + "blockquote", + "note", + "status", + "log", + "generic", + "section", +]); +const AX_TEXT_LEAF_ROLES = new Set(["inlinetextbox", "statictext", "text"]); +const AX_TEXT_STOP_ROLES = new Set([ + "button", + "link", + "combobox", + "listbox", + "menuitem", + "menuitemcheckbox", + "menuitemradio", + "option", + "radio", + "checkbox", + "textbox", + "searchbox", + "spinbutton", + "slider", + "switch", + "tab", + "treeitem", + "columnheader", + "rowheader", +]); +const SENSITIVE_AX_INPUT_TYPES = new Set(["password", "credit-card"]); + +function axPropertyString(axNode: CdpAxNode, name: string): string | undefined { + const prop = axNode.properties?.find((item) => item.name === name); + return axString(prop?.value); +} + +function buildAxSignals(axNodes: CdpAxNode[]): Map { + const axByNodeId = new Map(axNodes.map((node) => [node.nodeId, node])); + const virtualText = new Map(); + for (const node of axNodes) { + if (typeof node.backendDOMNodeId === "number") continue; + const role = axString(node.role)?.toLowerCase() ?? ""; + const name = axString(node.name); + if (name && AX_TEXT_LEAF_ROLES.has(role)) virtualText.set(node.nodeId, name); + } + + const collectLeafText = (nodeId: string, depth: number): string[] => { + if (depth > 8) return []; + const node = axByNodeId.get(nodeId); + if (!node) return []; + const parts: string[] = []; + for (const childId of node.childIds ?? []) { + const vtext = virtualText.get(childId); + if (vtext) { + parts.push(vtext); + continue; + } + const child = axByNodeId.get(childId); + if (!child) continue; + const childRole = axString(child.role)?.toLowerCase() ?? ""; + if (AX_TEXT_LEAF_ROLES.has(childRole)) continue; + if (AX_TEXT_STOP_ROLES.has(childRole)) continue; + const childName = axString(child.name); + if (childName && !AX_TEXT_AGGREGATE_ROLES.has(childRole)) { + parts.push(childName); + } else { + parts.push(...collectLeafText(childId, depth + 1)); + } + } + return parts; + }; + + const signals = new Map(); + for (const node of axNodes) { + const role = axString(node.role)?.toLowerCase() ?? ""; + const expanded = axPropertyString(node, "expanded") === "true"; + const selected = axPropertyString(node, "selected") === "true"; + const controls = axPropertyString(node, "controls") ?? ""; + const hasPopupValue = axPropertyString(node, "hasPopup") ?? ""; + const inputType = axPropertyString(node, "inputType") ?? ""; + let aggregatedText: string | undefined; + if (AX_TEXT_AGGREGATE_ROLES.has(role) && !axString(node.name)) { + aggregatedText = cleanAttr(collectLeafText(node.nodeId, 0).join(" ")); + } + signals.set(node.nodeId, { + hasPopup: hasPopupValue !== "" && hasPopupValue !== "false", + expanded, + selected, + controls, + sensitive: SENSITIVE_AX_INPUT_TYPES.has(inputType), + aggregatedText, + }); + } + return signals; +} + function findAxAncestor( axNode: CdpAxNode, axById: Map, @@ -331,6 +452,40 @@ function isRenderableIframeControl(node: CapturedNode): boolean { return tag !== "a" || name !== undefined; } +function normalizedControlType(node: VomNode | CapturedNode): string { + const attrs = node.attrs ?? {}; + const tag = normalizeTag(node.tag); + if (tag === "input") return (attrs.type ?? "text").toLowerCase(); + if ("sensitive" in node && node.sensitive) return "password"; + return tag; +} + +function sameLogicalIframeControl(existing: VomNode, candidate: CapturedNode, iframeId: number) { + const role = iframeRoleFor(candidate); + const name = cleanAttr(iframeNameFor(candidate)); + if (!role || !name) return false; + if ((existing.role ?? "").toLowerCase() !== role) return false; + if (cleanAttr(existing.name) !== name) return false; + + const candidateType = normalizedControlType(candidate); + const existingType = normalizedControlType(existing); + if (candidateType !== existingType) return false; + + return ( + existing.parentId === iframeId || + existing.domParentId === iframeId || + existing.domAncestorIds?.includes(iframeId) === true + ); +} + +function hasEquivalentIframeControl( + nodes: VomNode[], + candidate: CapturedNode, + iframeId: number, +): boolean { + return nodes.some((node) => sameLogicalIframeControl(node, candidate, iframeId)); +} + function capturedIframeNameFor(node: CapturedNode): string | undefined { const ariaLabel = node.attrs["aria-label"]?.replace(/\s+/g, " ").trim(); if (ariaLabel) return ariaLabel; @@ -350,6 +505,11 @@ function cleanAttr(value: string | undefined): string | undefined { const FORM_CONTROL_TAGS = new Set(["input", "textarea", "select"]); const NATIVE_CONTROL_TAGS = new Set(["button", "input", "select", "textarea"]); +const ACTIVE_SCOPE_MAX_BLOCKS = 8; +const ACTIVE_SCOPE_MAX_LINES = 40; +const ACTIVE_SCOPE_MAX_LINE_LENGTH = 160; +const ACTIVE_SCOPE_MAX_TOTAL_CHARS = 8_000; +const ACTIVE_SCOPE_SKIP_TAGS = new Set(["script", "style", "noscript", "template"]); function buildCapturedChildren(capturedNodes: CapturedNode[]): Map { const children = new Map(); @@ -427,6 +587,24 @@ function nearbyTextFor( return labels.length > 0 ? labels.join(" ") : undefined; } +function previousSiblingTextFor( + node: CapturedNode, + childrenByParentId: Map, +): string | undefined { + if (node.parentBackendNodeId === null) return undefined; + const siblings = childrenByParentId.get(node.parentBackendNodeId) ?? []; + const index = siblings.findIndex((sibling) => sibling.backendNodeId === node.backendNodeId); + if (index <= 0) return undefined; + + for (let i = index - 1; i >= Math.max(0, index - 4); i -= 1) { + const sibling = siblings[i]; + if (["input", "textarea", "select", "button", "a"].includes(normalizeTag(sibling.tag))) break; + const text = cleanAttr(sibling.textContent)?.replace(/[::]\s*$/, ""); + if (text && text.length <= 40) return text; + } + return undefined; +} + interface VomNodeDomSignals { capturedByBackendId: Map; childrenByParentId: Map; @@ -470,13 +648,18 @@ function formControlName( captured: CapturedNode | undefined, axName: string | undefined, axValue: string | undefined, + signals: VomNodeDomSignals, ): string | undefined { const placeholder = cleanAttr(captured?.attrs.placeholder); const ariaLabel = cleanAttr(captured?.attrs["aria-label"]); const title = cleanAttr(captured?.attrs.title); + const nearbyLabel = captured + ? previousSiblingTextFor(captured, signals.childrenByParentId) + : undefined; const hasValue = axValue !== undefined && axValue !== ""; + if (nearbyLabel) return nearbyLabel; if (!hasValue && placeholder) return placeholder; - return axName ?? ariaLabel ?? placeholder ?? title; + return axName ?? ariaLabel ?? placeholder ?? title ?? nearbyLabel; } function vomNodeFromCaptured( @@ -516,12 +699,17 @@ function axVomNode( capturedNode: CapturedNode | undefined, axById: Map, signals: VomNodeDomSignals, + axSignals: Map, ): VomNode { const role = axString(axNode.role); const value = axValue(axNode.value); - const name = FORM_CONTROL_TAGS.has(normalizeTag(capturedNode?.tag)) - ? formControlName(capturedNode, axString(axNode.name), value) - : axString(axNode.name); + const signalsForNode = axSignals.get(axNode.nodeId); + let name = FORM_CONTROL_TAGS.has(normalizeTag(capturedNode?.tag)) + ? formControlName(capturedNode, axString(axNode.name), value, signals) + : (axString(axNode.name) ?? signalsForNode?.aggregatedText); + if (name && signalsForNode?.hasPopup) { + name = signalsForNode.expanded ? `${name} [expanded]` : `${name} [has-submenu]`; + } const node: VomNode = { id: axNode.backendDOMNodeId as number, parentId: nearestBackendParent(axNode, axById), @@ -531,12 +719,22 @@ function axVomNode( position: capturedNode?.position || "static", pointerEvents: capturedNode?.pointerEvents || "auto", modal: isModalSignal(axNode, capturedNode), - sensitive: isSensitive(capturedNode), + sensitive: isSensitive(capturedNode) || signalsForNode?.sensitive === true, }; if (role) node.role = role; if (name) node.name = name; if (value !== undefined) node.value = value; - return applyCapturedSignals(node, capturedNode, signals); + const enriched = applyCapturedSignals(node, capturedNode, signals); + if (signalsForNode?.selected && !enriched.attrs?.["aria-selected"]) { + enriched.attrs = { ...(enriched.attrs ?? {}), "aria-selected": "true" }; + } + if (signalsForNode?.expanded && !enriched.attrs?.["aria-expanded"]) { + enriched.attrs = { ...(enriched.attrs ?? {}), "aria-expanded": "true" }; + } + if (signalsForNode?.controls && !enriched.attrs?.["aria-controls"]) { + enriched.attrs = { ...(enriched.attrs ?? {}), "aria-controls": signalsForNode.controls }; + } + return enriched; } function iframeSignals(iframeNodes: CapturedNode[]): VomNodeDomSignals { @@ -561,6 +759,144 @@ function capturedOnlySignals(capturedNodes: CapturedNode[]): VomNodeDomSignals { }; } +function normalizeProbeKey(value: string | undefined): string { + return cleanAttr(value)?.toLowerCase() ?? ""; +} + +function controlledIds(attrs: Record): string[] { + return (attrs["aria-controls"] ?? "").split(/\s+/).filter(Boolean); +} + +function isActiveScopeTrigger(node: VomNode): boolean { + const attrs = node.attrs ?? {}; + if (controlledIds(attrs).length === 0) return false; + const role = normalizeTag(attrs.role) || normalizedVomRole(node); + return ( + (role === "tab" && (attrs["aria-selected"] ?? "").toLowerCase() === "true") || + (attrs["aria-expanded"] ?? "").toLowerCase() === "true" + ); +} + +function normalizedVomRole(node: VomNode): string { + return node.role?.toLowerCase() ?? ""; +} + +function collectScopeLines( + root: CapturedNode, + triggerLabel: string, + childrenByParentId: Map, +): string[] { + const lines: string[] = []; + const seen = new Set(); + const triggerKey = normalizeProbeKey(triggerLabel); + const stack = [root]; + while (stack.length > 0 && lines.length < ACTIVE_SCOPE_MAX_LINES) { + const node = stack.shift() as CapturedNode; + const tag = normalizeTag(node.tag); + if (ACTIVE_SCOPE_SKIP_TAGS.has(tag)) continue; + if ((node.attrs.type ?? "").toLowerCase() === "password") continue; + + const text = cleanAttr(node.textContent); + const key = normalizeProbeKey(text); + if (text && key !== triggerKey && !seen.has(key)) { + seen.add(key); + lines.push( + text.length > ACTIVE_SCOPE_MAX_LINE_LENGTH + ? text.slice(0, ACTIVE_SCOPE_MAX_LINE_LENGTH) + : text, + ); + } + stack.push(...(childrenByParentId.get(node.backendNodeId) ?? [])); + } + return lines; +} + +function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): ActiveScopeBlock[] { + const panelByDomId = new Map(); + for (const capturedNode of signals.capturedByBackendId.values()) { + const domId = capturedNode.attrs.id; + if (domId) panelByDomId.set(domId, capturedNode); + } + + const blocks: ActiveScopeBlock[] = []; + let totalChars = 0; + for (const node of nodes) { + if (blocks.length >= ACTIVE_SCOPE_MAX_BLOCKS) break; + if (!isActiveScopeTrigger(node)) continue; + const label = + cleanAttr(node.name) ?? cleanAttr(node.text) ?? cleanAttr(node.attrs?.["aria-label"]); + if (!label) continue; + + const lines: string[] = []; + for (const controlId of controlledIds(node.attrs ?? {})) { + const panel = panelByDomId.get(controlId); + if (!panel) continue; + lines.push(...collectScopeLines(panel, label, signals.childrenByParentId)); + if (lines.length >= ACTIVE_SCOPE_MAX_LINES) break; + } + + const uniqueLines: string[] = []; + const seen = new Set(); + for (const line of lines) { + const key = normalizeProbeKey(line); + if (!key || seen.has(key)) continue; + seen.add(key); + uniqueLines.push(line); + totalChars += line.length; + if ( + uniqueLines.length >= ACTIVE_SCOPE_MAX_LINES || + totalChars >= ACTIVE_SCOPE_MAX_TOTAL_CHARS + ) { + break; + } + } + if (uniqueLines.length === 0) continue; + blocks.push({ triggerId: node.id, label, lines: uniqueLines }); + if (totalChars >= ACTIVE_SCOPE_MAX_TOTAL_CHARS) break; + } + return blocks; +} + +function labelForSurfaceMatch(node: VomNode): string { + return ( + cleanAttr(node.name) ?? + cleanAttr(node.text) ?? + cleanAttr(node.attrs?.["aria-label"]) ?? + cleanAttr(node.attrs?.title) ?? + "" + ); +} + +function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] { + const probes = captured.surfaceProbes ?? []; + if (probes.length === 0) return []; + + const surfaces: CondSurface[] = []; + const nodesWithLabels = nodes + .map((node) => ({ node, key: normalizeProbeKey(labelForSurfaceMatch(node)) })) + .filter((entry) => entry.key.length > 0); + const used = new Set(); + for (const probe of probes) { + const probeKey = normalizeProbeKey(probe.triggerLabel); + if (!probeKey || probe.subItems.length === 0) continue; + const match = + nodesWithLabels.find((entry) => entry.key === probeKey && !used.has(entry.node.id)) ?? + nodesWithLabels.find( + (entry) => + !used.has(entry.node.id) && + (probeKey.startsWith(entry.key) || entry.key.startsWith(probeKey)), + ); + if (!match) continue; + used.add(match.node.id); + surfaces.push({ + triggerId: match.node.id, + triggerAction: probe.triggerAction, + subItems: probe.subItems, + }); + } + return surfaces; +} + function axNodeInOverlaySubtree( axNode: CdpAxNode, axById: Map, @@ -585,6 +921,7 @@ function axNodeInOverlaySubtree( export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel): VomScene { const signals = capturedOnlySignals(captured.nodes); const { capturedByBackendId } = signals; + const axSignals = buildAxSignals(axNodes); const axById = new Map(); for (const node of axNodes) { @@ -601,7 +938,7 @@ export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel) const capturedNode = capturedByBackendId.get(axNode.backendDOMNodeId); const role = axString(axNode.role); - const vomNode = axVomNode(axNode, capturedNode, axById, signals); + const vomNode = axVomNode(axNode, capturedNode, axById, signals, axSignals); // For link nodes, attach external hostname so the renderer can annotate it. // Relative hrefs and same-origin hrefs are omitted to avoid noise. @@ -633,7 +970,8 @@ export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel) if ( seenBackendIds.has(iframeNode.backendNodeId) || excludedBackendNodeIds.has(iframeNode.backendNodeId) || - !isRenderableIframeControl(iframeNode) + !isRenderableIframeControl(iframeNode) || + hasEquivalentIframeControl(nodes, iframeNode, iframeBackendId) ) { continue; } @@ -651,7 +989,14 @@ export function buildVomScene(axNodes: CdpAxNode[], captured: CapturedViewModel) } } - return { viewport: captured.viewport, nodes }; + const activeScopeBlocks = buildActiveScopeBlocks(nodes, signals); + const surfaces = buildConditionalSurfaces(nodes, captured); + return { + viewport: captured.viewport, + nodes, + ...(surfaces.length > 0 ? { surfaces } : {}), + ...(activeScopeBlocks.length > 0 ? { activeScopeBlocks } : {}), + }; } export interface SnapshotDeps { @@ -825,7 +1170,11 @@ export async function handleSnapshot( const axNodes = result.nodes ?? []; const captured = await captureForVom(deps.cdp, target.tabId); const scene = buildVomScene(axNodes, captured); - const rendered = renderVom(scene, { maxDepth: params.max_depth, maxTokens: params.max_tokens }); + const rendered = renderVom(scene, { + maxDepth: params.max_depth, + maxTokens: params.max_tokens, + activeRegionPolicy: true, + }); ctx.refStore.replace( rendered.refs.map( (r) => [r.ref, { backendNodeId: r.backendNodeId, tabId: target.tabId }] as const, diff --git a/apps/extension/src/tools/vom/capture.ts b/apps/extension/src/tools/vom/capture.ts index 0b739b5..7b474f1 100644 --- a/apps/extension/src/tools/vom/capture.ts +++ b/apps/extension/src/tools/vom/capture.ts @@ -35,10 +35,17 @@ export interface CapturedNode { export type CapturedIframeNodes = Map; +export interface CapturedSurfaceProbe { + triggerLabel: string; + triggerAction: "hover" | "focus" | string; + subItems: string[]; +} + export interface CapturedViewModel { nodes: CapturedNode[]; viewport: Viewport; iframeNodes: CapturedIframeNodes; + surfaceProbes?: CapturedSurfaceProbe[]; /** Backend node ids belonging to the agent overlay host + its shadow subtree. */ excludedBackendNodeIds: Set; } @@ -88,6 +95,20 @@ interface LayoutMetricsReply { layoutViewport?: { clientWidth?: number; clientHeight?: number; pageX?: number; pageY?: number }; } +interface RuntimeEvaluateReply { + result?: { + value?: unknown; + }; +} + +interface HoverCandidate { + triggerSel: string; + affectedSub: string; + label: string; + x: number; + y: number; +} + interface CdpDomNode { backendNodeId?: number; children?: CdpDomNode[]; @@ -126,6 +147,160 @@ function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set(reply: RuntimeEvaluateReply): T | undefined { + return reply.result?.value as T | undefined; +} + +function hoverCssScanExpression(): string { + return `(() => { + const visibilityProps = ["display", "visibility", "opacity", "maxHeight", "height", "overflow"]; + const pairs = []; + const seenRules = new Set(); + for (const sheet of Array.from(document.styleSheets)) { + let rules; + try { rules = sheet.cssRules; } catch { continue; } + if (!rules) continue; + for (const rule of Array.from(rules)) { + if (rule.type !== 1 || !rule.selectorText || !rule.style) continue; + const selectorText = String(rule.selectorText); + if (!selectorText.includes(":hover")) continue; + if (!visibilityProps.some((prop) => rule.style[prop])) continue; + for (const rawPart of selectorText.split(",")) { + const part = rawPart.trim(); + const hoverIndex = part.indexOf(":hover"); + if (hoverIndex < 0) continue; + const triggerSel = part.slice(0, hoverIndex).trim(); + const affectedSub = part.slice(hoverIndex + 6).trim(); + if (!triggerSel) continue; + const key = triggerSel + "||" + affectedSub; + if (seenRules.has(key)) continue; + seenRules.add(key); + pairs.push({ triggerSel, affectedSub }); + } + } + } + + const candidates = []; + const seenLabels = new Set(); + for (const pair of pairs) { + let elements; + try { elements = Array.from(document.querySelectorAll(pair.triggerSel)); } catch { continue; } + for (const el of elements) { + if (!(el instanceof Element)) continue; + const rect = el.getBoundingClientRect(); + const style = getComputedStyle(el); + if (rect.width <= 0 || rect.height <= 0 || style.visibility === "hidden" || style.display === "none") continue; + const label = (el.textContent || "").replace(/\\s+/g, " ").trim(); + if (!label || seenLabels.has(label)) continue; + seenLabels.add(label); + candidates.push({ + triggerSel: pair.triggerSel, + affectedSub: pair.affectedSub, + label, + x: rect.left + rect.width / 2, + y: rect.top + rect.height / 2, + }); + if (candidates.length >= ${MAX_HOVER_TRIGGERS}) return candidates; + } + } + return candidates; + })()`; +} + +function hoverCollectExpression(candidate: HoverCandidate): string { + return `(() => { + const triggerSel = ${JSON.stringify(candidate.triggerSel)}; + const rawAffectedSub = ${JSON.stringify(candidate.affectedSub)}; + const x = ${JSON.stringify(candidate.x)}; + const y = ${JSON.stringify(candidate.y)}; + const hit = document.elementFromPoint(x, y); + const trigger = hit instanceof Element ? hit.closest(triggerSel) : null; + if (!trigger) return []; + + const affectedSub = rawAffectedSub.replace(/^[>+~]\\s*/, "").trim(); + let targets = []; + if (!affectedSub) { + targets = [trigger]; + } else { + try { targets = Array.from(trigger.querySelectorAll(affectedSub)); } catch { targets = []; } + } + const items = []; + const seen = new Set(); + const push = (value) => { + const text = String(value || "").replace(/\\s+/g, " ").trim(); + if (!text || seen.has(text)) return; + seen.add(text); + items.push(text); + }; + for (const target of targets) { + if (!(target instanceof Element)) continue; + const style = getComputedStyle(target); + if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0") continue; + const clickable = target.querySelectorAll('a, button, [role="menuitem"], [role="option"]'); + if (clickable.length > 0) { + for (const child of Array.from(clickable)) push(child.textContent); + } else { + push(target.textContent); + } + } + return items.slice(0, 12); + })()`; +} + +async function wait(ms: number): Promise { + await new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function probeHoverSurfaces(cdp: CdpRunner, tabId: number): Promise { + const started = Date.now(); + try { + const scan = await cdp.send(tabId, "Runtime.evaluate", { + expression: hoverCssScanExpression(), + returnByValue: true, + }); + const candidates = runtimeValue(scan) ?? []; + const results: CapturedSurfaceProbe[] = []; + const seen = new Set(); + for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) { + if (Date.now() - started > MAX_HOVER_PROBE_MS) break; + if (!candidate.label || seen.has(candidate.label)) continue; + try { + await cdp.send(tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: candidate.x, + y: candidate.y, + }); + await wait(HOVER_SETTLE_MS); + const collected = await cdp.send(tabId, "Runtime.evaluate", { + expression: hoverCollectExpression(candidate), + returnByValue: true, + }); + await cdp.send(tabId, "Input.dispatchMouseEvent", { type: "mouseMoved", x: 0, y: 0 }); + const subItems = (runtimeValue(collected) ?? []) + .map((item) => item.replace(/\s+/g, " ").trim()) + .filter(Boolean); + if (subItems.length === 0) continue; + seen.add(candidate.label); + results.push({ + triggerLabel: candidate.label, + triggerAction: "hover", + subItems, + }); + } catch { + continue; + } + } + return results; + } catch (err) { + console.debug("[bsk capture] hover surface probe failed", err); + return []; + } +} + /** * When DOMSnapshot is unavailable, locate the marked overlay host via CDP and * collect every backendNodeId in its pierced subtree (open shadow included). @@ -319,7 +494,13 @@ export async function captureViewModel(cdp: CdpRunner, tabId: number): Promise { ]), ); - expect(out.text).toContain('@e1 textbox "密码" ="•••"'); + expect(out.text).toContain('@e1 textbox "密码" [filled] ="•••"'); expect(out.text).not.toContain("hunter2"); }); - it("masks sensitive textboxes even when the value is missing", () => { + it("does not imply a sensitive textbox has a value when the value is missing", () => { const out = renderVom( scene([ node({ id: 1, role: "RootWebArea", name: "Login" }), @@ -154,7 +154,8 @@ describe("renderVom single-layer page", () => { ]), ); - expect(out.text).toContain('@e1 textbox "密码" ="•••"'); + expect(out.text).toContain('@e1 textbox "密码" [empty]'); + expect(out.text).not.toContain('="•••"'); }); it("continues rendering later siblings after a branch exceeds maxDepth", () => { @@ -278,6 +279,237 @@ describe("renderVom single-layer page", () => { expect(out.text).toContain('@e1 button "View" [ctx: Project Alpha]'); expect(out.text).toContain('@e2 button "View" [ctx: Project Beta]'); }); + + it("filters noisy template/layout text from weak label context", () => { + const out = renderVom( + scene([ + node({ id: 1, role: "RootWebArea", name: "Doc" }), + node({ + id: 2, + parentId: 1, + role: "section", + name: "top 开始 top 结束 bigpic、登录 开始 { + const out = renderVom( + scene([ + node({ id: 1, role: "RootWebArea", name: "Doc" }), + node({ + id: 2, + parentId: 1, + role: "button", + name: "Background", + tag: "button", + rect: { x: 120, y: 120, w: 120, h: 40 }, + paintOrder: 1, + }), + node({ + id: 3, + parentId: 1, + role: "generic", + tag: "div", + rect: { x: 100, y: 100, w: 180, h: 120 }, + paintOrder: 10, + position: "fixed", + }), + ]), + ); + + expect(out.text).toContain("@layers 1 focus=L1"); + expect(out.text).toContain('@e1 button "Background"'); + expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); + }); + + it("can filter refs blocked by active positioned regions without changing layer output", () => { + const out = renderVom( + scene([ + node({ id: 1, role: "RootWebArea", name: "Doc" }), + node({ + id: 2, + parentId: 1, + role: "button", + name: "Background", + tag: "button", + rect: { x: 120, y: 120, w: 120, h: 40 }, + paintOrder: 1, + }), + node({ + id: 3, + parentId: 1, + role: "generic", + tag: "div", + rect: { x: 100, y: 100, w: 180, h: 120 }, + paintOrder: 10, + position: "fixed", + }), + node({ + id: 4, + parentId: 1, + role: "button", + name: "Foreground", + tag: "button", + rect: { x: 320, y: 120, w: 120, h: 40 }, + paintOrder: 11, + }), + ]), + { activeRegionPolicy: true }, + ); + + expect(out.text).toContain("@layers 1 focus=L1"); + expect(out.text).toContain("L1 page"); + expect(out.text).not.toContain("active-region"); + expect(out.text).not.toContain("Background"); + expect(out.text).toContain('@e1 button "Foreground"'); + expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]); + }); + + it("renders conditional surface items inline on the trigger", () => { + const out = renderVom({ + ...scene([ + node({ id: 1, role: "RootWebArea", name: "Doc" }), + node({ id: 2, parentId: 1, role: "button", name: "Products", tag: "button" }), + ]), + surfaces: [ + { + triggerId: 2, + triggerAction: "hover", + subItems: ["Shoes", "Bags", "Accessories"], + }, + ], + }); + + expect(out.text).toContain('@e1 button "Products" [→ Shoes | Bags | Accessories]'); + expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); + }); + + it("injects active scope blocks immediately after their trigger without refs", () => { + const out = renderVom({ + ...scene([ + node({ id: 1, role: "RootWebArea", name: "Doc" }), + node({ id: 2, parentId: 1, role: "tab", name: "Reviews (12)", tag: "button" }), + ]), + activeScopeBlocks: [ + { + triggerId: 2, + label: "Reviews (12)", + lines: ["Jane - ear cups are small", "Bob - great sound"], + }, + ], + }); + + expect(out.text).toContain( + [ + ' @e1 tab "Reviews (12)"', + " [§ active: Reviews (12)]", + " Jane - ear cups are small", + " Bob - great sound", + ].join("\n"), + ); + expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); + }); + + it("does not render redundant children inside named ref controls", () => { + const out = renderVom( + scene([ + node({ id: 1, role: "RootWebArea", name: "Doc" }), + node({ id: 2, parentId: 1, role: "button", name: "Save", tag: "button" }), + node({ id: 3, parentId: 2, role: "img", name: "disk icon", tag: "svg" }), + ]), + ); + + expect(out.text).toContain('@e1 button "Save"'); + expect(out.text).not.toContain("disk icon"); + expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); + }); + + it("does not render redundant children inside native text inputs", () => { + const out = renderVom( + scene([ + node({ id: 1, role: "RootWebArea", name: "Doc" }), + node({ + id: 2, + parentId: 1, + role: "textbox", + name: "手机号", + value: "13311030827", + tag: "input", + }), + node({ id: 3, parentId: 2, role: "StaticText", name: "13311030827" }), + ]), + ); + + expect(out.text).toContain('@e1 textbox "手机号" [filled] ="13311030827"'); + expect(out.text).not.toContain('StaticText "13311030827"'); + }); + + it("adds semantic collection summaries for repeated item containers", () => { + const out = renderVom( + scene([ + node({ id: 1, role: "RootWebArea", name: "Forum" }), + node({ id: 2, parentId: 1, role: "main", name: "Posts" }), + node({ id: 3, parentId: 2, role: "searchbox", name: "Search posts", tag: "input" }), + node({ id: 10, parentId: 2, role: "article" }), + node({ id: 11, parentId: 10, role: "heading", name: "First post" }), + node({ id: 12, parentId: 10, role: "link", name: "First post", tag: "a" }), + node({ id: 13, parentId: 10, role: "button", name: "Upvote", tag: "button" }), + node({ id: 20, parentId: 2, role: "article" }), + node({ id: 21, parentId: 20, role: "heading", name: "Second post" }), + node({ id: 22, parentId: 20, role: "link", name: "Second post", tag: "a" }), + node({ id: 23, parentId: 20, role: "button", name: "Upvote", tag: "button" }), + ]), + { semanticCollections: true }, + ); + + expect(out.text).toContain("@collections"); + expect(out.text).toContain( + 'collection c1 role=main item_role=article label="First post" items=2', + ); + expect(out.text).toContain('controls: search=@e1 "Search posts"'); + expect(out.text).toContain('title="First post" title_handle=@e2'); + expect(out.text).toContain('actions=[@e3 "Upvote"]'); + }); + + it("does not summarize login forms as semantic collections", () => { + const out = renderVom( + scene([ + node({ id: 1, role: "RootWebArea", name: "北京市小客车指标调控管理信息系统" }), + node({ id: 2, parentId: 1, role: "list" }), + node({ id: 3, parentId: 2, role: "listitem", name: "用户登录" }), + node({ id: 4, parentId: 2, role: "listitem", name: "手机号:" }), + node({ + id: 5, + parentId: 4, + role: "textbox", + value: "13311030827", + tag: "input", + }), + node({ id: 6, parentId: 2, role: "listitem", name: "密 码:" }), + node({ + id: 7, + parentId: 6, + role: "textbox", + value: "secret", + tag: "input", + sensitive: true, + }), + node({ id: 8, parentId: 2, role: "listitem", name: "验证码:" }), + node({ id: 9, parentId: 8, role: "textbox", value: "h8Mh", tag: "input" }), + node({ id: 10, parentId: 2, role: "listitem" }), + node({ id: 11, parentId: 10, role: "button", name: "登录", tag: "button" }), + ]), + ); + + expect(out.text).not.toContain("@collections"); + expect(out.text).toContain('@e2 textbox [filled] ="•••"'); + }); }); describe("renderVom double-layer page", () => { @@ -331,7 +563,7 @@ describe("renderVom double-layer page", () => { "@layers 2 focus=L1", "L1 modal cover=100%", ' dialog "提示"', - ' @e1 textbox "密码" ="•••"', + ' @e1 textbox "密码" [filled] ="•••"', ' @e2 button "确认"', "L2 page … occluded by L1 (~2 nodes, not actionable)", ].join("\n"), @@ -385,8 +617,8 @@ describe("renderVom double-layer page", () => { expect(out.text).toContain("L1 modal cover=100%"); expect(out.text).toContain(' Iframe "登录框"'); - expect(out.text).toContain(' @e1 textbox "请输入手机号"'); - expect(out.text).toContain(' @e2 textbox "密码" ="•••"'); + expect(out.text).toContain(' @e1 textbox "请输入手机号" [empty]'); + expect(out.text).toContain(' @e2 textbox "密码" [filled] ="•••"'); expect(out.refs.map((r) => r.backendNodeId)).toEqual([101, 102]); }); }); diff --git a/packages/vom/src/index.ts b/packages/vom/src/index.ts index 837473f..45fe4ac 100644 --- a/packages/vom/src/index.ts +++ b/packages/vom/src/index.ts @@ -1,6 +1,8 @@ export { renderVom } from "./render"; export type { + ActiveScopeBlock, BlockingLayer, + CondSurface, LayerKind, Rect, Viewport, diff --git a/packages/vom/src/render.ts b/packages/vom/src/render.ts index 1701aec..8c81055 100644 --- a/packages/vom/src/render.ts +++ b/packages/vom/src/render.ts @@ -1,5 +1,14 @@ -import { detectBlockingLayer } from "./layers"; -import type { BlockingLayer, VomNode, VomOptions, VomResult, VomScene } from "./types"; +import { coverage, detectBlockingLayer } from "./layers"; +import type { + ActiveScopeBlock, + BlockingLayer, + CondSurface, + Rect, + VomNode, + VomOptions, + VomResult, + VomScene, +} from "./types"; const SKIP_ROLES = new Set(["generic", "none", "presentation", "inlinetextbox"]); @@ -72,6 +81,13 @@ const MAX_VALUE_LENGTH = 200; const MAX_CUSTOM_RECOVERY_AREA = 180_000; const MAX_RECOVERABLE_SINGLE_INTENT_AREA = 360_000; const MAX_HANDLE_CONTEXT_ITEMS = 3; +const MAX_SURFACE_ITEMS = 12; +const MAX_SCOPE_LINES = 40; +const MAX_SEMANTIC_COLLECTIONS = 3; +const MAX_SEMANTIC_ITEMS = 20; +const MAX_SEMANTIC_CONTROLS = 8; +const MAX_SEMANTIC_ACTIONS = 6; +const MAX_SEMANTIC_FIELD_CHARS = 120; const RECOVERY_HANDLER_ATTRS = ["onclick", "onmousedown", "onkeydown", "onkeyup", "onkeypress"]; const NATIVE_TAGS = new Set(["button", "input", "select", "textarea"]); @@ -105,6 +121,8 @@ const WEAK_ACTION_LABELS = new Set([ "continue", ]); +const ACTIVE_REGION_POSITIONS = new Set(["fixed", "absolute", "sticky"]); + function estimateTokens(text: string): number { return Math.ceil(text.length / 4); } @@ -114,6 +132,31 @@ function cleaned(value: string | undefined): string | undefined { return trimmed ? trimmed : undefined; } +function isNoisyText(value: string): boolean { + const clean = value.trim(); + const key = clean.toLowerCase(); + if (!clean) return true; + if (clean.includes("")) return true; + if (/<\/?[a-z][^>]*>/i.test(clean)) return true; + if (/\$\(function|document\.|window\.|\.onclick\s*=|function\s*\(|var\s+\w+\s*=/.test(clean)) { + return true; + } + if (/(https?:\/\/|encodeURIComponent|site_code|target="_blank")/.test(clean)) return true; + if (/^\[?if\s+[^\]]+\]?>?$/i.test(clean)) return true; + if (/^$/i.test(clean)) return true; + if (/^(top|header|footer|nav|bigpic|login|登录|开始|结束)[、,\s]*(开始|结束)?$/i.test(clean)) { + return true; + } + if (/(^|[、,\s])(top|bigpic|登录)\s*(开始|结束)($|[、,\s])/.test(key)) return true; + return false; +} + +function cleanSemanticText(value: string | undefined): string | undefined { + const clean = cleaned(value); + if (!clean || isNoisyText(clean)) return undefined; + return clean; +} + function normalizedRole(node: VomNode): string { return node.role?.toLowerCase() ?? ""; } @@ -275,6 +318,21 @@ function buildParentMap(nodes: VomNode[]): Map { return parentMap; } +function isAncestorOf( + parentMap: Map, + ancestorId: number, + nodeId: number, +): boolean { + let current = parentMap.get(nodeId) ?? null; + let guard = 0; + while (current !== null && guard <= parentMap.size) { + if (current === ancestorId) return true; + current = parentMap.get(current) ?? null; + guard += 1; + } + return false; +} + function applyRecovery(nodes: VomNode[]): VomNode[] { const parentMap = buildParentMap(nodes); const children = buildChildren(nodes); @@ -342,7 +400,7 @@ function duplicateReferenceNames(nodes: VomNode[]): Set { function needsHandleContext(nodeName: string, duplicateRefNames: Set): boolean { const key = normalizeContextKey(nodeName); if (!key) return false; - return duplicateRefNames.has(key) || WEAK_ACTION_LABELS.has(key); + return duplicateRefNames.has(key); } function contextLabel(node: VomNode, targetName: string): string | undefined { @@ -368,7 +426,7 @@ function weakLabelContext(node: VomNode, targetName: string): string | undefined } function weakContextText(label: string | undefined, targetName: string): string | undefined { - const clean = cleaned(label); + const clean = cleanSemanticText(label); if (!clean) return undefined; const key = normalizeContextKey(clean); if ( @@ -416,7 +474,7 @@ function dedupeContexts(contexts: ContextCandidate[]): string[] { ); const unique: string[] = []; for (const ctx of sorted) { - const clean = cleaned(ctx.text); + const clean = cleanSemanticText(ctx.text); if (!clean) continue; const key = normalizeContextKey(clean); const category = `${ctx.source}:${contextCategory(key)}`; @@ -537,6 +595,7 @@ function renderNodeLine( depth: number, ref: string | undefined, context: string[] = [], + surface: CondSurface | undefined = undefined, ): string { let line = `${" ".repeat(depth)}${ref ? `@${ref} ` : ""}${node.role}`; @@ -550,12 +609,41 @@ function renderNodeLine( line += ` [→ ${node.href}]`; } - const value = node.sensitive ? SENSITIVE_MASK : cleaned(node.value)?.slice(0, MAX_VALUE_LENGTH); + const rawValue = cleaned(node.value); + const role = normalizedRole(node); + if (["textbox", "searchbox"].includes(role)) { + line += rawValue === undefined ? " [empty]" : " [filled]"; + } + const value = node.sensitive + ? rawValue !== undefined + ? SENSITIVE_MASK + : undefined + : rawValue?.slice(0, MAX_VALUE_LENGTH); if (value !== undefined) line += ` =${JSON.stringify(value)}`; + if (surface && surface.subItems.length > 0) { + const items = surface.subItems.slice(0, MAX_SURFACE_ITEMS).join(" | "); + const suffix = surface.subItems.length > MAX_SURFACE_ITEMS ? " | …" : ""; + line += ` [→ ${items}${suffix}]`; + } + return line; } +function shouldSkipRedundantRefChildren(node: VomNode): boolean { + const role = normalizedRole(node); + if ( + ["textbox", "searchbox"].includes(role) && + ["input", "textarea"].includes((node.tag ?? "").toLowerCase()) + ) { + return true; + } + if (!cleaned(node.name)) return false; + return ["button", "link", "menuitem", "tab", "switch", "checkbox", "radio", "combobox"].includes( + role, + ); +} + interface RenderState { lines: string[]; refs: Array<{ ref: string; backendNodeId: number }>; @@ -569,6 +657,37 @@ interface RenderState { parentMap: Map; nodesById: Map; duplicateRefNames: Set; + surfaceMap: Map; + scopeMap: Map; +} + +function emitActiveScopeBlock(scope: ActiveScopeBlock, depth: number, state: RenderState): void { + if (state.stopped || scope.lines.length === 0) return; + const indent = " ".repeat(depth); + const header = `${indent}[§ active: ${scope.label}]`; + const headerTokens = estimateTokens(header); + if (state.tokens + headerTokens > state.maxTokens) { + state.truncated = true; + state.stopped = true; + return; + } + state.lines.push(header); + state.tokens += headerTokens; + + for (const text of scope.lines.slice(0, MAX_SCOPE_LINES)) { + if (state.stopped) return; + const clean = cleaned(text); + if (!clean) continue; + const line = `${indent} ${clean}`; + const nextTokens = state.tokens + estimateTokens(line); + if (nextTokens > state.maxTokens) { + state.truncated = true; + state.stopped = true; + return; + } + state.lines.push(line); + state.tokens = nextTokens; + } } function renderTree( @@ -595,7 +714,8 @@ function renderTree( const ref = shouldReference(node) ? `e${state.nextRef}` : undefined; const context = ref ? handleContext(node, state) : []; - const line = renderNodeLine(node, depth, ref, context); + const surface = state.surfaceMap.get(node.id); + const line = renderNodeLine(node, depth, ref, context, surface); const nextTokens = state.tokens + estimateTokens(line); if (nextTokens > state.maxTokens) { state.truncated = true; @@ -610,11 +730,21 @@ function renderTree( state.nextRef += 1; } + const scope = state.scopeMap.get(node.id); + if (scope) emitActiveScopeBlock(scope, depth + 1, state); + + if (ref && shouldSkipRedundantRefChildren(node)) continue; renderTree(children, node.id, depth + 1, state); } } -function renderNodes(nodes: VomNode[], options: VomOptions, initialLines: string[]): RenderState { +function renderNodes( + nodes: VomNode[], + options: VomOptions, + initialLines: string[], + surfaces: CondSurface[] = [], + activeScopeBlocks: ActiveScopeBlock[] = [], +): RenderState { const children = buildChildren(nodes); const state: RenderState = { lines: [...initialLines], @@ -629,6 +759,8 @@ function renderNodes(nodes: VomNode[], options: VomOptions, initialLines: string parentMap: buildParentMap(nodes), nodesById: new Map(nodes.map((node) => [node.id, node])), duplicateRefNames: duplicateReferenceNames(nodes), + surfaceMap: new Map(surfaces.map((surface) => [surface.triggerId, surface])), + scopeMap: new Map(activeScopeBlocks.map((scope) => [scope.triggerId, scope])), }; renderTree(children, null, 1, state); @@ -640,6 +772,122 @@ function renderPageOcclusionLine(hiddenNodeCount: number): string { return `L2 page … occluded by L1 (~${hiddenNodeCount} nodes, not actionable)`; } +function isPositionedRegionCandidate(node: VomNode): boolean { + return ( + node.pointerEvents !== "none" && + node.rect !== null && + ACTIVE_REGION_POSITIONS.has(node.position) + ); +} + +function rectContains(rect: Rect, x: number, y: number): boolean { + return x >= rect.x && x <= rect.x + rect.w && y >= rect.y && y <= rect.y + rect.h; +} + +function interactionPoints(rect: Rect): Array<[number, number]> { + const epsX = Math.min(rect.w * 0.15, 8); + const epsY = Math.min(rect.h * 0.15, 8); + const midX = rect.x + rect.w / 2; + const midY = rect.y + rect.h / 2; + return [ + [midX, midY], + [rect.x + epsX, rect.y + epsY], + [rect.x + rect.w - epsX, rect.y + epsY], + [rect.x + epsX, rect.y + rect.h - epsY], + [rect.x + rect.w - epsX, rect.y + rect.h - epsY], + [midX, rect.y + epsY], + [midX, rect.y + rect.h - epsY], + [rect.x + epsX, midY], + [rect.x + rect.w - epsX, midY], + ]; +} + +function isModalLike(node: VomNode): boolean { + const role = normalizedRole(node); + return ( + node.modal === true || node.tag === "dialog" || role === "dialog" || role === "alertdialog" + ); +} + +function activeRegionCandidatePriority(node: VomNode, viewportCoverage: number): number { + if (isModalLike(node)) return 4; + if (viewportCoverage >= 0.9) return 3; + if (viewportCoverage >= 0.15) return 2; + return 1; +} + +function isBlockedByRegion( + target: VomNode, + blocker: VomNode, + parentMap: Map, + viewportCoverage: number, +): boolean { + if (target.id === blocker.id) return false; + if (isAncestorOf(parentMap, blocker.id, target.id)) return false; + if (isAncestorOf(parentMap, target.id, blocker.id)) return false; + if (!target.rect || !blocker.rect) return false; + + const points = interactionPoints(target.rect); + + if ( + viewportCoverage < 0.9 && + target.paintOrder >= blocker.paintOrder && + points.some(([x, y]) => rectContains(blocker.rect as Rect, x, y)) + ) { + return false; + } + + if (blocker.paintOrder < target.paintOrder) return false; + if (blocker.paintOrder === target.paintOrder && blocker.id <= target.id) return false; + if (viewportCoverage >= 0.9) return true; + + return points.some(([x, y]) => rectContains(blocker.rect as Rect, x, y)); +} + +function collectDescendantsOfIds(nodes: VomNode[], roots: Set): Set { + const included = new Set(roots); + let changed = true; + while (changed) { + changed = false; + for (const node of nodes) { + if (included.has(node.id)) continue; + if (node.parentId !== null && included.has(node.parentId)) { + included.add(node.id); + changed = true; + } + } + } + return included; +} + +function applyActiveRegionPolicy(nodes: VomNode[], scene: VomScene): VomNode[] { + const parentMap = buildParentMap(nodes); + const candidates = nodes + .filter(isPositionedRegionCandidate) + .map((node) => { + const viewportCoverage = coverage(node.rect, scene.viewport); + return { + node, + viewportCoverage, + priority: activeRegionCandidatePriority(node, viewportCoverage), + }; + }) + .filter((candidate) => candidate.priority > 1 || candidate.viewportCoverage > 0); + + const blockedRoots = new Set(); + for (const target of nodes) { + if (!shouldReference(target)) continue; + const blocked = candidates.some((candidate) => + isBlockedByRegion(target, candidate.node, parentMap, candidate.viewportCoverage), + ); + if (blocked) blockedRoots.add(target.id); + } + + if (blockedRoots.size === 0) return nodes; + const blocked = collectDescendantsOfIds(nodes, blockedRoots); + return nodes.filter((node) => !blocked.has(node.id)); +} + function collectDescendantsOfMembers(nodes: VomNode[], members: Set): Set { const included = new Set(members); let changed = true; @@ -662,6 +910,386 @@ function countRenderable(nodes: VomNode[]): number { return nodes.filter(shouldRender).length; } +interface SemanticControl { + kind: string; + handle?: string; + label: string; +} + +interface SemanticAction { + handle: string; + label: string; +} + +interface SemanticItem { + title?: string; + titleHandle?: string; + author?: string; + time?: string; + score?: string; + comments?: string; + commentsHandle?: string; + actions: SemanticAction[]; +} + +interface SemanticCollection { + role: string; + itemRole: string; + label?: string; + controls: SemanticControl[]; + items: SemanticItem[]; +} + +function truncateField(value: string): string { + if ([...value].length <= MAX_SEMANTIC_FIELD_CHARS) return value; + return `${[...value].slice(0, MAX_SEMANTIC_FIELD_CHARS - 1).join("")}…`; +} + +function quoteField(value: string): string { + return JSON.stringify(truncateField(value)); +} + +function roleIsCollectionItem(role: string): boolean { + return role === "article" || role === "row" || role === "listitem"; +} + +function roleIsCollectionContainer(role: string): boolean { + return ["main", "feed", "list", "table", "grid", "rowgroup", "region", "section"].includes(role); +} + +function textForSemantics(node: VomNode): string | undefined { + return ( + cleanSemanticText(node.name) ?? + cleanSemanticText(node.value) ?? + cleanSemanticText(node.text) ?? + cleanSemanticText(node.nearbyText) + ); +} + +function buildRefLookup(refs: Array<{ ref: string; backendNodeId: number }>): Map { + return new Map( + refs.map((entry) => [ + entry.backendNodeId, + entry.ref.startsWith("@") ? entry.ref : `@${entry.ref}`, + ]), + ); +} + +function collectDescendantNodes( + children: Map, + rootId: number, + nodesById: Map, +): VomNode[] { + const out: VomNode[] = []; + const stack = [rootId]; + const seen = new Set(); + while (stack.length > 0) { + const id = stack.pop() as number; + if (!seen.add(id)) continue; + const node = nodesById.get(id); + if (node) out.push(node); + for (const child of [...(children.get(id) ?? [])].reverse()) { + stack.push(child.id); + } + } + return out; +} + +function collectDescendantIdSet( + children: Map, + rootId: number, +): Set { + const out = new Set(); + const stack = [rootId]; + while (stack.length > 0) { + const id = stack.pop() as number; + if (!out.add(id)) continue; + for (const child of [...(children.get(id) ?? [])].reverse()) stack.push(child.id); + } + return out; +} + +function collectionLabel( + node: VomNode, + children: Map, + nodesById: Map, +): string | undefined { + return ( + contextLabel(node, "") ?? + collectDescendantNodes(children, node.id, nodesById).find( + (candidate) => normalizedRole(candidate) === "heading" && textForSemantics(candidate), + )?.name + ); +} + +function classifyControl(label: string, role: string): string | undefined { + const key = label.toLowerCase(); + if (role === "searchbox" || key.includes("search")) return "search"; + if (key.includes("sort") || key.includes("order")) return "sort"; + if (key.includes("filter")) return "filter"; + if (role === "tab") return "tab"; + if (key === "next" || key.includes("next page")) return "pagination"; + if (key === "previous" || key.includes("previous page")) return "pagination"; + return undefined; +} + +function collectCollectionControls( + container: VomNode, + items: VomNode[], + children: Map, + nodesById: Map, + refLookup: Map, +): SemanticControl[] { + const itemIds = new Set(items.flatMap((item) => [...collectDescendantIdSet(children, item.id)])); + const controls: SemanticControl[] = []; + for (const node of collectDescendantNodes(children, container.id, nodesById)) { + if (itemIds.has(node.id)) continue; + const role = normalizedRole(node); + if (!["button", "combobox", "searchbox", "tab", "link"].includes(role)) continue; + const label = textForSemantics(node); + if (!label) continue; + const kind = classifyControl(label, role); + if (!kind) continue; + controls.push({ kind, handle: refLookup.get(node.id), label }); + if (controls.length >= MAX_SEMANTIC_CONTROLS) break; + } + return controls; +} + +function isFormLikeCollection( + container: VomNode, + items: VomNode[], + children: Map, + nodesById: Map, +): boolean { + const descendants = collectDescendantNodes(children, container.id, nodesById); + const interactive = descendants.filter((node) => shouldReference(node)); + const textboxCount = interactive.filter((node) => + ["textbox", "searchbox"].includes(normalizedRole(node)), + ).length; + const sensitiveCount = descendants.filter((node) => node.sensitive).length; + const labels = descendants + .map((node) => textForSemantics(node)) + .filter((label): label is string => label !== undefined) + .join(" ") + .toLowerCase(); + const hasLoginSignals = + /(用户登录|登录|验证码|密码|忘记密码|手机号|证书口令|captcha|password|sign in|log in)/i.test( + labels, + ); + + return ( + (textboxCount >= 2 && hasLoginSignals) || + (sensitiveCount > 0 && hasLoginSignals) || + (items.length <= 8 && textboxCount > 0 && interactive.length >= 3 && hasLoginSignals) + ); +} + +function isCommentLabel(label: string): boolean { + const key = label.toLowerCase(); + return key.includes("comment") || key === "no comments"; +} + +function parseScoreBetweenVoteButtons( + item: VomNode, + children: Map, +): string | undefined { + for (const descendantId of collectDescendantIdSet(children, item.id)) { + const siblings = children.get(descendantId) ?? []; + for (let index = 0; index + 2 < siblings.length; index += 1) { + const prev = textForSemantics(siblings[index])?.toLowerCase() ?? ""; + const score = textForSemantics(siblings[index + 1]); + const next = textForSemantics(siblings[index + 2])?.toLowerCase() ?? ""; + if ( + prev.includes("upvote") && + next.includes("downvote") && + score !== undefined && + /^-?\d+$/.test(score.replaceAll(",", "")) + ) { + return score; + } + } + } + return undefined; +} + +function extractAuthor(descendants: VomNode[]): string | undefined { + for (let index = 0; index < descendants.length; index += 1) { + const label = textForSemantics(descendants[index]); + if (!label) continue; + const key = label.toLowerCase(); + if (key === "submitted by" || key === "by" || key.endsWith(" by")) { + for (const candidate of descendants.slice(index + 1, index + 9)) { + if (normalizedRole(candidate) === "link") return textForSemantics(candidate); + } + } + } + return undefined; +} + +function extractSemanticItem( + item: VomNode, + children: Map, + nodesById: Map, + refLookup: Map, +): SemanticItem { + const descendants = collectDescendantNodes(children, item.id, nodesById); + let title: string | undefined; + let titleHandle: string | undefined; + for (const role of ["heading", "link"]) { + const node = descendants.find( + (candidate) => normalizedRole(candidate) === role && textForSemantics(candidate), + ); + if (!node) continue; + title = textForSemantics(node); + titleHandle = refLookup.get(node.id); + if (!titleHandle && title) { + const matchingLink = descendants.find( + (candidate) => + normalizedRole(candidate) === "link" && textForSemantics(candidate) === title, + ); + if (matchingLink) titleHandle = refLookup.get(matchingLink.id); + } + break; + } + if (!title) { + title = textForSemantics(item); + titleHandle = refLookup.get(item.id); + } + + let time: string | undefined; + let comments: string | undefined; + let commentsHandle: string | undefined; + const actions: SemanticAction[] = []; + for (const node of descendants) { + const role = normalizedRole(node); + const label = textForSemantics(node); + if (!time && role === "time") time = label; + if (!comments && label && isCommentLabel(label)) { + comments = label; + commentsHandle = refLookup.get(node.id); + } + if ( + actions.length < MAX_SEMANTIC_ACTIONS && + shouldReference(node) && + !["link", "textbox", "searchbox"].includes(role) + ) { + const handle = refLookup.get(node.id); + if (handle && label) actions.push({ handle, label }); + } + } + + return { + title, + titleHandle, + author: extractAuthor(descendants), + time, + score: parseScoreBetweenVoteButtons(item, children), + comments, + commentsHandle, + actions, + }; +} + +function detectSemanticCollections( + nodes: VomNode[], + refs: Array<{ ref: string; backendNodeId: number }>, +): SemanticCollection[] { + const children = buildChildren(nodes); + const nodesById = new Map(nodes.map((node) => [node.id, node])); + const refLookup = buildRefLookup(refs); + const collections: SemanticCollection[] = []; + const coveredItems = new Set(); + + for (const container of nodes) { + if (collections.length >= MAX_SEMANTIC_COLLECTIONS) break; + const role = normalizedRole(container); + if (!roleIsCollectionContainer(role)) continue; + const childList = children.get(container.id) ?? []; + const counts = new Map(); + for (const child of childList) { + const childRole = normalizedRole(child); + if (roleIsCollectionItem(childRole)) counts.set(childRole, (counts.get(childRole) ?? 0) + 1); + } + const best = [...counts].filter(([, count]) => count >= 2).sort((a, b) => b[1] - a[1])[0]; + if (!best) continue; + const [itemRole] = best; + const items = childList.filter( + (child) => normalizedRole(child) === itemRole && !coveredItems.has(child.id), + ); + if (items.length < 2) continue; + if (isFormLikeCollection(container, items, children, nodesById)) continue; + for (const item of items) coveredItems.add(item.id); + collections.push({ + role, + itemRole, + label: collectionLabel(container, children, nodesById), + controls: collectCollectionControls(container, items, children, nodesById, refLookup), + items: items + .slice(0, MAX_SEMANTIC_ITEMS) + .map((item) => extractSemanticItem(item, children, nodesById, refLookup)), + }); + } + return collections; +} + +function renderSemanticCollections(collections: SemanticCollection[]): string[] { + if (collections.length === 0) return []; + const lines = ["@collections"]; + for (const [index, collection] of collections.entries()) { + const label = collection.label ? ` label=${quoteField(collection.label)}` : ""; + lines.push( + `collection c${index + 1} role=${collection.role} item_role=${collection.itemRole}${label} items=${collection.items.length}`, + ); + if (collection.controls.length > 0) { + const rendered = collection.controls + .map( + (control) => + `${control.kind}=${control.handle ? `${control.handle} ` : ""}${quoteField(control.label)}`, + ) + .join("; "); + lines.push(` controls: ${rendered}`); + } + lines.push( + " columns: index title title_handle author time score comments comments_handle actions", + ); + collection.items.forEach((item, itemIndex) => { + const parts = [`${itemIndex + 1}. `]; + if (item.title) parts.push(`title=${quoteField(item.title)}`); + if (item.titleHandle) parts.push(`title_handle=${item.titleHandle}`); + if (item.author) parts.push(`author=${quoteField(item.author)}`); + if (item.time) parts.push(`time=${quoteField(item.time)}`); + if (item.score) parts.push(`score=${quoteField(item.score)}`); + if (item.comments) parts.push(`comments=${quoteField(item.comments)}`); + if (item.commentsHandle) parts.push(`comments_handle=${item.commentsHandle}`); + if (item.actions.length > 0) { + parts.push( + `actions=[${item.actions.map((action) => `${action.handle} ${quoteField(action.label)}`).join(", ")}]`, + ); + } + lines.push(` ${parts.join(" ")}`); + }); + } + return lines; +} + +function insertAfterHeader(text: string, insertLines: string[]): string { + if (insertLines.length === 0) return text; + const lines = text.split("\n"); + const l1Index = lines.findIndex((line) => line.startsWith("L1 ")); + const insertAt = l1Index >= 0 ? l1Index + 1 : Math.min(lines.length, 3); + lines.splice(insertAt, 0, ...insertLines); + return lines.join("\n"); +} + +function augmentVomResult(result: VomResult, nodes: VomNode[], options: VomOptions): VomResult { + if (!options.semanticCollections) return result; + const semanticLines = renderSemanticCollections(detectSemanticCollections(nodes, result.refs)); + return semanticLines.length === 0 + ? result + : { ...result, text: insertAfterHeader(result.text, semanticLines) }; +} + function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOptions): VomResult { const included = collectDescendantsOfMembers(scene.nodes, layer.members); const visibleNodes = scene.nodes.filter((node) => included.has(node.id)); @@ -672,14 +1300,18 @@ function renderDoubleLayer(scene: VomScene, layer: BlockingLayer, options: VomOp "@layers 2 focus=L1", `L1 ${layer.kind} cover=${Math.round(layer.coverage * 100)}%`, ]; - const state = renderNodes(visibleNodes, options, header); + const state = renderNodes(visibleNodes, options, header, scene.surfaces, scene.activeScopeBlocks); state.lines.push(renderPageOcclusionLine(hiddenCount)); - return { - text: state.lines.join("\n"), - refs: state.refs, - truncated: state.truncated, - }; + return augmentVomResult( + { + text: state.lines.join("\n"), + refs: state.refs, + truncated: state.truncated, + }, + visibleNodes, + options, + ); } export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult { @@ -687,17 +1319,28 @@ export function renderVom(scene: VomScene, options: VomOptions = {}): VomResult const renderScene = { ...scene, nodes }; const layer = detectBlockingLayer(nodes, scene.viewport); if (layer) return renderDoubleLayer(renderScene, layer, options); + const visibleNodes = options.activeRegionPolicy ? applyActiveRegionPolicy(nodes, scene) : nodes; - const state = renderNodes(nodes, options, [ - "@vom 1", - `@view ${scene.viewport.width}x${scene.viewport.height}`, - "@layers 1 focus=L1", - "L1 page", - ]); + const state = renderNodes( + visibleNodes, + options, + [ + "@vom 1", + `@view ${scene.viewport.width}x${scene.viewport.height}`, + "@layers 1 focus=L1", + "L1 page", + ], + scene.surfaces, + scene.activeScopeBlocks, + ); - return { - text: state.lines.join("\n"), - refs: state.refs, - truncated: state.truncated, - }; + return augmentVomResult( + { + text: state.lines.join("\n"), + refs: state.refs, + truncated: state.truncated, + }, + visibleNodes, + options, + ); } diff --git a/packages/vom/src/types.ts b/packages/vom/src/types.ts index 30d736e..47a0629 100644 --- a/packages/vom/src/types.ts +++ b/packages/vom/src/types.ts @@ -50,11 +50,37 @@ export interface VomNode { export interface VomScene { viewport: Viewport; nodes: VomNode[]; + surfaces?: CondSurface[]; + activeScopeBlocks?: ActiveScopeBlock[]; } export interface VomOptions { maxDepth?: number; maxTokens?: number; + /** + * Experimental: filter refs that are geometrically blocked by foreground + * fixed/absolute/sticky regions. Disabled by default and does not alter the + * public layer header format. + */ + activeRegionPolicy?: boolean; + /** + * Experimental: emit semantic summaries for repeated collection-like + * containers. Disabled by default because false positives in menus/forms are + * still noisy. + */ + semanticCollections?: boolean; +} + +export interface CondSurface { + triggerId: number; + triggerAction: "hover" | "focus" | string; + subItems: string[]; +} + +export interface ActiveScopeBlock { + triggerId: number; + label: string; + lines: string[]; } export interface VomResult {