diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index a560f3e..2e18a41 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -3155,6 +3155,98 @@ describe("handleSnapshot", () => { }); }); + it("decorates live observations with active controlled content", async () => { + const strings = [ + "html", + "body", + "button", + "div", + "#text", + "role", + "tab", + "aria-selected", + "true", + "aria-controls", + "reviews-panel", + "id", + "static", + "auto", + "visible", + "1", + "A detailed review", + ]; + const index = (value: string) => strings.indexOf(value); + const style = [index("static"), index("auto"), index("auto"), index("visible"), index("1")]; + const snapshot = { + strings, + documents: [ + { + nodes: { + parentIndex: [-1, 0, 1, 1, 3], + nodeName: [index("html"), index("body"), index("button"), index("div"), index("#text")], + backendNodeId: [10, 11, 12, 13, 14], + attributes: [ + [], + [], + [ + index("role"), + index("tab"), + index("aria-selected"), + index("true"), + index("aria-controls"), + index("reviews-panel"), + ], + [index("id"), index("reviews-panel")], + [], + ], + nodeValue: [-1, -1, -1, -1, index("A detailed review")], + }, + layout: { + nodeIndex: [0, 1, 2, 3, 4], + styles: [style, style, style, style, style], + bounds: [ + [0, 0, 1000, 800], + [0, 0, 1000, 800], + [20, 20, 120, 40], + [20, 80, 500, 200], + [20, 80, 200, 20], + ], + paintOrders: [0, 0, 1, 1, 1], + }, + }, + ], + }; + const ax: CdpAxNode[] = [ + { + nodeId: "root", + backendDOMNodeId: 11, + role: { type: "role", value: "RootWebArea" }, + childIds: ["reviews"], + }, + { + nodeId: "reviews", + parentId: "root", + backendDOMNodeId: 12, + role: { type: "role", value: "tab" }, + name: { type: "computedString", value: "Reviews" }, + properties: [ + { name: "selected", value: { value: true } }, + { name: "controls", value: { value: "reviews-panel" } }, + ], + }, + ]; + + const result = await captureVomObservation( + makeOverlayDeps(ax, snapshot, VP_METRICS).cdp, + 4, + "https://example.com", + ); + + expect(result.text).toContain('@e1 tab "Reviews"'); + expect(result.text).toContain("[§ active: Reviews]"); + expect(result.text).toContain("A detailed review"); + }); + it("does not leak form secrets through the record-safe observation payload", async () => { const secrets = { email: "user@example.com", diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index e7a7edd..ec78d18 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -817,6 +817,14 @@ export function buildFrameVomScene( excludedBackendNodeIds: captured.excludedBackendNodeIds, supplementalNames: options.supplementalNames, }); + return attachCapturedSceneAnnotations(scene, documents, captured); +} + +function attachCapturedSceneAnnotations( + scene: VomScene, + documents: VomFrameDocument[], + captured: CapturedViewModel, +): VomScene { const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId); const signals = capturedOnlySignals(rootDocument?.domNodes ?? captured.nodes); const activeScopeBlocks = buildActiveScopeBlocks(scene.nodes, signals); @@ -1055,7 +1063,8 @@ export async function captureVomObservation( resolveSemanticGraph(semanticGraph, { supplementalNames: tooltipNames }), ), ); - const rendered = renderVom(scene, { + const decoratedScene = attachCapturedSceneAnnotations(scene, normalizedDocuments, captured); + const rendered = renderVom(decoratedScene, { maxDepth: options.maxDepth, maxTokens: options.maxTokens, redactValues: options.redactValues, diff --git a/apps/extension/src/tools/vom/__tests__/capture.test.ts b/apps/extension/src/tools/vom/__tests__/capture.test.ts index de38ca5..9463383 100644 --- a/apps/extension/src/tools/vom/__tests__/capture.test.ts +++ b/apps/extension/src/tools/vom/__tests__/capture.test.ts @@ -36,7 +36,7 @@ function fakeSnapshotReply() { }, layout: { nodeIndex: [1, 2, 3], - // styles columns follow requested computedStyles order: [position, pointer-events] + // Legacy fixture: omitted style columns fall back to visible defaults. styles: [ [i("static"), i("auto")], [i("fixed"), i("auto")], @@ -682,6 +682,46 @@ describe("captureViewModel", () => { expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto"); }); + it("captures painted visibility as DOM fallback evidence", async () => { + const S = ["html", "body", "button", "static", "auto", "visible", "hidden", "1", "0"]; + const i = (value: string) => S.indexOf(value); + const snapshot = { + strings: S, + documents: [ + { + nodes: { + parentIndex: [-1, 0, 1, 1, 1], + nodeName: [i("html"), i("body"), i("button"), i("button"), i("button")], + backendNodeId: [10, 11, 12, 13, 14], + attributes: [[], [], [], [], []], + }, + layout: { + nodeIndex: [1, 2, 3, 4], + styles: [ + [i("static"), i("auto"), i("auto"), i("visible"), i("1")], + [i("static"), i("auto"), i("auto"), i("visible"), i("1")], + [i("static"), i("auto"), i("auto"), i("hidden"), i("1")], + [i("static"), i("auto"), i("auto"), i("visible"), i("0")], + ], + bounds: [ + [0, 0, 1000, 800], + [10, 10, 100, 30], + [10, 50, 100, 30], + [10, 90, 100, 30], + ], + paintOrders: [0, 1, 2, 3], + }, + }, + ], + }; + + const { nodes } = await captureViewModel(makeCdp(snapshot), 4); + + expect(nodes.find((node) => node.backendNodeId === 12)?.rendered).toBe(true); + expect(nodes.find((node) => node.backendNodeId === 13)?.rendered).toBe(false); + expect(nodes.find((node) => node.backendNodeId === 14)?.rendered).toBe(false); + }); + it("subtracts scroll offset to make rects viewport-relative", async () => { const cdp = { send: vi.fn(async (_t: number, method: string) => { diff --git a/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts b/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts index d4dda00..f26883a 100644 --- a/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts +++ b/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts @@ -271,6 +271,63 @@ describe("semantic VOM graph", () => { ]); }); + it("requires render evidence before promoting DOM fallback semantics", () => { + const hiddenOwner = { + ...dom(10, 1, "iframe", { hidden: "" }), + rect: null, + localRect: null, + rendered: false, + }; + const hiddenButton = { + ...dom(2, 1, "button", { "aria-label": "Hidden action" }), + rendered: false, + }; + const scene = buildSemanticVomScene({ + viewport: { width: 800, height: 600 }, + rootFrameId: "main", + documents: [ + document( + "main", + [ + { + nodeId: "root", + backendDOMNodeId: 1, + role: { type: "role", value: "RootWebArea" }, + }, + ], + [dom(1, null, "body"), hiddenButton, hiddenOwner], + ), + document("hidden-frame", [], [dom(20, null, "button", {}, "Frame action")], { + parentFrameId: "main", + ownerBackendNodeId: 10, + }), + ], + }); + + expect(scene.nodes.some((node) => node.backendNodeId === 2)).toBe(false); + expect(scene.nodes.some((node) => node.backendNodeId === 10)).toBe(false); + expect(scene.nodes.some((node) => node.frameId === "hidden-frame")).toBe(false); + expect(renderVom(scene).refs).toEqual([]); + }); + + it("retains rendered DOM fallback controls when AX semantics are unavailable", () => { + const scene = buildSemanticVomScene({ + viewport: { width: 800, height: 600 }, + rootFrameId: "main", + documents: [ + document( + "main", + [], + [dom(1, null, "body"), { ...dom(2, 1, "button", {}, "Save"), rendered: true }], + ), + ], + }); + + expect(renderVom(scene).refs).toEqual([ + expect.objectContaining({ backendNodeId: 2, role: "button", name: "Save" }), + ]); + }); + it("lets explicit DOM structure override ignored AX metadata", () => { const scene = buildSemanticVomScene({ viewport: { width: 800, height: 600 }, diff --git a/apps/extension/src/tools/vom/capture.ts b/apps/extension/src/tools/vom/capture.ts index c221d8f..1db0e29 100644 --- a/apps/extension/src/tools/vom/capture.ts +++ b/apps/extension/src/tools/vom/capture.ts @@ -1,8 +1,8 @@ // CDP capture adapter: DOMSnapshot.captureSnapshot + Page.getLayoutMetrics // → CapturedNode[] + Viewport. This is the ONLY VOM module that touches // raw CDP. The captureSnapshot reply is columnar (parallel arrays + a -// shared string table); we request exactly three computed styles so the -// `styles` columns are [position, pointer-events, cursor] in that order. +// shared string table); requested computed styles are decoded through +// STYLE_COL so capture and visibility policy share one explicit contract. import type { Rect, Viewport } from "@browser-skill/vom"; import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy"; @@ -11,7 +11,7 @@ import { childFrameProjection, type GeometryProjection, projectRectToViewport } import type { CdpRunner } from "../shared"; import { clearHover, waitForHover } from "./hover-perception"; -const REQUESTED_STYLES = ["position", "pointer-events", "cursor"] as const; +const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const; const STYLE_COL = Object.fromEntries( REQUESTED_STYLES.map((name, index) => [name, index]), ) as Record<(typeof REQUESTED_STYLES)[number], number>; @@ -39,6 +39,12 @@ export interface CapturedNode { * `textContent`: the live parser always sets it, hand-built fixtures may not. */ cursor?: string; + /** + * Whether the live DOM snapshot provides a painted, non-hidden box for this + * node. Semantic resolution uses this only for DOM fallback nodes; AX-backed + * nodes remain authoritative even when they are outside the viewport. + */ + rendered?: boolean; textContent?: string; formState?: "empty" | "filled" | "default"; formValue?: string; @@ -788,6 +794,8 @@ function parseDocumentNodes( const posCol = STYLE_COL.position; const peCol = STYLE_COL["pointer-events"]; const cursorCol = STYLE_COL.cursor; + const visibilityCol = STYLE_COL.visibility; + const opacityCol = STYLE_COL.opacity; const inputValues = sparseIndexMap(dn.inputValue); const textValues = sparseIndexMap(dn.textValue); const checkedInputs = new Set(dn.inputChecked?.index ?? []); @@ -832,6 +840,7 @@ function parseDocumentNodes( let position = "static"; let pointerEvents = "auto"; let cursor = "auto"; + let rendered = false; const li = layoutByNode.get(n); if (li !== undefined) { const b = dl?.bounds?.[li]; @@ -852,6 +861,13 @@ function parseDocumentNodes( position = str(strings, styleRow[posCol]) || "static"; pointerEvents = str(strings, styleRow[peCol]) || "auto"; cursor = str(strings, styleRow[cursorCol]) || "auto"; + const visibility = str(strings, styleRow[visibilityCol]) || "visible"; + const opacity = str(strings, styleRow[opacityCol]) || "1"; + rendered = + localRect !== null && + visibility !== "hidden" && + visibility !== "collapse" && + (Number.parseFloat(opacity) || 0) > 0; } // Skip non-element nodes (#text, #cdata-section, etc.) — they carry no @@ -887,6 +903,7 @@ function parseDocumentNodes( position, pointerEvents, cursor, + rendered, textContent, ...(formValue !== undefined ? { formValue } : {}), ...(tag === "input" || tag === "textarea" diff --git a/apps/extension/src/tools/vom/semantic-graph/structure.ts b/apps/extension/src/tools/vom/semantic-graph/structure.ts index 9b5b1a6..72df95f 100644 --- a/apps/extension/src/tools/vom/semantic-graph/structure.ts +++ b/apps/extension/src/tools/vom/semantic-graph/structure.ts @@ -49,6 +49,23 @@ function explicitStructural(node: ResolvedSemanticNode): boolean { ); } +function hasUsableAxSemantics(node: ResolvedSemanticNode): boolean { + return node.ax !== undefined && node.ax.ignored !== true; +} + +function isRenderedDomFallback(node: ResolvedSemanticNode): boolean { + if (!node.dom) return false; + const attrs = node.dom.attrs; + if ( + Object.prototype.hasOwnProperty.call(attrs, "hidden") || + Object.prototype.hasOwnProperty.call(attrs, "inert") || + (attrs["aria-hidden"] ?? "").toLowerCase() === "true" + ) { + return false; + } + return node.dom.rendered ?? (node.dom.localRect != null || node.dom.rect != null); +} + function initialParentId( graph: ResolvedSemanticGraph, node: ResolvedSemanticNode, @@ -142,6 +159,9 @@ function baseDecision( ): StructureDecision { if (node.excluded) return { kind: "excluded", reason: "excluded" }; if (redundant.has(node.id)) return { kind: "excluded", reason: "redundant-source" }; + if (!hasUsableAxSemantics(node) && !isRenderedDomFallback(node)) { + return { kind: "excluded", reason: "non-rendered-dom-fallback" }; + } if (frameOwnerIds.has(node.id)) return { kind: "keep", reason: "frame-owner" }; if (node.frameId !== graph.rootFrameId && ROOT_ROLES.has(output.role?.toLowerCase() ?? "")) { return { kind: "transparent", reason: "child-document-root" }; diff --git a/apps/extension/src/tools/vom/semantic-graph/types.ts b/apps/extension/src/tools/vom/semantic-graph/types.ts index bd26d76..b71b458 100644 --- a/apps/extension/src/tools/vom/semantic-graph/types.ts +++ b/apps/extension/src/tools/vom/semantic-graph/types.ts @@ -65,7 +65,10 @@ export type StructureDecision = | { kind: "keep"; reason: "semantic" | "frame-owner" } | { kind: "inferred-group"; reason: "independent-interaction-branches" } | { kind: "transparent"; reason: "wrapper" | "child-document-root" | "ax-ignored" } - | { kind: "excluded"; reason: "excluded" | "redundant-source" }; + | { + kind: "excluded"; + reason: "excluded" | "redundant-source" | "non-rendered-dom-fallback"; + }; export interface StructuredSemanticNode extends ResolvedSemanticNode { structure: StructureDecision;