diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index aa1a6a5..e7a7edd 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -54,12 +54,11 @@ import { collectOverlayExcludedBackendIds, } from "./vom/capture"; import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture"; +import { probeTooltipNames } from "./vom/name-enrichment"; import { type CaptureVomObservationResult, projectRecordSafeObservation, } from "./vom/record-safe-observation"; -import type { FrameDocument } from "./vom/frame-document"; -import { probeTooltipNames } from "./vom/name-enrichment"; import { buildSemanticGraph, buildSemanticVomScene, @@ -812,7 +811,7 @@ export function buildFrameVomScene( options: BuildVomSceneOptions = {}, ): VomScene { const scene = buildSemanticVomScene({ - documents: documents as FrameDocument[], + documents, viewport: captured.viewport, rootFrameId: captured.rootFrameId, excludedBackendNodeIds: captured.excludedBackendNodeIds, @@ -1038,7 +1037,7 @@ export async function captureVomObservation( ? legacyFrameDocuments(documents[0].axNodes, captured, url, documents[0].target) : documents; const semanticGraph = buildSemanticGraph({ - documents: normalizedDocuments as FrameDocument[], + documents: normalizedDocuments, viewport: captured.viewport, rootFrameId: captured.rootFrameId, excludedBackendNodeIds: captured.excludedBackendNodeIds, diff --git a/apps/extension/src/tools/vom/__tests__/name-enrichment.test.ts b/apps/extension/src/tools/vom/__tests__/name-enrichment.test.ts index 5165ff8..425517e 100644 --- a/apps/extension/src/tools/vom/__tests__/name-enrichment.test.ts +++ b/apps/extension/src/tools/vom/__tests__/name-enrichment.test.ts @@ -110,6 +110,28 @@ describe("tooltip name enrichment", () => { expect(cdp.sendToTarget).not.toHaveBeenCalled(); }); + it("does not collect supplemental names from form controls", async () => { + const cdp: CdpRunner = { + send: vi.fn(async () => ({})) as CdpRunner["send"], + sendToTarget: vi.fn(async () => ({ executionContextId: 12 })) as CdpRunner["sendToTarget"], + }; + const inputDocument = document({ tabId: 8 }, 602); + inputDocument.domNodes[0] = { + ...inputDocument.domNodes[0], + tag: "input", + attrs: { type: "email" }, + formValue: "user@example.com", + formDefaultValue: "", + formState: "filled", + }; + + const names = await probeTooltipNames(cdp, 8, [inputDocument], semantics([inputDocument])); + + expect(names.size).toBe(0); + expect(cdp.send).not.toHaveBeenCalled(); + expect(cdp.sendToTarget).not.toHaveBeenCalled(); + }); + it("keeps tooltip evidence isolated across sibling frame sessions", async () => { let hoveredX = -10; const cdp: CdpRunner = { diff --git a/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts b/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts index b5f95c1..d4dda00 100644 --- a/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts +++ b/apps/extension/src/tools/vom/__tests__/semantic-graph.test.ts @@ -381,6 +381,58 @@ describe("semantic VOM graph", () => { expect(group?.name).toBeUndefined(); expect(group && isVomReferenceNode(group)).toBe(false); expect(buttons.map((node) => node.parentId)).toEqual([group?.id, group?.id]); + + const rendered = renderVom(scene); + const lines = rendered.text.split("\n"); + const groupLine = lines.findIndex((line) => line.trim() === "group"); + expect(groupLine).toBeGreaterThanOrEqual(0); + for (const ref of rendered.refs) { + expect(ref.line).toBeGreaterThan(groupLine); + expect(lines[ref.line]).toMatch(/^\s+@e\d+ button/); + } + }); + + it("never promotes a form value into its semantic name through supplemental evidence", () => { + const secret = "user@example.com"; + const input = { + ...dom(2, 1, "input", { type: "email" }), + formValue: secret, + formDefaultValue: "", + formState: "filled" as const, + }; + const scene = buildSemanticVomScene({ + viewport: { width: 800, height: 600 }, + rootFrameId: "main", + supplementalNames: new Map([["main\u00002", secret]]), + identifierFallback: false, + documents: [ + document( + "main", + [ + { + nodeId: "root", + backendDOMNodeId: 1, + role: { type: "role", value: "RootWebArea" }, + childIds: ["email"], + }, + { + nodeId: "email", + parentId: "root", + backendDOMNodeId: 2, + role: { type: "role", value: "textbox" }, + value: { value: secret }, + }, + ], + [dom(1, null, "body"), input], + ), + ], + }); + + const field = scene.nodes.find((node) => node.backendNodeId === 2); + expect(field?.name).toBeUndefined(); + const rendered = renderVom(scene, { redactValues: true }); + expect(rendered.text).toContain("•••"); + expect(JSON.stringify(rendered)).not.toContain(secret); }); it("keeps uncertain single-branch wrappers flat", () => { diff --git a/apps/extension/src/tools/vom/name-enrichment.ts b/apps/extension/src/tools/vom/name-enrichment.ts index c94eaf2..1f42ee3 100644 --- a/apps/extension/src/tools/vom/name-enrichment.ts +++ b/apps/extension/src/tools/vom/name-enrichment.ts @@ -10,6 +10,7 @@ const MAX_TOOLTIP_PROBE_MS = 2_400; const TOOLTIP_SETTLE_MS = 250; const CACHE_TTL_MS = 5 * 60_000; const MAX_CACHE_ENTRIES = 500; +const FORM_CONTROL_TAGS = new Set(["input", "select", "textarea"]); interface TooltipCacheEntry { name: string; @@ -108,6 +109,7 @@ function candidates( for (const node of graph.nodes.values()) { if ( node.vom.name || + FORM_CONTROL_TAGS.has(node.dom?.tag.toLowerCase() ?? "") || !node.referenceable || node.backendNodeId === undefined || !isVomReferenceNode({ diff --git a/apps/extension/src/tools/vom/semantic-graph/resolve.ts b/apps/extension/src/tools/vom/semantic-graph/resolve.ts index 03281cb..ad5b2cb 100644 --- a/apps/extension/src/tools/vom/semantic-graph/resolve.ts +++ b/apps/extension/src/tools/vom/semantic-graph/resolve.ts @@ -399,7 +399,7 @@ function resolvedName( formControl ? precedingAxText(node, graph, indexes) : undefined, formControl ? precedingDomText(node, graph, indexes) : undefined, formControl ? clean(node.dom?.formPlaceholder ?? attrs.placeholder) : undefined, - !interactive || node.backendNodeId === undefined + formControl || !interactive || node.backendNodeId === undefined ? undefined : clean(supplementalNames.get(frameBackendKey(node.frameId, node.backendNodeId))), identifierFallback && interactive ? stableIdentifierName(attrs) : undefined,