From 45a246c41b43364440398ebb51bae6c42beb41ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chaonan=E2=80=9D?= Date: Mon, 17 Aug 2026 17:47:37 +0800 Subject: [PATCH] feat(vom): expose record-safe semantic observations Share the VOM capture pipeline with recording, enrich rendered refs, and support value redaction without regressing cooperative cancellation. --- .../src/tools/__tests__/observation.test.ts | 48 ++++++- .../src/tools/capture-vom-observation.ts | 5 + apps/extension/src/tools/observation.ts | 127 ++++++++++++------ packages/vom/src/__tests__/render.test.ts | 64 +++++++-- packages/vom/src/index.ts | 1 + packages/vom/src/render.ts | 27 ++-- packages/vom/src/types.ts | 14 +- 7 files changed, 217 insertions(+), 69 deletions(-) create mode 100644 apps/extension/src/tools/capture-vom-observation.ts diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index a849843..af117f5 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -8,6 +8,7 @@ import { buildFrameVomScene, buildVomScene, type CdpAxNode, + captureVomObservation, handleGetHtml, handleObserve, handleScreenshot, @@ -1329,7 +1330,7 @@ describe("buildVomScene", () => { expect(rendered).toContain(' @e2 textbox "验证码"'); }); - it("does not mark non-password inputs sensitive from password-like labels", () => { + it("marks current-password autocomplete as sensitive even for text inputs", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", @@ -1364,7 +1365,7 @@ describe("buildVomScene", () => { expect(buildVomScene(axNodes, captured).nodes[0]).toEqual( expect.objectContaining({ id: 10, - sensitive: false, + sensitive: true, }), ); }); @@ -1659,7 +1660,9 @@ describe("buildVomScene", () => { ); const rendered = renderVom(scene); expect(rendered.text).toContain('@e1 button "close"'); - expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]); + expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([ + { ref: "e1", backendNodeId: 20 }, + ]); }); it("does not promote a clickable container that wraps a real interactive control", () => { @@ -1803,7 +1806,9 @@ describe("buildVomScene", () => { expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic"); const rendered = renderVom(scene); expect(rendered.text).toContain('@e1 button "收藏"'); - expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]); + expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([ + { ref: "e1", backendNodeId: 20 }, + ]); }); it("builds active scope blocks from active aria-controls relationships", () => { @@ -2908,6 +2913,41 @@ describe("handleSnapshot", () => { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; + it("exposes the frame-aware observation result for recording", async () => { + const ax: CdpAxNode[] = [ + { + nodeId: "root", + role: { type: "role", value: "RootWebArea" }, + backendDOMNodeId: 11, + childIds: ["password"], + }, + { + nodeId: "password", + parentId: "root", + role: { type: "role", value: "textbox" }, + name: { type: "computedString", value: "Password" }, + value: { value: "hunter2" }, + backendDOMNodeId: 13, + }, + ]; + const deps = makeOverlayDeps(ax, loginSnapshotReply(), VP_METRICS); + + const result = await captureVomObservation(deps.cdp, 4, "https://example.com", { + redactValues: true, + }); + + expect(result.text).not.toContain("hunter2"); + expect(result.text).toContain("•••"); + expect(result.refs[0]).toMatchObject({ + backendNodeId: 13, + role: "textbox", + name: "Password", + line: expect.any(Number), + }); + expect(result.captured.some((node) => node.backendNodeId === 13)).toBe(true); + expect(result.documents).toHaveLength(1); + }); + it("renders a blocking login overlay as the focused top layer", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); await sm.start("aa11"); diff --git a/apps/extension/src/tools/capture-vom-observation.ts b/apps/extension/src/tools/capture-vom-observation.ts new file mode 100644 index 0000000..10b6d69 --- /dev/null +++ b/apps/extension/src/tools/capture-vom-observation.ts @@ -0,0 +1,5 @@ +export { + type CaptureVomObservationOptions, + type CaptureVomObservationResult, + captureVomObservation, +} from "./observation"; diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index 406a821..d4fe91b 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -8,8 +8,10 @@ import { applyVomInteractionRecovery, type CondSurface, isVomReferenceNode, + type RenderedRef, renderVom, type VomNode, + type VomOptions, type VomScene, } from "@browser-skill/vom"; import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; @@ -48,6 +50,7 @@ import { import { resolveSnapshotRef } from "./snapshot-ref"; import { type CapturedNode, + type CapturedSurfaceProbe, type CapturedViewModel, captureViewModel, collectOverlayExcludedBackendIds, @@ -411,7 +414,15 @@ function isModalSignal( function isSensitive(capturedNode: CapturedNode | undefined): boolean { const attrs = capturedNode?.attrs ?? {}; - return (attrs.type ?? "").toLowerCase() === "password"; + const type = (attrs.type ?? "").toLowerCase(); + if (type === "password") return true; + const autocomplete = (attrs.autocomplete ?? "").toLowerCase(); + return ( + autocomplete.startsWith("cc-") || + autocomplete === "one-time-code" || + autocomplete === "current-password" || + autocomplete === "new-password" + ); } interface AxNodeSignals { @@ -461,7 +472,13 @@ const AX_TEXT_STOP_ROLES = new Set([ "columnheader", "rowheader", ]); -const SENSITIVE_AX_INPUT_TYPES = new Set(["password", "credit-card"]); +const SENSITIVE_AX_INPUT_TYPES = new Set([ + "password", + "credit-card", + "one-time-code", + "current-password", + "new-password", +]); function axPropertyString(axNode: CdpAxNode, name: string): string | undefined { const prop = axNode.properties?.find((item) => item.name === name); @@ -1537,14 +1554,13 @@ async function fallbackCapturedViewModel( try { const metrics = await cdp.send(tabId, "Page.getLayoutMetrics", {}); throwIfAborted(signal, "observation"); - const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {}; + const source = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {}; viewport = { - width: vpSrc.clientWidth ?? 0, - height: vpSrc.clientHeight ?? 0, + width: source.clientWidth ?? 0, + height: source.clientHeight ?? 0, }; - } catch (err) { - if (isAbortError(err)) throw err; - // viewport stays zero-sized + } catch (error) { + if (isAbortError(error)) throw error; } const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal); throwIfAborted(signal, "observation"); @@ -1554,22 +1570,69 @@ async function fallbackCapturedViewModel( async function captureForVom( cdp: CdpRunner, tabId: number, - conditionalSurfaceProbe: boolean, - hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise, - signal?: AbortSignal, + options: CaptureVomObservationOptions, ): Promise { try { return await captureViewModel(cdp, tabId, { - conditionalSurfaceProbe, - hoverProbeBypassOverlay, - signal, + conditionalSurfaceProbe: options.conditionalSurfaceProbe ?? false, + hoverProbeBypassOverlay: options.hoverProbeBypassOverlay, + signal: options.signal, }); - } catch (err) { - if (isAbortError(err)) throw err; - return fallbackCapturedViewModel(cdp, tabId, signal); + } catch (error) { + if (isAbortError(error)) throw error; + return fallbackCapturedViewModel(cdp, tabId, options.signal); } } +export interface CaptureVomObservationOptions extends VomOptions { + conditionalSurfaceProbe?: boolean; + hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise; + signal?: AbortSignal; +} + +export interface CaptureVomObservationResult { + text: string; + refs: RenderedRef[]; + truncated: boolean; + captured: CapturedNode[]; + documents: CapturedFrameDocument[]; + surfaceProbes?: CapturedSurfaceProbe[]; +} + +export async function captureVomObservation( + cdp: CdpRunner, + tabId: number, + url: string | undefined, + options: CaptureVomObservationOptions = {}, +): Promise { + throwIfAborted(options.signal, "observation"); + await cdp.ensureAttachedToUrl?.(tabId, url); + throwIfAborted(options.signal, "observation"); + const captured = await captureForVom(cdp, tabId, options); + throwIfAborted(options.signal, "observation"); + const documents = await captureFrameData(cdp, tabId, captured, options.signal); + throwIfAborted(options.signal, "observation"); + const scene = + captured.rootFrameId && captured.frameNodes?.size + ? buildFrameVomScene(documents, captured, { pageUrl: url }) + : buildVomScene(documents[0]?.axNodes ?? [], captured, { pageUrl: url }); + const rendered = renderVom(scene, { + maxDepth: options.maxDepth, + maxTokens: options.maxTokens, + redactValues: options.redactValues, + activeRegionPolicy: options.activeRegionPolicy, + }); + throwIfAborted(options.signal, "observation"); + return { + text: rendered.text, + refs: rendered.refs, + truncated: rendered.truncated, + captured: captured.nodes, + documents, + surfaceProbes: captured.surfaceProbes, + }; +} + async function handleVomObservation( manager: SessionManager, params: SnapshotParams | ObserveParams, @@ -1599,37 +1662,19 @@ async function handleVomObservation( try { throwIfAborted(signal, toolName); deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); - await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url); - throwIfAborted(signal, toolName); const effectiveConditionalSurfaceProbe = deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe; - const captured = await captureForVom( - deps.cdp, - target.tabId, - effectiveConditionalSurfaceProbe, - deps.hoverProbeBypassOverlay, - signal, - ); - throwIfAborted(signal, toolName); - const frameDocuments = await captureFrameData( - deps.cdp, - target.tabId, - captured, - signal, - ); - throwIfAborted(signal, toolName); - const scene = - captured.rootFrameId && captured.frameNodes?.size - ? buildFrameVomScene(frameDocuments, captured, { pageUrl: target.url }) - : buildVomScene(frameDocuments[0]?.axNodes ?? [], captured, { pageUrl: target.url }); - const rendered = renderVom(scene, { + const rendered = await captureVomObservation(deps.cdp, target.tabId, target.url, { maxDepth: params.max_depth, maxTokens: params.max_tokens, activeRegionPolicy: true, + conditionalSurfaceProbe: effectiveConditionalSurfaceProbe, + hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay, + signal, }); throwIfAborted(signal, toolName); const targetByFrameId = new Map( - frameDocuments.map((document) => [document.frameId, document.target]), + rendered.documents.map((document) => [document.frameId, document.target]), ); ctx.refStore.replace( rendered.refs.map((ref) => { @@ -1653,7 +1698,7 @@ async function handleVomObservation( ...(toolName === "observe" && (params as ObserveParams).debug_surfaces ? { debug: { - surface_probes: (captured.surfaceProbes ?? []).map((probe) => ({ + surface_probes: (rendered.surfaceProbes ?? []).map((probe) => ({ trigger_backend_node_id: probe.triggerBackendNodeId, ...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}), trigger_action: probe.triggerAction, diff --git a/packages/vom/src/__tests__/render.test.ts b/packages/vom/src/__tests__/render.test.ts index d757c3b..d8a3d04 100644 --- a/packages/vom/src/__tests__/render.test.ts +++ b/packages/vom/src/__tests__/render.test.ts @@ -24,6 +24,10 @@ function scene(nodes: VomNode[]): VomScene { }; } +function coreRefs(refs: ReturnType["refs"]) { + return refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId })); +} + describe("renderVom single-layer page", () => { it("does not derive handle context across frame scopes", () => { const out = renderVom( @@ -130,7 +134,7 @@ describe("renderVom single-layer page", () => { ' @e2 button "加入购物车"', ].join("\n"), ); - expect(out.refs).toEqual([ + expect(coreRefs(out.refs)).toEqual([ { ref: "e1", backendNodeId: 3 }, { ref: "e2", backendNodeId: 6 }, ]); @@ -151,7 +155,7 @@ describe("renderVom single-layer page", () => { expect(out.text).toContain(' heading "Title"'); expect(out.text).toContain(' img "Logo"'); expect(out.text).toContain(' @e1 button "Continue"'); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]); }); it("assigns refs to listbox controls", () => { @@ -163,7 +167,7 @@ describe("renderVom single-layer page", () => { ); expect(out.text).toContain(' @e1 listbox "Choices"'); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]); }); it("annotates external links regardless of role casing", () => { @@ -200,7 +204,7 @@ describe("renderVom single-layer page", () => { expect(out.text).toContain(" alertdialog"); expect(out.text).toContain(" section"); expect(out.text).toContain(' @e1 button "Close"'); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 5 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 5 }]); }); it("skips generic nodes without adding an extra indentation level", () => { @@ -291,7 +295,7 @@ describe("renderVom single-layer page", () => { expect(out.text).toContain(" section"); expect(out.text).not.toContain("Too deep"); expect(out.text).toContain(' @e1 button "Later"'); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]); }); it("truncates when maxTokens is exceeded while keeping refs in sync with rendered text", () => { @@ -326,7 +330,7 @@ describe("renderVom single-layer page", () => { ); expect(out.text).toContain('@e1 button "Open settings"'); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]); }); it("keeps the deepest custom control when wrapper and child both look clickable", () => { @@ -356,7 +360,7 @@ describe("renderVom single-layer page", () => { expect(out.text).not.toContain('@e1 button "Card"'); expect(out.text).toContain('@e1 button "Details"'); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]); }); it("does not recover clickable wrappers around native controls", () => { @@ -379,7 +383,7 @@ describe("renderVom single-layer page", () => { expect(out.text).not.toContain('@e1 button "Checkout"'); expect(out.text).toContain('@e1 button "Pay"'); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]); }); it("adds context to duplicate weak action labels", () => { @@ -442,7 +446,7 @@ describe("renderVom single-layer page", () => { expect(out.text).toContain("@layers 1 focus=L1"); expect(out.text).toContain('@e1 button "Background"'); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]); }); it("can filter refs blocked by active positioned regions without changing layer output", () => { @@ -485,7 +489,7 @@ describe("renderVom single-layer page", () => { expect(out.text).not.toContain("active-region"); expect(out.text).not.toContain("Background"); expect(out.text).toContain('@e1 button "Foreground"'); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]); }); it("keeps paint-order comparisons inside their frame document", () => { @@ -600,7 +604,7 @@ describe("renderVom single-layer page", () => { }); expect(out.text).toContain('@e1 button "Products" [hover first: Shoes | Bags | Accessories]'); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]); }); it("injects active scope blocks immediately after their trigger without refs", () => { @@ -626,7 +630,7 @@ describe("renderVom single-layer page", () => { " Bob - great sound", ].join("\n"), ); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]); }); it("does not render redundant children inside named ref controls", () => { @@ -640,7 +644,7 @@ describe("renderVom single-layer page", () => { expect(out.text).toContain('@e1 button "Save"'); expect(out.text).not.toContain("disk icon"); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]); }); it("does not render redundant children inside native text inputs", () => { @@ -807,7 +811,7 @@ describe("renderVom double-layer page", () => { ].join("\n"), ); expect(out.text).not.toContain("底层按钮"); - expect(out.refs).toEqual([ + expect(coreRefs(out.refs)).toEqual([ { ref: "e1", backendNodeId: 4 }, { ref: "e2", backendNodeId: 5 }, ]); @@ -859,4 +863,36 @@ describe("renderVom double-layer page", () => { expect(out.text).toContain(' @e2 textbox "密码" [filled] ="•••"'); expect(out.refs.map((r) => r.backendNodeId)).toEqual([101, 102]); }); + + it("redacts form values and returns semantic metadata for rendered refs", () => { + const out = renderVom( + scene([ + node({ id: 1, role: "RootWebArea", name: "Form" }), + node({ + id: 2, + backendNodeId: 42, + frameId: "child-frame", + parentId: 1, + role: "textbox", + name: "Email", + value: "user@example.com", + inputState: "filled", + tag: "input", + }), + ]), + { redactValues: true }, + ); + + expect(out.text).toContain('[filled] ="•••"'); + expect(out.text).not.toContain("user@example.com"); + expect(out.refs[0]).toMatchObject({ + ref: "e1", + backendNodeId: 42, + frameId: "child-frame", + role: "textbox", + name: "Email", + line: expect.any(Number), + }); + expect(out.text.split("\n")[out.refs[0]?.line ?? -1]).toContain('@e1 textbox "Email"'); + }); }); diff --git a/packages/vom/src/index.ts b/packages/vom/src/index.ts index a0ea267..aab8507 100644 --- a/packages/vom/src/index.ts +++ b/packages/vom/src/index.ts @@ -5,6 +5,7 @@ export type { CondSurface, LayerKind, Rect, + RenderedRef, Viewport, VomNode, VomOptions, diff --git a/packages/vom/src/render.ts b/packages/vom/src/render.ts index 40960ca..c5f76fd 100644 --- a/packages/vom/src/render.ts +++ b/packages/vom/src/render.ts @@ -4,9 +4,9 @@ import type { BlockingLayer, CondSurface, Rect, + RenderedRef, VomNode, VomOptions, - VomRef, VomResult, VomScene, } from "./types"; @@ -615,6 +615,7 @@ function renderNodeLine( ref: string | undefined, context: string[] = [], surface: CondSurface | undefined = undefined, + redactValues = false, ): string { let line = `${" ".repeat(depth)}${ref ? `@${ref} ` : ""}${node.role}`; @@ -636,13 +637,14 @@ function renderNodeLine( } const placeholder = cleaned(node.placeholder); if (placeholder) line += ` placeholder=${JSON.stringify(placeholder)}`; - const sensitiveHasValue = + const hasValue = rawValue !== undefined || node.inputState === "filled" || node.inputState === "default"; - const value = node.sensitive - ? sensitiveHasValue - ? SENSITIVE_MASK - : undefined - : rawValue?.slice(0, MAX_VALUE_LENGTH); + const value = + node.sensitive || redactValues + ? hasValue + ? SENSITIVE_MASK + : undefined + : rawValue?.slice(0, MAX_VALUE_LENGTH); if (value !== undefined) line += ` =${JSON.stringify(value)}`; if (surface && surface.subItems.length > 0) { @@ -695,11 +697,12 @@ function shouldSkipRedundantChildren(node: VomNode, state: RenderState): boolean interface RenderState { lines: string[]; - refs: VomRef[]; + refs: RenderedRef[]; nextRef: number; tokens: number; maxDepth: number; maxTokens: number; + redactValues: boolean; truncated: boolean; stopped: boolean; children: Map; @@ -764,7 +767,7 @@ function renderTree( const ref = isVomReferenceNode(node) ? `e${state.nextRef}` : undefined; const context = ref ? handleContext(node, state) : []; const surface = state.surfaceMap.get(node.id); - const line = renderNodeLine(node, depth, ref, context, surface); + const line = renderNodeLine(node, depth, ref, context, surface, state.redactValues); const nextTokens = state.tokens + estimateTokens(line); if (nextTokens > state.maxTokens) { state.truncated = true; @@ -775,10 +778,15 @@ function renderTree( state.lines.push(line); state.tokens = nextTokens; if (ref) { + const name = cleaned(node.name); state.refs.push({ ref, backendNodeId: node.backendNodeId ?? node.id, ...(node.frameId ? { frameId: node.frameId } : {}), + ...(node.role ? { role: node.role } : {}), + ...(name ? { name } : {}), + ...(context.length > 0 ? { ctx: context.join(" > ") } : {}), + line: state.lines.length - 1, }); state.nextRef += 1; } @@ -808,6 +816,7 @@ function renderNodes( tokens: estimateTokens(initialLines.join("\n")), maxDepth: options.maxDepth ?? DEFAULT_MAX_DEPTH, maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS, + redactValues: options.redactValues ?? false, truncated: false, stopped: false, children, diff --git a/packages/vom/src/types.ts b/packages/vom/src/types.ts index 6a1e891..187091b 100644 --- a/packages/vom/src/types.ts +++ b/packages/vom/src/types.ts @@ -64,6 +64,10 @@ export interface VomScene { export interface VomOptions { maxDepth?: number; maxTokens?: number; + /** + * When true, form values are rendered as masks instead of literal values. + */ + redactValues?: boolean; /** * Experimental: filter refs that are geometrically blocked by foreground * fixed/absolute/sticky regions. Disabled by default and does not alter the @@ -90,9 +94,17 @@ export interface VomRef { frameId?: string; } +export interface RenderedRef extends VomRef { + role?: string; + name?: string; + ctx?: string; + /** Zero-based line index in `VomResult.text`. */ + line: number; +} + export interface VomResult { text: string; - refs: VomRef[]; + refs: RenderedRef[]; truncated: boolean; }