From 45a246c41b43364440398ebb51bae6c42beb41ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chaonan=E2=80=9D?= Date: Mon, 17 Aug 2026 17:47:37 +0800 Subject: [PATCH 1/6] feat(vom): expose record-safe semantic observations Share the VOM capture pipeline with recording, enrich rendered refs, and support value redaction without regressing cooperative cancellation. --- .../src/tools/__tests__/observation.test.ts | 48 ++++++- .../src/tools/capture-vom-observation.ts | 5 + apps/extension/src/tools/observation.ts | 127 ++++++++++++------ packages/vom/src/__tests__/render.test.ts | 64 +++++++-- packages/vom/src/index.ts | 1 + packages/vom/src/render.ts | 27 ++-- packages/vom/src/types.ts | 14 +- 7 files changed, 217 insertions(+), 69 deletions(-) create mode 100644 apps/extension/src/tools/capture-vom-observation.ts diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index a849843..af117f5 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -8,6 +8,7 @@ import { buildFrameVomScene, buildVomScene, type CdpAxNode, + captureVomObservation, handleGetHtml, handleObserve, handleScreenshot, @@ -1329,7 +1330,7 @@ describe("buildVomScene", () => { expect(rendered).toContain(' @e2 textbox "验证码"'); }); - it("does not mark non-password inputs sensitive from password-like labels", () => { + it("marks current-password autocomplete as sensitive even for text inputs", () => { const axNodes: CdpAxNode[] = [ { nodeId: "1", @@ -1364,7 +1365,7 @@ describe("buildVomScene", () => { expect(buildVomScene(axNodes, captured).nodes[0]).toEqual( expect.objectContaining({ id: 10, - sensitive: false, + sensitive: true, }), ); }); @@ -1659,7 +1660,9 @@ describe("buildVomScene", () => { ); const rendered = renderVom(scene); expect(rendered.text).toContain('@e1 button "close"'); - expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]); + expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([ + { ref: "e1", backendNodeId: 20 }, + ]); }); it("does not promote a clickable container that wraps a real interactive control", () => { @@ -1803,7 +1806,9 @@ describe("buildVomScene", () => { expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic"); const rendered = renderVom(scene); expect(rendered.text).toContain('@e1 button "收藏"'); - expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]); + expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([ + { ref: "e1", backendNodeId: 20 }, + ]); }); it("builds active scope blocks from active aria-controls relationships", () => { @@ -2908,6 +2913,41 @@ describe("handleSnapshot", () => { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; + it("exposes the frame-aware observation result for recording", async () => { + const ax: CdpAxNode[] = [ + { + nodeId: "root", + role: { type: "role", value: "RootWebArea" }, + backendDOMNodeId: 11, + childIds: ["password"], + }, + { + nodeId: "password", + parentId: "root", + role: { type: "role", value: "textbox" }, + name: { type: "computedString", value: "Password" }, + value: { value: "hunter2" }, + backendDOMNodeId: 13, + }, + ]; + const deps = makeOverlayDeps(ax, loginSnapshotReply(), VP_METRICS); + + const result = await captureVomObservation(deps.cdp, 4, "https://example.com", { + redactValues: true, + }); + + expect(result.text).not.toContain("hunter2"); + expect(result.text).toContain("•••"); + expect(result.refs[0]).toMatchObject({ + backendNodeId: 13, + role: "textbox", + name: "Password", + line: expect.any(Number), + }); + expect(result.captured.some((node) => node.backendNodeId === 13)).toBe(true); + expect(result.documents).toHaveLength(1); + }); + it("renders a blocking login overlay as the focused top layer", async () => { const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); await sm.start("aa11"); diff --git a/apps/extension/src/tools/capture-vom-observation.ts b/apps/extension/src/tools/capture-vom-observation.ts new file mode 100644 index 0000000..10b6d69 --- /dev/null +++ b/apps/extension/src/tools/capture-vom-observation.ts @@ -0,0 +1,5 @@ +export { + type CaptureVomObservationOptions, + type CaptureVomObservationResult, + captureVomObservation, +} from "./observation"; diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index 406a821..d4fe91b 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -8,8 +8,10 @@ import { applyVomInteractionRecovery, type CondSurface, isVomReferenceNode, + type RenderedRef, renderVom, type VomNode, + type VomOptions, type VomScene, } from "@browser-skill/vom"; import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; @@ -48,6 +50,7 @@ import { import { resolveSnapshotRef } from "./snapshot-ref"; import { type CapturedNode, + type CapturedSurfaceProbe, type CapturedViewModel, captureViewModel, collectOverlayExcludedBackendIds, @@ -411,7 +414,15 @@ function isModalSignal( function isSensitive(capturedNode: CapturedNode | undefined): boolean { const attrs = capturedNode?.attrs ?? {}; - return (attrs.type ?? "").toLowerCase() === "password"; + const type = (attrs.type ?? "").toLowerCase(); + if (type === "password") return true; + const autocomplete = (attrs.autocomplete ?? "").toLowerCase(); + return ( + autocomplete.startsWith("cc-") || + autocomplete === "one-time-code" || + autocomplete === "current-password" || + autocomplete === "new-password" + ); } interface AxNodeSignals { @@ -461,7 +472,13 @@ const AX_TEXT_STOP_ROLES = new Set([ "columnheader", "rowheader", ]); -const SENSITIVE_AX_INPUT_TYPES = new Set(["password", "credit-card"]); +const SENSITIVE_AX_INPUT_TYPES = new Set([ + "password", + "credit-card", + "one-time-code", + "current-password", + "new-password", +]); function axPropertyString(axNode: CdpAxNode, name: string): string | undefined { const prop = axNode.properties?.find((item) => item.name === name); @@ -1537,14 +1554,13 @@ async function fallbackCapturedViewModel( try { const metrics = await cdp.send(tabId, "Page.getLayoutMetrics", {}); throwIfAborted(signal, "observation"); - const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {}; + const source = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {}; viewport = { - width: vpSrc.clientWidth ?? 0, - height: vpSrc.clientHeight ?? 0, + width: source.clientWidth ?? 0, + height: source.clientHeight ?? 0, }; - } catch (err) { - if (isAbortError(err)) throw err; - // viewport stays zero-sized + } catch (error) { + if (isAbortError(error)) throw error; } const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal); throwIfAborted(signal, "observation"); @@ -1554,22 +1570,69 @@ async function fallbackCapturedViewModel( async function captureForVom( cdp: CdpRunner, tabId: number, - conditionalSurfaceProbe: boolean, - hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise, - signal?: AbortSignal, + options: CaptureVomObservationOptions, ): Promise { try { return await captureViewModel(cdp, tabId, { - conditionalSurfaceProbe, - hoverProbeBypassOverlay, - signal, + conditionalSurfaceProbe: options.conditionalSurfaceProbe ?? false, + hoverProbeBypassOverlay: options.hoverProbeBypassOverlay, + signal: options.signal, }); - } catch (err) { - if (isAbortError(err)) throw err; - return fallbackCapturedViewModel(cdp, tabId, signal); + } catch (error) { + if (isAbortError(error)) throw error; + return fallbackCapturedViewModel(cdp, tabId, options.signal); } } +export interface CaptureVomObservationOptions extends VomOptions { + conditionalSurfaceProbe?: boolean; + hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise; + signal?: AbortSignal; +} + +export interface CaptureVomObservationResult { + text: string; + refs: RenderedRef[]; + truncated: boolean; + captured: CapturedNode[]; + documents: CapturedFrameDocument[]; + surfaceProbes?: CapturedSurfaceProbe[]; +} + +export async function captureVomObservation( + cdp: CdpRunner, + tabId: number, + url: string | undefined, + options: CaptureVomObservationOptions = {}, +): Promise { + throwIfAborted(options.signal, "observation"); + await cdp.ensureAttachedToUrl?.(tabId, url); + throwIfAborted(options.signal, "observation"); + const captured = await captureForVom(cdp, tabId, options); + throwIfAborted(options.signal, "observation"); + const documents = await captureFrameData(cdp, tabId, captured, options.signal); + throwIfAborted(options.signal, "observation"); + const scene = + captured.rootFrameId && captured.frameNodes?.size + ? buildFrameVomScene(documents, captured, { pageUrl: url }) + : buildVomScene(documents[0]?.axNodes ?? [], captured, { pageUrl: url }); + const rendered = renderVom(scene, { + maxDepth: options.maxDepth, + maxTokens: options.maxTokens, + redactValues: options.redactValues, + activeRegionPolicy: options.activeRegionPolicy, + }); + throwIfAborted(options.signal, "observation"); + return { + text: rendered.text, + refs: rendered.refs, + truncated: rendered.truncated, + captured: captured.nodes, + documents, + surfaceProbes: captured.surfaceProbes, + }; +} + async function handleVomObservation( manager: SessionManager, params: SnapshotParams | ObserveParams, @@ -1599,37 +1662,19 @@ async function handleVomObservation( try { throwIfAborted(signal, toolName); deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); - await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url); - throwIfAborted(signal, toolName); const effectiveConditionalSurfaceProbe = deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe; - const captured = await captureForVom( - deps.cdp, - target.tabId, - effectiveConditionalSurfaceProbe, - deps.hoverProbeBypassOverlay, - signal, - ); - throwIfAborted(signal, toolName); - const frameDocuments = await captureFrameData( - deps.cdp, - target.tabId, - captured, - signal, - ); - throwIfAborted(signal, toolName); - const scene = - captured.rootFrameId && captured.frameNodes?.size - ? buildFrameVomScene(frameDocuments, captured, { pageUrl: target.url }) - : buildVomScene(frameDocuments[0]?.axNodes ?? [], captured, { pageUrl: target.url }); - const rendered = renderVom(scene, { + const rendered = await captureVomObservation(deps.cdp, target.tabId, target.url, { maxDepth: params.max_depth, maxTokens: params.max_tokens, activeRegionPolicy: true, + conditionalSurfaceProbe: effectiveConditionalSurfaceProbe, + hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay, + signal, }); throwIfAborted(signal, toolName); const targetByFrameId = new Map( - frameDocuments.map((document) => [document.frameId, document.target]), + rendered.documents.map((document) => [document.frameId, document.target]), ); ctx.refStore.replace( rendered.refs.map((ref) => { @@ -1653,7 +1698,7 @@ async function handleVomObservation( ...(toolName === "observe" && (params as ObserveParams).debug_surfaces ? { debug: { - surface_probes: (captured.surfaceProbes ?? []).map((probe) => ({ + surface_probes: (rendered.surfaceProbes ?? []).map((probe) => ({ trigger_backend_node_id: probe.triggerBackendNodeId, ...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}), trigger_action: probe.triggerAction, diff --git a/packages/vom/src/__tests__/render.test.ts b/packages/vom/src/__tests__/render.test.ts index d757c3b..d8a3d04 100644 --- a/packages/vom/src/__tests__/render.test.ts +++ b/packages/vom/src/__tests__/render.test.ts @@ -24,6 +24,10 @@ function scene(nodes: VomNode[]): VomScene { }; } +function coreRefs(refs: ReturnType["refs"]) { + return refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId })); +} + describe("renderVom single-layer page", () => { it("does not derive handle context across frame scopes", () => { const out = renderVom( @@ -130,7 +134,7 @@ describe("renderVom single-layer page", () => { ' @e2 button "加入购物车"', ].join("\n"), ); - expect(out.refs).toEqual([ + expect(coreRefs(out.refs)).toEqual([ { ref: "e1", backendNodeId: 3 }, { ref: "e2", backendNodeId: 6 }, ]); @@ -151,7 +155,7 @@ describe("renderVom single-layer page", () => { expect(out.text).toContain(' heading "Title"'); expect(out.text).toContain(' img "Logo"'); expect(out.text).toContain(' @e1 button "Continue"'); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]); }); it("assigns refs to listbox controls", () => { @@ -163,7 +167,7 @@ describe("renderVom single-layer page", () => { ); expect(out.text).toContain(' @e1 listbox "Choices"'); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]); }); it("annotates external links regardless of role casing", () => { @@ -200,7 +204,7 @@ describe("renderVom single-layer page", () => { expect(out.text).toContain(" alertdialog"); expect(out.text).toContain(" section"); expect(out.text).toContain(' @e1 button "Close"'); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 5 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 5 }]); }); it("skips generic nodes without adding an extra indentation level", () => { @@ -291,7 +295,7 @@ describe("renderVom single-layer page", () => { expect(out.text).toContain(" section"); expect(out.text).not.toContain("Too deep"); expect(out.text).toContain(' @e1 button "Later"'); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]); }); it("truncates when maxTokens is exceeded while keeping refs in sync with rendered text", () => { @@ -326,7 +330,7 @@ describe("renderVom single-layer page", () => { ); expect(out.text).toContain('@e1 button "Open settings"'); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]); }); it("keeps the deepest custom control when wrapper and child both look clickable", () => { @@ -356,7 +360,7 @@ describe("renderVom single-layer page", () => { expect(out.text).not.toContain('@e1 button "Card"'); expect(out.text).toContain('@e1 button "Details"'); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]); }); it("does not recover clickable wrappers around native controls", () => { @@ -379,7 +383,7 @@ describe("renderVom single-layer page", () => { expect(out.text).not.toContain('@e1 button "Checkout"'); expect(out.text).toContain('@e1 button "Pay"'); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]); }); it("adds context to duplicate weak action labels", () => { @@ -442,7 +446,7 @@ describe("renderVom single-layer page", () => { expect(out.text).toContain("@layers 1 focus=L1"); expect(out.text).toContain('@e1 button "Background"'); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]); }); it("can filter refs blocked by active positioned regions without changing layer output", () => { @@ -485,7 +489,7 @@ describe("renderVom single-layer page", () => { expect(out.text).not.toContain("active-region"); expect(out.text).not.toContain("Background"); expect(out.text).toContain('@e1 button "Foreground"'); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]); }); it("keeps paint-order comparisons inside their frame document", () => { @@ -600,7 +604,7 @@ describe("renderVom single-layer page", () => { }); expect(out.text).toContain('@e1 button "Products" [hover first: Shoes | Bags | Accessories]'); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]); }); it("injects active scope blocks immediately after their trigger without refs", () => { @@ -626,7 +630,7 @@ describe("renderVom single-layer page", () => { " Bob - great sound", ].join("\n"), ); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]); }); it("does not render redundant children inside named ref controls", () => { @@ -640,7 +644,7 @@ describe("renderVom single-layer page", () => { expect(out.text).toContain('@e1 button "Save"'); expect(out.text).not.toContain("disk icon"); - expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); + expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]); }); it("does not render redundant children inside native text inputs", () => { @@ -807,7 +811,7 @@ describe("renderVom double-layer page", () => { ].join("\n"), ); expect(out.text).not.toContain("底层按钮"); - expect(out.refs).toEqual([ + expect(coreRefs(out.refs)).toEqual([ { ref: "e1", backendNodeId: 4 }, { ref: "e2", backendNodeId: 5 }, ]); @@ -859,4 +863,36 @@ describe("renderVom double-layer page", () => { expect(out.text).toContain(' @e2 textbox "密码" [filled] ="•••"'); expect(out.refs.map((r) => r.backendNodeId)).toEqual([101, 102]); }); + + it("redacts form values and returns semantic metadata for rendered refs", () => { + const out = renderVom( + scene([ + node({ id: 1, role: "RootWebArea", name: "Form" }), + node({ + id: 2, + backendNodeId: 42, + frameId: "child-frame", + parentId: 1, + role: "textbox", + name: "Email", + value: "user@example.com", + inputState: "filled", + tag: "input", + }), + ]), + { redactValues: true }, + ); + + expect(out.text).toContain('[filled] ="•••"'); + expect(out.text).not.toContain("user@example.com"); + expect(out.refs[0]).toMatchObject({ + ref: "e1", + backendNodeId: 42, + frameId: "child-frame", + role: "textbox", + name: "Email", + line: expect.any(Number), + }); + expect(out.text.split("\n")[out.refs[0]?.line ?? -1]).toContain('@e1 textbox "Email"'); + }); }); diff --git a/packages/vom/src/index.ts b/packages/vom/src/index.ts index a0ea267..aab8507 100644 --- a/packages/vom/src/index.ts +++ b/packages/vom/src/index.ts @@ -5,6 +5,7 @@ export type { CondSurface, LayerKind, Rect, + RenderedRef, Viewport, VomNode, VomOptions, diff --git a/packages/vom/src/render.ts b/packages/vom/src/render.ts index 40960ca..c5f76fd 100644 --- a/packages/vom/src/render.ts +++ b/packages/vom/src/render.ts @@ -4,9 +4,9 @@ import type { BlockingLayer, CondSurface, Rect, + RenderedRef, VomNode, VomOptions, - VomRef, VomResult, VomScene, } from "./types"; @@ -615,6 +615,7 @@ function renderNodeLine( ref: string | undefined, context: string[] = [], surface: CondSurface | undefined = undefined, + redactValues = false, ): string { let line = `${" ".repeat(depth)}${ref ? `@${ref} ` : ""}${node.role}`; @@ -636,13 +637,14 @@ function renderNodeLine( } const placeholder = cleaned(node.placeholder); if (placeholder) line += ` placeholder=${JSON.stringify(placeholder)}`; - const sensitiveHasValue = + const hasValue = rawValue !== undefined || node.inputState === "filled" || node.inputState === "default"; - const value = node.sensitive - ? sensitiveHasValue - ? SENSITIVE_MASK - : undefined - : rawValue?.slice(0, MAX_VALUE_LENGTH); + const value = + node.sensitive || redactValues + ? hasValue + ? SENSITIVE_MASK + : undefined + : rawValue?.slice(0, MAX_VALUE_LENGTH); if (value !== undefined) line += ` =${JSON.stringify(value)}`; if (surface && surface.subItems.length > 0) { @@ -695,11 +697,12 @@ function shouldSkipRedundantChildren(node: VomNode, state: RenderState): boolean interface RenderState { lines: string[]; - refs: VomRef[]; + refs: RenderedRef[]; nextRef: number; tokens: number; maxDepth: number; maxTokens: number; + redactValues: boolean; truncated: boolean; stopped: boolean; children: Map; @@ -764,7 +767,7 @@ function renderTree( const ref = isVomReferenceNode(node) ? `e${state.nextRef}` : undefined; const context = ref ? handleContext(node, state) : []; const surface = state.surfaceMap.get(node.id); - const line = renderNodeLine(node, depth, ref, context, surface); + const line = renderNodeLine(node, depth, ref, context, surface, state.redactValues); const nextTokens = state.tokens + estimateTokens(line); if (nextTokens > state.maxTokens) { state.truncated = true; @@ -775,10 +778,15 @@ function renderTree( state.lines.push(line); state.tokens = nextTokens; if (ref) { + const name = cleaned(node.name); state.refs.push({ ref, backendNodeId: node.backendNodeId ?? node.id, ...(node.frameId ? { frameId: node.frameId } : {}), + ...(node.role ? { role: node.role } : {}), + ...(name ? { name } : {}), + ...(context.length > 0 ? { ctx: context.join(" > ") } : {}), + line: state.lines.length - 1, }); state.nextRef += 1; } @@ -808,6 +816,7 @@ function renderNodes( tokens: estimateTokens(initialLines.join("\n")), maxDepth: options.maxDepth ?? DEFAULT_MAX_DEPTH, maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS, + redactValues: options.redactValues ?? false, truncated: false, stopped: false, children, diff --git a/packages/vom/src/types.ts b/packages/vom/src/types.ts index 6a1e891..187091b 100644 --- a/packages/vom/src/types.ts +++ b/packages/vom/src/types.ts @@ -64,6 +64,10 @@ export interface VomScene { export interface VomOptions { maxDepth?: number; maxTokens?: number; + /** + * When true, form values are rendered as masks instead of literal values. + */ + redactValues?: boolean; /** * Experimental: filter refs that are geometrically blocked by foreground * fixed/absolute/sticky regions. Disabled by default and does not alter the @@ -90,9 +94,17 @@ export interface VomRef { frameId?: string; } +export interface RenderedRef extends VomRef { + role?: string; + name?: string; + ctx?: string; + /** Zero-based line index in `VomResult.text`. */ + line: number; +} + export interface VomResult { text: string; - refs: VomRef[]; + refs: RenderedRef[]; truncated: boolean; } From f86e506d47b47d5f31871d39ab77e3280155fd94 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Wed, 19 Aug 2026 11:19:06 +0800 Subject: [PATCH 2/6] feat(vom): update contract --- .../src/tools/__tests__/observation.test.ts | 157 +++++++++++++++++- apps/extension/src/tools/observation.ts | 22 +-- .../extension/src/tools/vom/frame-document.ts | 7 +- 3 files changed, 171 insertions(+), 15 deletions(-) diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index af117f5..7fb6ccf 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -2913,6 +2913,124 @@ describe("handleSnapshot", () => { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; + function makeFrameAwareDeps() { + const strings = ["html", "body", "iframe", "button", "title", "Remote", "static", "auto"]; + const i = (value: string) => strings.indexOf(value); + const styles = [i("static"), i("auto"), i("auto")]; + const snapshot = { + strings, + documents: [ + { + frameId: "main", + nodes: { + parentIndex: [-1, 0, 1], + nodeName: [i("html"), i("body"), i("iframe")], + backendNodeId: [10, 11, 12], + attributes: [[], [], [i("title"), i("Remote")]], + contentDocumentIndex: { index: [2], value: [1] }, + }, + layout: { + nodeIndex: [0, 1, 2], + styles: [styles, styles, styles], + bounds: [ + [0, 0, 1000, 800], + [0, 0, 1000, 800], + [100, 100, 400, 300], + ], + paintOrders: [0, 0, 1], + }, + }, + { + frameId: "child", + nodes: { + parentIndex: [-1, 0, 1], + nodeName: [i("html"), i("body"), i("button")], + backendNodeId: [20, 21, 22], + attributes: [[], [], []], + }, + layout: { + nodeIndex: [0, 1, 2], + styles: [styles, styles, styles], + bounds: [ + [0, 0, 400, 300], + [0, 0, 400, 300], + [20, 30, 120, 40], + ], + paintOrders: [0, 0, 1], + }, + }, + ], + }; + const mainAx: CdpAxNode[] = [ + { + nodeId: "main-root", + role: { type: "role", value: "RootWebArea" }, + backendDOMNodeId: 11, + childIds: ["frame-owner"], + }, + { + nodeId: "frame-owner", + parentId: "main-root", + role: { type: "role", value: "Iframe" }, + name: { type: "computedString", value: "Remote" }, + backendDOMNodeId: 12, + }, + ]; + const childAx: CdpAxNode[] = [ + { + nodeId: "child-root", + role: { type: "role", value: "RootWebArea" }, + backendDOMNodeId: 21, + childIds: ["child-button"], + }, + { + nodeId: "child-button", + parentId: "child-root", + role: { type: "role", value: "button" }, + name: { type: "computedString", value: "Frame action" }, + backendDOMNodeId: 22, + }, + ]; + const send = vi.fn(async (_tabId: number, method: string) => { + if (method === "Page.getLayoutMetrics") return VP_METRICS; + if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {}; + if (method === "DOMSnapshot.captureSnapshot") return snapshot; + if (method === "Accessibility.getFullAXTree") return { nodes: mainAx }; + throw new Error(`unexpected root CDP method ${method}`); + }); + const sendToTarget = vi.fn(async (_target, method: string) => { + if (method === "Accessibility.enable") return {}; + if (method === "Accessibility.getFullAXTree") return { nodes: childAx }; + throw new Error(`unexpected child CDP method ${method}`); + }); + const cdp = { + send: send as unknown as CdpRunner["send"], + sendToTarget: sendToTarget as unknown as NonNullable, + getFrameGraph: vi.fn(async () => ({ + rootFrameId: "main", + frames: [ + { frameId: "main", target: { tabId: 4 } }, + { + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 12, + target: { tabId: 4, sessionId: "child-session" }, + }, + ], + })), + trackSessionTab: vi.fn(), + } satisfies CdpRunner; + return { + cdp, + tabsApi: { + get: vi.fn( + async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab, + ), + query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]), + }, + }; + } + it("exposes the frame-aware observation result for recording", async () => { const ax: CdpAxNode[] = [ { @@ -2944,8 +3062,43 @@ describe("handleSnapshot", () => { name: "Password", line: expect.any(Number), }); - expect(result.captured.some((node) => node.backendNodeId === 13)).toBe(true); - expect(result.documents).toHaveLength(1); + expect(result.rootFrameId).toBe("root"); + expect(result.frameDocuments).toHaveLength(1); + expect(result.frameDocuments[0]?.domNodes.some((node) => node.backendNodeId === 13)).toBe(true); + }); + + it("keeps frame documents and rendered refs on the same frame identity", async () => { + const deps = makeFrameAwareDeps(); + + const result = await captureVomObservation(deps.cdp, 4, "https://example.com"); + + expect(result.rootFrameId).toBe("main"); + expect(result.frameDocuments).toHaveLength(2); + expect(result.frameDocuments.find((document) => document.frameId === "child")?.target).toEqual({ + tabId: 4, + sessionId: "child-session", + }); + expect(result.refs.find((ref) => ref.backendNodeId === 22)).toMatchObject({ + frameId: "child", + name: "Frame action", + }); + }); + + it("stores iframe refs with their owning CDP session", async () => { + const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); + const ctx = await sm.start("aa11"); + const deps = makeFrameAwareDeps(); + + const result = await handleSnapshot(sm, { session_id: "aa11" }, deps); + + if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`); + expect(result.text).toContain('@e1 button "Frame action"'); + const frameRef = [...ctx.refStore.entries()].find(([, entry]) => entry.backendNodeId === 22); + expect(frameRef?.[1]).toMatchObject({ + tabId: 4, + frameId: "child", + cdpSessionId: "child-session", + }); }); it("renders a blocking login overlay as the focused top layer", async () => { diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index d4fe91b..75d346f 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -1594,8 +1594,8 @@ export interface CaptureVomObservationResult { text: string; refs: RenderedRef[]; truncated: boolean; - captured: CapturedNode[]; - documents: CapturedFrameDocument[]; + rootFrameId: string; + frameDocuments: CapturedFrameDocument[]; surfaceProbes?: CapturedSurfaceProbe[]; } @@ -1627,8 +1627,8 @@ export async function captureVomObservation( text: rendered.text, refs: rendered.refs, truncated: rendered.truncated, - captured: captured.nodes, - documents, + rootFrameId: captured.rootFrameId ?? documents[0]?.frameId ?? "root", + frameDocuments: documents, surfaceProbes: captured.surfaceProbes, }; } @@ -1664,7 +1664,7 @@ async function handleVomObservation( deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); const effectiveConditionalSurfaceProbe = deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe; - const rendered = await captureVomObservation(deps.cdp, target.tabId, target.url, { + const observation = await captureVomObservation(deps.cdp, target.tabId, target.url, { maxDepth: params.max_depth, maxTokens: params.max_tokens, activeRegionPolicy: true, @@ -1674,10 +1674,10 @@ async function handleVomObservation( }); throwIfAborted(signal, toolName); const targetByFrameId = new Map( - rendered.documents.map((document) => [document.frameId, document.target]), + observation.frameDocuments.map((document) => [document.frameId, document.target]), ); ctx.refStore.replace( - rendered.refs.map((ref) => { + observation.refs.map((ref) => { const refTarget = ref.frameId ? targetByFrameId.get(ref.frameId) : undefined; return [ ref.ref, @@ -1691,14 +1691,14 @@ async function handleVomObservation( }), ); return attachDialogs(deps.cdp, target.tabId, dialogCursor, { - text: rendered.text, - ref_count: rendered.refs.length, + text: observation.text, + ref_count: observation.refs.length, tab_id: target.tabId, - truncated: rendered.truncated, + truncated: observation.truncated, ...(toolName === "observe" && (params as ObserveParams).debug_surfaces ? { debug: { - surface_probes: (rendered.surfaceProbes ?? []).map((probe) => ({ + surface_probes: (observation.surfaceProbes ?? []).map((probe) => ({ trigger_backend_node_id: probe.triggerBackendNodeId, ...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}), trigger_action: probe.triggerAction, diff --git a/apps/extension/src/tools/vom/frame-document.ts b/apps/extension/src/tools/vom/frame-document.ts index 7d0bda1..16e99a7 100644 --- a/apps/extension/src/tools/vom/frame-document.ts +++ b/apps/extension/src/tools/vom/frame-document.ts @@ -77,9 +77,12 @@ export function buildFrameDocuments( ): FrameDocument[] { const frames = frameList(graph, batches, captured); const frameById = new Map(frames.map((frame) => [frame.frameId, frame])); + const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId; + const domNodesForFrame = (frameId: string): CapturedNode[] => + captured.frameNodes?.get(frameId) ?? (frameId === rootFrameId ? captured.nodes : []); const backendOwner = new Map(); for (const frame of frames) { - for (const node of captured.frameNodes?.get(frame.frameId) ?? []) { + for (const node of domNodesForFrame(frame.frameId)) { backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId); } } @@ -165,6 +168,6 @@ export function buildFrameDocuments( ...frame, contextScopeId: frame.frameId, axNodes: axNodesByFrame.get(frame.frameId) ?? [], - domNodes: captured.frameNodes?.get(frame.frameId) ?? [], + domNodes: domNodesForFrame(frame.frameId), })); } From 3df27382800d80a3d38d521fa61e2fd4649d106e Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Wed, 19 Aug 2026 14:04:33 +0800 Subject: [PATCH 3/6] fix(vom): fix CI --- .../src/observation.ts | 9 ++--- .../tests/observation.test.ts | 36 ++++++++++++++++--- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/packages/dsh-plugin-browserskill/src/observation.ts b/packages/dsh-plugin-browserskill/src/observation.ts index 78991b6..e38412c 100644 --- a/packages/dsh-plugin-browserskill/src/observation.ts +++ b/packages/dsh-plugin-browserskill/src/observation.ts @@ -12,7 +12,7 @@ * events, and never move the registry's current pointer. */ -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { readFile, unlink } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -78,6 +78,7 @@ const FAILURE_BACKOFF_THRESHOLD = 3; const FRAME_RING_SIZE = 2; export class ObservationService { + private readonly scratchNamespace = randomUUID(); private readonly observations = new Map(); private readonly listeners = new Set<(event: ObservationEvent) => void>(); private readonly captureTimers = new Map(); @@ -304,9 +305,9 @@ export class ObservationService { if (current === undefined || current.dead === true) return; if (this.captureInFlight.has(sessionId)) return; this.captureInFlight.add(sessionId); - // One fixed scratch path per session (overwritten each frame) and always - // unlinked after the bytes are read — /tmp must not grow without bound. - const outPath = join(tmpdir(), `bsk-obs-${sessionId}.png`); + // Stable within this service, isolated from captures owned by other instances. + const sessionKey = createHash("sha256").update(sessionId).digest("hex").slice(0, 16); + const outPath = join(tmpdir(), `bsk-obs-${this.scratchNamespace}-${sessionKey}.png`); let writtenPath = outPath; try { const result = await this.deps.queue.run(sessionId, () => diff --git a/packages/dsh-plugin-browserskill/tests/observation.test.ts b/packages/dsh-plugin-browserskill/tests/observation.test.ts index e54a15d..7acd93d 100644 --- a/packages/dsh-plugin-browserskill/tests/observation.test.ts +++ b/packages/dsh-plugin-browserskill/tests/observation.test.ts @@ -5,7 +5,7 @@ import { existsSync, mkdtempSync, writeFileSync } from "node:fs"; import type { IncomingMessage, ServerResponse } from "node:http"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { basename, dirname } from "node:path"; import { describe, expect, it } from "vitest"; import { actionForLabel, @@ -548,8 +548,12 @@ describe("scratch files and thumbnail references", () => { ); const firstOut = runner.calls.filter((c) => c.args[0] === "screenshot").at(-1)?.args; const pathOf = (args: string[]) => args[args.indexOf("--out") + 1]; - expect(pathOf(firstOut ?? [])).toBe(join(tmpdir(), "bsk-obs-s1.png")); - expect(existsSync(pathOf(firstOut ?? []))).toBe(false); + const firstPath = pathOf(firstOut ?? []); + expect(dirname(firstPath)).toBe(tmpdir()); + expect(basename(firstPath)).toMatch( + /^bsk-obs-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-[0-9a-f]{16}\.png$/, + ); + expect(existsSync(firstPath)).toBe(false); // second frame: same path, deleted again service.endAction("s1"); scheduler.runNext(); @@ -559,11 +563,33 @@ describe("scratch files and thumbnail references", () => { scheduler.pending().length === 1, ); const secondOut = runner.calls.filter((c) => c.args[0] === "screenshot").at(-1)?.args; - expect(pathOf(secondOut ?? [])).toBe(join(tmpdir(), "bsk-obs-s1.png")); - await waitFor(() => !existsSync(pathOf(secondOut ?? []))); + const secondPath = pathOf(secondOut ?? []); + expect(secondPath).toBe(firstPath); + await waitFor(() => !existsSync(secondPath)); service.dispose(); }); + it("isolates scratch paths between service instances", async () => { + const first = setup({ scheduler: fakeScheduler(), runner: fakeRunner() }); + const second = setup({ scheduler: fakeScheduler(), runner: fakeRunner() }); + first.service.addSession("s1"); + second.service.addSession("s1"); + first.scheduler.runNext(); + second.scheduler.runNext(); + await waitFor( + () => + first.service.getState()[0]?.thumbnailAttachmentId !== undefined && + second.service.getState()[0]?.thumbnailAttachmentId !== undefined, + ); + const pathOf = (runner: FakeRunner) => { + const args = runner.calls.find((call) => call.args[0] === "screenshot")?.args ?? []; + return args[args.indexOf("--out") + 1]; + }; + expect(pathOf(first.runner)).not.toBe(pathOf(second.runner)); + first.service.dispose(); + second.service.dispose(); + }); + it("keeps the last two distinct frames and evicts the third", async () => { let n = 0; const runner = fakeRunner({ From 6aa931b60ad68c6308ac4949898cab4bc97ca120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chaonan=E2=80=9D?= Date: Wed, 19 Aug 2026 19:56:07 +0800 Subject: [PATCH 4/6] chore: retrigger opensource scan From 20554508db732abfb0c98d4712c1a7b2ccd175a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chaonan=E2=80=9D?= Date: Thu, 20 Aug 2026 20:45:19 +0800 Subject: [PATCH 5/6] fix(vom): omit raw AX/DOM trees from record-safe observations Return only frame identity and node geometry for matching so a serialized capture result cannot leak form values. --- .../src/tools/__tests__/observation.test.ts | 171 +++++++++++++++++- .../src/tools/capture-vom-observation.ts | 2 + apps/extension/src/tools/observation.ts | 51 +++++- 3 files changed, 214 insertions(+), 10 deletions(-) diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index 7fb6ccf..fd96156 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -2909,6 +2909,87 @@ describe("handleSnapshot", () => { ], }; } + + function secretsSnapshotReply(secrets: { + email: string; + password: string; + otp: string; + card: string; + }) { + const S = [ + "html", + "body", + "input", + "position", + "static", + "pointer-events", + "auto", + "type", + "text", + "password", + "autocomplete", + "one-time-code", + "cc-number", + "value", + secrets.email, + secrets.password, + secrets.otp, + secrets.card, + ]; + const i = (s: string) => S.indexOf(s); + const style = [i("static"), i("auto"), i("auto")]; + return { + strings: S, + documents: [ + { + nodes: { + parentIndex: [-1, 0, 1, 1, 1, 1], + nodeName: [i("html"), i("body"), i("input"), i("input"), i("input"), i("input")], + backendNodeId: [10, 11, 12, 13, 14, 15], + attributes: [ + [], + [], + [i("type"), i("text"), i("value"), i(secrets.email)], + [i("type"), i("password"), i("value"), i(secrets.password)], + [ + i("type"), + i("text"), + i("autocomplete"), + i("one-time-code"), + i("value"), + i(secrets.otp), + ], + [ + i("type"), + i("text"), + i("autocomplete"), + i("cc-number"), + i("value"), + i(secrets.card), + ], + ], + inputValue: { + index: [2, 3, 4, 5], + value: [i(secrets.email), i(secrets.password), i(secrets.otp), i(secrets.card)], + }, + }, + layout: { + nodeIndex: [1, 2, 3, 4, 5], + styles: [style, style, style, style, style], + bounds: [ + [0, 0, 1000, 800], + [10, 10, 200, 32], + [10, 50, 200, 32], + [10, 90, 200, 32], + [10, 130, 200, 32], + ], + paintOrders: [0, 1, 2, 3, 4], + }, + }, + ], + }; + } + const VP_METRICS = { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, }; @@ -3063,20 +3144,96 @@ describe("handleSnapshot", () => { line: expect.any(Number), }); expect(result.rootFrameId).toBe("root"); - expect(result.frameDocuments).toHaveLength(1); - expect(result.frameDocuments[0]?.domNodes.some((node) => node.backendNodeId === 13)).toBe(true); + expect(result).not.toHaveProperty("frameDocuments"); + expect(result.frames).toEqual([{ frameId: "root", target: { tabId: 4 } }]); + expect(result.nodes.find((node) => node.backendNodeId === 13)).toMatchObject({ + frameId: "root", + backendNodeId: 13, + tag: "input", + rect: { x: 400, y: 300, w: 200, h: 40 }, + }); }); - it("keeps frame documents and rendered refs on the same frame identity", async () => { + it("does not leak form secrets through the record-safe observation payload", async () => { + const secrets = { + email: "user@example.com", + password: "hunter2", + otp: "847291", + card: "4111111111111111", + }; + const ax: CdpAxNode[] = [ + { + nodeId: "root", + role: { type: "role", value: "RootWebArea" }, + backendDOMNodeId: 11, + childIds: ["email", "password", "otp", "card"], + }, + { + nodeId: "email", + parentId: "root", + role: { type: "role", value: "textbox" }, + name: { type: "computedString", value: "Email" }, + value: { value: secrets.email }, + backendDOMNodeId: 12, + }, + { + nodeId: "password", + parentId: "root", + role: { type: "role", value: "textbox" }, + name: { type: "computedString", value: "Password" }, + value: { value: secrets.password }, + backendDOMNodeId: 13, + }, + { + nodeId: "otp", + parentId: "root", + role: { type: "role", value: "textbox" }, + name: { type: "computedString", value: "Code" }, + value: { value: secrets.otp }, + backendDOMNodeId: 14, + }, + { + nodeId: "card", + parentId: "root", + role: { type: "role", value: "textbox" }, + name: { type: "computedString", value: "Card" }, + value: { value: secrets.card }, + backendDOMNodeId: 15, + }, + ]; + const deps = makeOverlayDeps(ax, secretsSnapshotReply(secrets), VP_METRICS); + + const result = await captureVomObservation(deps.cdp, 4, "https://example.com", { + redactValues: true, + }); + const dumped = JSON.stringify(result); + + expect(result).not.toHaveProperty("frameDocuments"); + expect(dumped).not.toContain(secrets.email); + expect(dumped).not.toContain(secrets.password); + expect(dumped).not.toContain(secrets.otp); + expect(dumped).not.toContain(secrets.card); + expect(dumped).not.toContain("formValue"); + expect(dumped).not.toContain("axNodes"); + expect(result.text).toContain("•••"); + }); + + it("keeps frames and rendered refs on the same frame identity", async () => { const deps = makeFrameAwareDeps(); const result = await captureVomObservation(deps.cdp, 4, "https://example.com"); expect(result.rootFrameId).toBe("main"); - expect(result.frameDocuments).toHaveLength(2); - expect(result.frameDocuments.find((document) => document.frameId === "child")?.target).toEqual({ - tabId: 4, - sessionId: "child-session", + expect(result.frames).toHaveLength(2); + expect(result.frames.find((frame) => frame.frameId === "child")).toEqual({ + frameId: "child", + parentFrameId: "main", + ownerBackendNodeId: 12, + target: { tabId: 4, sessionId: "child-session" }, + }); + expect(result.nodes.find((node) => node.backendNodeId === 22)).toMatchObject({ + frameId: "child", + tag: "button", }); expect(result.refs.find((ref) => ref.backendNodeId === 22)).toMatchObject({ frameId: "child", diff --git a/apps/extension/src/tools/capture-vom-observation.ts b/apps/extension/src/tools/capture-vom-observation.ts index 10b6d69..c2bc08f 100644 --- a/apps/extension/src/tools/capture-vom-observation.ts +++ b/apps/extension/src/tools/capture-vom-observation.ts @@ -1,4 +1,6 @@ export { + type CaptureVomFrame, + type CaptureVomNode, type CaptureVomObservationOptions, type CaptureVomObservationResult, captureVomObservation, diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index 75d346f..f8a25ca 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -8,6 +8,7 @@ import { applyVomInteractionRecovery, type CondSurface, isVomReferenceNode, + type Rect, type RenderedRef, renderVom, type VomNode, @@ -1590,15 +1591,58 @@ export interface CaptureVomObservationOptions extends VomOptions { signal?: AbortSignal; } +/** Frame identity needed to click an `@eN` that lives in an iframe. */ +export interface CaptureVomFrame { + frameId: string; + target: CdpTarget; + parentFrameId?: string; + ownerBackendNodeId?: number; +} + +/** Geometry index for matching a recorded click to a captured node. */ +export interface CaptureVomNode { + frameId: string; + backendNodeId: number; + tag: string; + rect: Rect | null; +} + +/** + * Record-safe observation: rendered VOM text plus the matching index. + * Raw AX/DOM trees are not returned — they carry form values. + */ export interface CaptureVomObservationResult { text: string; refs: RenderedRef[]; truncated: boolean; rootFrameId: string; - frameDocuments: CapturedFrameDocument[]; + frames: CaptureVomFrame[]; + nodes: CaptureVomNode[]; surfaceProbes?: CapturedSurfaceProbe[]; } +function recordSafeFrames(documents: CapturedFrameDocument[]): CaptureVomFrame[] { + return documents.map((document) => ({ + frameId: document.frameId, + target: document.target, + ...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}), + ...(document.ownerBackendNodeId !== undefined + ? { ownerBackendNodeId: document.ownerBackendNodeId } + : {}), + })); +} + +function recordSafeNodes(documents: CapturedFrameDocument[]): CaptureVomNode[] { + return documents.flatMap((document) => + document.domNodes.map((node) => ({ + frameId: document.frameId, + backendNodeId: node.backendNodeId, + tag: node.tag, + rect: node.rect, + })), + ); +} + export async function captureVomObservation( cdp: CdpRunner, tabId: number, @@ -1628,7 +1672,8 @@ export async function captureVomObservation( refs: rendered.refs, truncated: rendered.truncated, rootFrameId: captured.rootFrameId ?? documents[0]?.frameId ?? "root", - frameDocuments: documents, + frames: recordSafeFrames(documents), + nodes: recordSafeNodes(documents), surfaceProbes: captured.surfaceProbes, }; } @@ -1674,7 +1719,7 @@ async function handleVomObservation( }); throwIfAborted(signal, toolName); const targetByFrameId = new Map( - observation.frameDocuments.map((document) => [document.frameId, document.target]), + observation.frames.map((frame) => [frame.frameId, frame.target]), ); ctx.refStore.replace( observation.refs.map((ref) => { From 005a6fd5346f9d841ddaa75f16f59c28bd21380c Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Fri, 21 Aug 2026 11:26:52 +0800 Subject: [PATCH 6/6] fix(vom): add record safe observations, unify definition and construction of safe DTO --- .../src/tools/__tests__/observation.test.ts | 9 ++- .../src/tools/capture-vom-observation.ts | 8 +- apps/extension/src/tools/observation.ts | 70 ++--------------- .../src/tools/vom/record-safe-observation.ts | 77 +++++++++++++++++++ 4 files changed, 97 insertions(+), 67 deletions(-) create mode 100644 apps/extension/src/tools/vom/record-safe-observation.ts diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index fd96156..bd8200b 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -3146,11 +3146,12 @@ describe("handleSnapshot", () => { expect(result.rootFrameId).toBe("root"); expect(result).not.toHaveProperty("frameDocuments"); expect(result.frames).toEqual([{ frameId: "root", target: { tabId: 4 } }]); - expect(result.nodes.find((node) => node.backendNodeId === 13)).toMatchObject({ + expect(result.matchNodes.find((node) => node.backendNodeId === 13)).toMatchObject({ frameId: "root", backendNodeId: 13, tag: "input", rect: { x: 400, y: 300, w: 200, h: 40 }, + localRect: { x: 400, y: 300, w: 200, h: 40 }, }); }); @@ -3214,7 +3215,10 @@ describe("handleSnapshot", () => { expect(dumped).not.toContain(secrets.otp); expect(dumped).not.toContain(secrets.card); expect(dumped).not.toContain("formValue"); + expect(dumped).not.toContain("formDefaultValue"); expect(dumped).not.toContain("axNodes"); + expect(dumped).not.toContain("domNodes"); + expect(dumped).not.toContain("attrs"); expect(result.text).toContain("•••"); }); @@ -3231,9 +3235,10 @@ describe("handleSnapshot", () => { ownerBackendNodeId: 12, target: { tabId: 4, sessionId: "child-session" }, }); - expect(result.nodes.find((node) => node.backendNodeId === 22)).toMatchObject({ + expect(result.matchNodes.find((node) => node.backendNodeId === 22)).toMatchObject({ frameId: "child", tag: "button", + localRect: { x: 20, y: 30, w: 120, h: 40 }, }); expect(result.refs.find((ref) => ref.backendNodeId === 22)).toMatchObject({ frameId: "child", diff --git a/apps/extension/src/tools/capture-vom-observation.ts b/apps/extension/src/tools/capture-vom-observation.ts index c2bc08f..5a24814 100644 --- a/apps/extension/src/tools/capture-vom-observation.ts +++ b/apps/extension/src/tools/capture-vom-observation.ts @@ -1,7 +1,9 @@ export { - type CaptureVomFrame, - type CaptureVomNode, type CaptureVomObservationOptions, - type CaptureVomObservationResult, captureVomObservation, } from "./observation"; +export { + type CaptureVomFrame, + type CaptureVomMatchNode, + type CaptureVomObservationResult, +} from "./vom/record-safe-observation"; diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index f8a25ca..f25d97b 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -8,8 +8,6 @@ import { applyVomInteractionRecovery, type CondSurface, isVomReferenceNode, - type Rect, - type RenderedRef, renderVom, type VomNode, type VomOptions, @@ -51,12 +49,15 @@ import { import { resolveSnapshotRef } from "./snapshot-ref"; import { type CapturedNode, - type CapturedSurfaceProbe, type CapturedViewModel, captureViewModel, collectOverlayExcludedBackendIds, } from "./vom/capture"; import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture"; +import { + type CaptureVomObservationResult, + projectRecordSafeObservation, +} from "./vom/record-safe-observation"; // --------------------------------------------------------------------------- // Shared helpers (legacy aliases — observation.ts kept exporting these @@ -1591,58 +1592,6 @@ export interface CaptureVomObservationOptions extends VomOptions { signal?: AbortSignal; } -/** Frame identity needed to click an `@eN` that lives in an iframe. */ -export interface CaptureVomFrame { - frameId: string; - target: CdpTarget; - parentFrameId?: string; - ownerBackendNodeId?: number; -} - -/** Geometry index for matching a recorded click to a captured node. */ -export interface CaptureVomNode { - frameId: string; - backendNodeId: number; - tag: string; - rect: Rect | null; -} - -/** - * Record-safe observation: rendered VOM text plus the matching index. - * Raw AX/DOM trees are not returned — they carry form values. - */ -export interface CaptureVomObservationResult { - text: string; - refs: RenderedRef[]; - truncated: boolean; - rootFrameId: string; - frames: CaptureVomFrame[]; - nodes: CaptureVomNode[]; - surfaceProbes?: CapturedSurfaceProbe[]; -} - -function recordSafeFrames(documents: CapturedFrameDocument[]): CaptureVomFrame[] { - return documents.map((document) => ({ - frameId: document.frameId, - target: document.target, - ...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}), - ...(document.ownerBackendNodeId !== undefined - ? { ownerBackendNodeId: document.ownerBackendNodeId } - : {}), - })); -} - -function recordSafeNodes(documents: CapturedFrameDocument[]): CaptureVomNode[] { - return documents.flatMap((document) => - document.domNodes.map((node) => ({ - frameId: document.frameId, - backendNodeId: node.backendNodeId, - tag: node.tag, - rect: node.rect, - })), - ); -} - export async function captureVomObservation( cdp: CdpRunner, tabId: number, @@ -1667,15 +1616,12 @@ export async function captureVomObservation( activeRegionPolicy: options.activeRegionPolicy, }); throwIfAborted(options.signal, "observation"); - return { - text: rendered.text, - refs: rendered.refs, - truncated: rendered.truncated, + return projectRecordSafeObservation({ rootFrameId: captured.rootFrameId ?? documents[0]?.frameId ?? "root", - frames: recordSafeFrames(documents), - nodes: recordSafeNodes(documents), + frameDocuments: documents, + rendered, surfaceProbes: captured.surfaceProbes, - }; + }); } async function handleVomObservation( diff --git a/apps/extension/src/tools/vom/record-safe-observation.ts b/apps/extension/src/tools/vom/record-safe-observation.ts new file mode 100644 index 0000000..3b5befd --- /dev/null +++ b/apps/extension/src/tools/vom/record-safe-observation.ts @@ -0,0 +1,77 @@ +import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom"; +import type { CdpTarget } from "@/browser-driver/frame-graph"; +import type { CapturedSurfaceProbe } from "./capture"; +import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture"; + +/** Frame identity needed to resolve an `@eN` that lives in an iframe. */ +export interface CaptureVomFrame { + frameId: string; + target: CdpTarget; + parentFrameId?: string; + ownerBackendNodeId?: number; +} + +/** Allowlisted geometry used to match a recorded action to a rendered ref. */ +export interface CaptureVomMatchNode { + frameId: string; + backendNodeId: number; + tag: string; + /** Top-level viewport-relative geometry. */ + rect: Rect | null; + /** Frame-local viewport-relative geometry, when available. */ + localRect?: Rect | null; +} + +/** + * Raw AX/DOM trees are never returned. When `redactValues` is enabled by the + * caller, form values in `text` are masked as well. + */ +export interface CaptureVomObservationResult { + text: string; + refs: RenderedRef[]; + truncated: boolean; + rootFrameId: string; + frames: CaptureVomFrame[]; + matchNodes: CaptureVomMatchNode[]; + surfaceProbes?: CapturedSurfaceProbe[]; +} + +function projectFrames(documents: CapturedFrameDocument[]): CaptureVomFrame[] { + return documents.map((document) => ({ + frameId: document.frameId, + target: document.target, + ...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}), + ...(document.ownerBackendNodeId !== undefined + ? { ownerBackendNodeId: document.ownerBackendNodeId } + : {}), + })); +} + +function projectMatchNodes(documents: CapturedFrameDocument[]): CaptureVomMatchNode[] { + return documents.flatMap((document) => + document.domNodes.map((node) => ({ + frameId: document.frameId, + backendNodeId: node.backendNodeId, + tag: node.tag, + rect: node.rect, + ...(node.localRect !== undefined ? { localRect: node.localRect } : {}), + })), + ); +} + +export function projectRecordSafeObservation(input: { + rootFrameId: string; + frameDocuments: CapturedFrameDocument[]; + rendered: VomResult; + surfaceProbes?: CapturedSurfaceProbe[]; +}): CaptureVomObservationResult { + return { + text: input.rendered.text, + refs: input.rendered.refs, + truncated: input.rendered.truncated, + rootFrameId: input.rootFrameId, + frames: projectFrames(input.frameDocuments), + matchNodes: projectMatchNodes(input.frameDocuments), + surfaceProbes: input.surfaceProbes, + }; +}