diff --git a/apps/extension/src/tools/__tests__/observation.test.ts b/apps/extension/src/tools/__tests__/observation.test.ts index fd96156..bd8200b 100644 --- a/apps/extension/src/tools/__tests__/observation.test.ts +++ b/apps/extension/src/tools/__tests__/observation.test.ts @@ -3146,11 +3146,12 @@ describe("handleSnapshot", () => { expect(result.rootFrameId).toBe("root"); expect(result).not.toHaveProperty("frameDocuments"); expect(result.frames).toEqual([{ frameId: "root", target: { tabId: 4 } }]); - expect(result.nodes.find((node) => node.backendNodeId === 13)).toMatchObject({ + expect(result.matchNodes.find((node) => node.backendNodeId === 13)).toMatchObject({ frameId: "root", backendNodeId: 13, tag: "input", rect: { x: 400, y: 300, w: 200, h: 40 }, + localRect: { x: 400, y: 300, w: 200, h: 40 }, }); }); @@ -3214,7 +3215,10 @@ describe("handleSnapshot", () => { expect(dumped).not.toContain(secrets.otp); expect(dumped).not.toContain(secrets.card); expect(dumped).not.toContain("formValue"); + expect(dumped).not.toContain("formDefaultValue"); expect(dumped).not.toContain("axNodes"); + expect(dumped).not.toContain("domNodes"); + expect(dumped).not.toContain("attrs"); expect(result.text).toContain("•••"); }); @@ -3231,9 +3235,10 @@ describe("handleSnapshot", () => { ownerBackendNodeId: 12, target: { tabId: 4, sessionId: "child-session" }, }); - expect(result.nodes.find((node) => node.backendNodeId === 22)).toMatchObject({ + expect(result.matchNodes.find((node) => node.backendNodeId === 22)).toMatchObject({ frameId: "child", tag: "button", + localRect: { x: 20, y: 30, w: 120, h: 40 }, }); expect(result.refs.find((ref) => ref.backendNodeId === 22)).toMatchObject({ frameId: "child", diff --git a/apps/extension/src/tools/capture-vom-observation.ts b/apps/extension/src/tools/capture-vom-observation.ts index c2bc08f..5a24814 100644 --- a/apps/extension/src/tools/capture-vom-observation.ts +++ b/apps/extension/src/tools/capture-vom-observation.ts @@ -1,7 +1,9 @@ export { - type CaptureVomFrame, - type CaptureVomNode, type CaptureVomObservationOptions, - type CaptureVomObservationResult, captureVomObservation, } from "./observation"; +export { + type CaptureVomFrame, + type CaptureVomMatchNode, + type CaptureVomObservationResult, +} from "./vom/record-safe-observation"; diff --git a/apps/extension/src/tools/observation.ts b/apps/extension/src/tools/observation.ts index f8a25ca..f25d97b 100644 --- a/apps/extension/src/tools/observation.ts +++ b/apps/extension/src/tools/observation.ts @@ -8,8 +8,6 @@ import { applyVomInteractionRecovery, type CondSurface, isVomReferenceNode, - type Rect, - type RenderedRef, renderVom, type VomNode, type VomOptions, @@ -51,12 +49,15 @@ import { import { resolveSnapshotRef } from "./snapshot-ref"; import { type CapturedNode, - type CapturedSurfaceProbe, type CapturedViewModel, captureViewModel, collectOverlayExcludedBackendIds, } from "./vom/capture"; import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture"; +import { + type CaptureVomObservationResult, + projectRecordSafeObservation, +} from "./vom/record-safe-observation"; // --------------------------------------------------------------------------- // Shared helpers (legacy aliases — observation.ts kept exporting these @@ -1591,58 +1592,6 @@ export interface CaptureVomObservationOptions extends VomOptions { signal?: AbortSignal; } -/** Frame identity needed to click an `@eN` that lives in an iframe. */ -export interface CaptureVomFrame { - frameId: string; - target: CdpTarget; - parentFrameId?: string; - ownerBackendNodeId?: number; -} - -/** Geometry index for matching a recorded click to a captured node. */ -export interface CaptureVomNode { - frameId: string; - backendNodeId: number; - tag: string; - rect: Rect | null; -} - -/** - * Record-safe observation: rendered VOM text plus the matching index. - * Raw AX/DOM trees are not returned — they carry form values. - */ -export interface CaptureVomObservationResult { - text: string; - refs: RenderedRef[]; - truncated: boolean; - rootFrameId: string; - frames: CaptureVomFrame[]; - nodes: CaptureVomNode[]; - surfaceProbes?: CapturedSurfaceProbe[]; -} - -function recordSafeFrames(documents: CapturedFrameDocument[]): CaptureVomFrame[] { - return documents.map((document) => ({ - frameId: document.frameId, - target: document.target, - ...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}), - ...(document.ownerBackendNodeId !== undefined - ? { ownerBackendNodeId: document.ownerBackendNodeId } - : {}), - })); -} - -function recordSafeNodes(documents: CapturedFrameDocument[]): CaptureVomNode[] { - return documents.flatMap((document) => - document.domNodes.map((node) => ({ - frameId: document.frameId, - backendNodeId: node.backendNodeId, - tag: node.tag, - rect: node.rect, - })), - ); -} - export async function captureVomObservation( cdp: CdpRunner, tabId: number, @@ -1667,15 +1616,12 @@ export async function captureVomObservation( activeRegionPolicy: options.activeRegionPolicy, }); throwIfAborted(options.signal, "observation"); - return { - text: rendered.text, - refs: rendered.refs, - truncated: rendered.truncated, + return projectRecordSafeObservation({ rootFrameId: captured.rootFrameId ?? documents[0]?.frameId ?? "root", - frames: recordSafeFrames(documents), - nodes: recordSafeNodes(documents), + frameDocuments: documents, + rendered, surfaceProbes: captured.surfaceProbes, - }; + }); } async function handleVomObservation( diff --git a/apps/extension/src/tools/vom/record-safe-observation.ts b/apps/extension/src/tools/vom/record-safe-observation.ts new file mode 100644 index 0000000..3b5befd --- /dev/null +++ b/apps/extension/src/tools/vom/record-safe-observation.ts @@ -0,0 +1,77 @@ +import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom"; +import type { CdpTarget } from "@/browser-driver/frame-graph"; +import type { CapturedSurfaceProbe } from "./capture"; +import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture"; + +/** Frame identity needed to resolve an `@eN` that lives in an iframe. */ +export interface CaptureVomFrame { + frameId: string; + target: CdpTarget; + parentFrameId?: string; + ownerBackendNodeId?: number; +} + +/** Allowlisted geometry used to match a recorded action to a rendered ref. */ +export interface CaptureVomMatchNode { + frameId: string; + backendNodeId: number; + tag: string; + /** Top-level viewport-relative geometry. */ + rect: Rect | null; + /** Frame-local viewport-relative geometry, when available. */ + localRect?: Rect | null; +} + +/** + * Raw AX/DOM trees are never returned. When `redactValues` is enabled by the + * caller, form values in `text` are masked as well. + */ +export interface CaptureVomObservationResult { + text: string; + refs: RenderedRef[]; + truncated: boolean; + rootFrameId: string; + frames: CaptureVomFrame[]; + matchNodes: CaptureVomMatchNode[]; + surfaceProbes?: CapturedSurfaceProbe[]; +} + +function projectFrames(documents: CapturedFrameDocument[]): CaptureVomFrame[] { + return documents.map((document) => ({ + frameId: document.frameId, + target: document.target, + ...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}), + ...(document.ownerBackendNodeId !== undefined + ? { ownerBackendNodeId: document.ownerBackendNodeId } + : {}), + })); +} + +function projectMatchNodes(documents: CapturedFrameDocument[]): CaptureVomMatchNode[] { + return documents.flatMap((document) => + document.domNodes.map((node) => ({ + frameId: document.frameId, + backendNodeId: node.backendNodeId, + tag: node.tag, + rect: node.rect, + ...(node.localRect !== undefined ? { localRect: node.localRect } : {}), + })), + ); +} + +export function projectRecordSafeObservation(input: { + rootFrameId: string; + frameDocuments: CapturedFrameDocument[]; + rendered: VomResult; + surfaceProbes?: CapturedSurfaceProbe[]; +}): CaptureVomObservationResult { + return { + text: input.rendered.text, + refs: input.rendered.refs, + truncated: input.rendered.truncated, + rootFrameId: input.rootFrameId, + frames: projectFrames(input.frameDocuments), + matchNodes: projectMatchNodes(input.frameDocuments), + surfaceProbes: input.surfaceProbes, + }; +}