Merge pull request #100 from Tencent/feat/record-vom-observation

feat(vom): expose record-safe semantic observations

apps/extension/src/tools/__tests__/observation.test.ts

@@ -8,6 +8,7 @@ import {
   buildFrameVomScene,
   buildVomScene,
   type CdpAxNode,
+  captureVomObservation,
   handleGetHtml,
   handleObserve,
   handleScreenshot,
@@ -1329,7 +1330,7 @@ describe("buildVomScene", () => {
     expect(rendered).toContain('      @e2 textbox "验证码"');
   });
 
-  it("does not mark non-password inputs sensitive from password-like labels", () => {
+  it("marks current-password autocomplete as sensitive even for text inputs", () => {
     const axNodes: CdpAxNode[] = [
       {
         nodeId: "1",
@@ -1364,7 +1365,7 @@ describe("buildVomScene", () => {
     expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
       expect.objectContaining({
         id: 10,
-        sensitive: false,
+        sensitive: true,
       }),
     );
   });
@@ -1659,7 +1660,9 @@ describe("buildVomScene", () => {
     );
     const rendered = renderVom(scene);
     expect(rendered.text).toContain('@e1 button "close"');
-    expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
+    expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
+      { ref: "e1", backendNodeId: 20 },
+    ]);
   });
 
   it("does not promote a clickable container that wraps a real interactive control", () => {
@@ -1803,7 +1806,9 @@ describe("buildVomScene", () => {
     expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic");
     const rendered = renderVom(scene);
     expect(rendered.text).toContain('@e1 button "收藏"');
-    expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
+    expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
+      { ref: "e1", backendNodeId: 20 },
+    ]);
   });
 
   it("builds active scope blocks from active aria-controls relationships", () => {
@@ -2904,10 +2909,360 @@ describe("handleSnapshot", () => {
       ],
     };
   }
+
+  function secretsSnapshotReply(secrets: {
+    email: string;
+    password: string;
+    otp: string;
+    card: string;
+  }) {
+    const S = [
+      "html",
+      "body",
+      "input",
+      "position",
+      "static",
+      "pointer-events",
+      "auto",
+      "type",
+      "text",
+      "password",
+      "autocomplete",
+      "one-time-code",
+      "cc-number",
+      "value",
+      secrets.email,
+      secrets.password,
+      secrets.otp,
+      secrets.card,
+    ];
+    const i = (s: string) => S.indexOf(s);
+    const style = [i("static"), i("auto"), i("auto")];
+    return {
+      strings: S,
+      documents: [
+        {
+          nodes: {
+            parentIndex: [-1, 0, 1, 1, 1, 1],
+            nodeName: [i("html"), i("body"), i("input"), i("input"), i("input"), i("input")],
+            backendNodeId: [10, 11, 12, 13, 14, 15],
+            attributes: [
+              [],
+              [],
+              [i("type"), i("text"), i("value"), i(secrets.email)],
+              [i("type"), i("password"), i("value"), i(secrets.password)],
+              [
+                i("type"),
+                i("text"),
+                i("autocomplete"),
+                i("one-time-code"),
+                i("value"),
+                i(secrets.otp),
+              ],
+              [
+                i("type"),
+                i("text"),
+                i("autocomplete"),
+                i("cc-number"),
+                i("value"),
+                i(secrets.card),
+              ],
+            ],
+            inputValue: {
+              index: [2, 3, 4, 5],
+              value: [i(secrets.email), i(secrets.password), i(secrets.otp), i(secrets.card)],
+            },
+          },
+          layout: {
+            nodeIndex: [1, 2, 3, 4, 5],
+            styles: [style, style, style, style, style],
+            bounds: [
+              [0, 0, 1000, 800],
+              [10, 10, 200, 32],
+              [10, 50, 200, 32],
+              [10, 90, 200, 32],
+              [10, 130, 200, 32],
+            ],
+            paintOrders: [0, 1, 2, 3, 4],
+          },
+        },
+      ],
+    };
+  }
+
   const VP_METRICS = {
     cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
   };
 
+  function makeFrameAwareDeps() {
+    const strings = ["html", "body", "iframe", "button", "title", "Remote", "static", "auto"];
+    const i = (value: string) => strings.indexOf(value);
+    const styles = [i("static"), i("auto"), i("auto")];
+    const snapshot = {
+      strings,
+      documents: [
+        {
+          frameId: "main",
+          nodes: {
+            parentIndex: [-1, 0, 1],
+            nodeName: [i("html"), i("body"), i("iframe")],
+            backendNodeId: [10, 11, 12],
+            attributes: [[], [], [i("title"), i("Remote")]],
+            contentDocumentIndex: { index: [2], value: [1] },
+          },
+          layout: {
+            nodeIndex: [0, 1, 2],
+            styles: [styles, styles, styles],
+            bounds: [
+              [0, 0, 1000, 800],
+              [0, 0, 1000, 800],
+              [100, 100, 400, 300],
+            ],
+            paintOrders: [0, 0, 1],
+          },
+        },
+        {
+          frameId: "child",
+          nodes: {
+            parentIndex: [-1, 0, 1],
+            nodeName: [i("html"), i("body"), i("button")],
+            backendNodeId: [20, 21, 22],
+            attributes: [[], [], []],
+          },
+          layout: {
+            nodeIndex: [0, 1, 2],
+            styles: [styles, styles, styles],
+            bounds: [
+              [0, 0, 400, 300],
+              [0, 0, 400, 300],
+              [20, 30, 120, 40],
+            ],
+            paintOrders: [0, 0, 1],
+          },
+        },
+      ],
+    };
+    const mainAx: CdpAxNode[] = [
+      {
+        nodeId: "main-root",
+        role: { type: "role", value: "RootWebArea" },
+        backendDOMNodeId: 11,
+        childIds: ["frame-owner"],
+      },
+      {
+        nodeId: "frame-owner",
+        parentId: "main-root",
+        role: { type: "role", value: "Iframe" },
+        name: { type: "computedString", value: "Remote" },
+        backendDOMNodeId: 12,
+      },
+    ];
+    const childAx: CdpAxNode[] = [
+      {
+        nodeId: "child-root",
+        role: { type: "role", value: "RootWebArea" },
+        backendDOMNodeId: 21,
+        childIds: ["child-button"],
+      },
+      {
+        nodeId: "child-button",
+        parentId: "child-root",
+        role: { type: "role", value: "button" },
+        name: { type: "computedString", value: "Frame action" },
+        backendDOMNodeId: 22,
+      },
+    ];
+    const send = vi.fn(async (_tabId: number, method: string) => {
+      if (method === "Page.getLayoutMetrics") return VP_METRICS;
+      if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
+      if (method === "DOMSnapshot.captureSnapshot") return snapshot;
+      if (method === "Accessibility.getFullAXTree") return { nodes: mainAx };
+      throw new Error(`unexpected root CDP method ${method}`);
+    });
+    const sendToTarget = vi.fn(async (_target, method: string) => {
+      if (method === "Accessibility.enable") return {};
+      if (method === "Accessibility.getFullAXTree") return { nodes: childAx };
+      throw new Error(`unexpected child CDP method ${method}`);
+    });
+    const cdp = {
+      send: send as unknown as CdpRunner["send"],
+      sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
+      getFrameGraph: vi.fn(async () => ({
+        rootFrameId: "main",
+        frames: [
+          { frameId: "main", target: { tabId: 4 } },
+          {
+            frameId: "child",
+            parentFrameId: "main",
+            ownerBackendNodeId: 12,
+            target: { tabId: 4, sessionId: "child-session" },
+          },
+        ],
+      })),
+      trackSessionTab: vi.fn(),
+    } satisfies CdpRunner;
+    return {
+      cdp,
+      tabsApi: {
+        get: vi.fn(
+          async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
+        ),
+        query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
+      },
+    };
+  }
+
+  it("exposes the frame-aware observation result for recording", async () => {
+    const ax: CdpAxNode[] = [
+      {
+        nodeId: "root",
+        role: { type: "role", value: "RootWebArea" },
+        backendDOMNodeId: 11,
+        childIds: ["password"],
+      },
+      {
+        nodeId: "password",
+        parentId: "root",
+        role: { type: "role", value: "textbox" },
+        name: { type: "computedString", value: "Password" },
+        value: { value: "hunter2" },
+        backendDOMNodeId: 13,
+      },
+    ];
+    const deps = makeOverlayDeps(ax, loginSnapshotReply(), VP_METRICS);
+
+    const result = await captureVomObservation(deps.cdp, 4, "https://example.com", {
+      redactValues: true,
+    });
+
+    expect(result.text).not.toContain("hunter2");
+    expect(result.text).toContain("•••");
+    expect(result.refs[0]).toMatchObject({
+      backendNodeId: 13,
+      role: "textbox",
+      name: "Password",
+      line: expect.any(Number),
+    });
+    expect(result.rootFrameId).toBe("root");
+    expect(result).not.toHaveProperty("frameDocuments");
+    expect(result.frames).toEqual([{ frameId: "root", target: { tabId: 4 } }]);
+    expect(result.matchNodes.find((node) => node.backendNodeId === 13)).toMatchObject({
+      frameId: "root",
+      backendNodeId: 13,
+      tag: "input",
+      rect: { x: 400, y: 300, w: 200, h: 40 },
+      localRect: { x: 400, y: 300, w: 200, h: 40 },
+    });
+  });
+
+  it("does not leak form secrets through the record-safe observation payload", async () => {
+    const secrets = {
+      email: "user@example.com",
+      password: "hunter2",
+      otp: "847291",
+      card: "4111111111111111",
+    };
+    const ax: CdpAxNode[] = [
+      {
+        nodeId: "root",
+        role: { type: "role", value: "RootWebArea" },
+        backendDOMNodeId: 11,
+        childIds: ["email", "password", "otp", "card"],
+      },
+      {
+        nodeId: "email",
+        parentId: "root",
+        role: { type: "role", value: "textbox" },
+        name: { type: "computedString", value: "Email" },
+        value: { value: secrets.email },
+        backendDOMNodeId: 12,
+      },
+      {
+        nodeId: "password",
+        parentId: "root",
+        role: { type: "role", value: "textbox" },
+        name: { type: "computedString", value: "Password" },
+        value: { value: secrets.password },
+        backendDOMNodeId: 13,
+      },
+      {
+        nodeId: "otp",
+        parentId: "root",
+        role: { type: "role", value: "textbox" },
+        name: { type: "computedString", value: "Code" },
+        value: { value: secrets.otp },
+        backendDOMNodeId: 14,
+      },
+      {
+        nodeId: "card",
+        parentId: "root",
+        role: { type: "role", value: "textbox" },
+        name: { type: "computedString", value: "Card" },
+        value: { value: secrets.card },
+        backendDOMNodeId: 15,
+      },
+    ];
+    const deps = makeOverlayDeps(ax, secretsSnapshotReply(secrets), VP_METRICS);
+
+    const result = await captureVomObservation(deps.cdp, 4, "https://example.com", {
+      redactValues: true,
+    });
+    const dumped = JSON.stringify(result);
+
+    expect(result).not.toHaveProperty("frameDocuments");
+    expect(dumped).not.toContain(secrets.email);
+    expect(dumped).not.toContain(secrets.password);
+    expect(dumped).not.toContain(secrets.otp);
+    expect(dumped).not.toContain(secrets.card);
+    expect(dumped).not.toContain("formValue");
+    expect(dumped).not.toContain("formDefaultValue");
+    expect(dumped).not.toContain("axNodes");
+    expect(dumped).not.toContain("domNodes");
+    expect(dumped).not.toContain("attrs");
+    expect(result.text).toContain("•••");
+  });
+
+  it("keeps frames and rendered refs on the same frame identity", async () => {
+    const deps = makeFrameAwareDeps();
+
+    const result = await captureVomObservation(deps.cdp, 4, "https://example.com");
+
+    expect(result.rootFrameId).toBe("main");
+    expect(result.frames).toHaveLength(2);
+    expect(result.frames.find((frame) => frame.frameId === "child")).toEqual({
+      frameId: "child",
+      parentFrameId: "main",
+      ownerBackendNodeId: 12,
+      target: { tabId: 4, sessionId: "child-session" },
+    });
+    expect(result.matchNodes.find((node) => node.backendNodeId === 22)).toMatchObject({
+      frameId: "child",
+      tag: "button",
+      localRect: { x: 20, y: 30, w: 120, h: 40 },
+    });
+    expect(result.refs.find((ref) => ref.backendNodeId === 22)).toMatchObject({
+      frameId: "child",
+      name: "Frame action",
+    });
+  });
+
+  it("stores iframe refs with their owning CDP session", async () => {
+    const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
+    const ctx = await sm.start("aa11");
+    const deps = makeFrameAwareDeps();
+
+    const result = await handleSnapshot(sm, { session_id: "aa11" }, deps);
+
+    if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`);
+    expect(result.text).toContain('@e1 button "Frame action"');
+    const frameRef = [...ctx.refStore.entries()].find(([, entry]) => entry.backendNodeId === 22);
+    expect(frameRef?.[1]).toMatchObject({
+      tabId: 4,
+      frameId: "child",
+      cdpSessionId: "child-session",
+    });
+  });
+
   it("renders a blocking login overlay as the focused top layer", async () => {
     const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
     await sm.start("aa11");

apps/extension/src/tools/capture-vom-observation.ts

@@ -0,0 +1,9 @@
+export {
+  type CaptureVomObservationOptions,
+  captureVomObservation,
+} from "./observation";
+export {
+  type CaptureVomFrame,
+  type CaptureVomMatchNode,
+  type CaptureVomObservationResult,
+} from "./vom/record-safe-observation";

apps/extension/src/tools/observation.ts

@@ -10,6 +10,7 @@ import {
   isVomReferenceNode,
   renderVom,
   type VomNode,
+  type VomOptions,
   type VomScene,
 } from "@browser-skill/vom";
 import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
@@ -53,6 +54,10 @@ import {
   collectOverlayExcludedBackendIds,
 } from "./vom/capture";
 import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
+import {
+  type CaptureVomObservationResult,
+  projectRecordSafeObservation,
+} from "./vom/record-safe-observation";
 
 // ---------------------------------------------------------------------------
 // Shared helpers (legacy aliases — observation.ts kept exporting these
@@ -411,7 +416,15 @@ function isModalSignal(
 
 function isSensitive(capturedNode: CapturedNode | undefined): boolean {
   const attrs = capturedNode?.attrs ?? {};
-  return (attrs.type ?? "").toLowerCase() === "password";
+  const type = (attrs.type ?? "").toLowerCase();
+  if (type === "password") return true;
+  const autocomplete = (attrs.autocomplete ?? "").toLowerCase();
+  return (
+    autocomplete.startsWith("cc-") ||
+    autocomplete === "one-time-code" ||
+    autocomplete === "current-password" ||
+    autocomplete === "new-password"
+  );
 }
 
 interface AxNodeSignals {
@@ -461,7 +474,13 @@ const AX_TEXT_STOP_ROLES = new Set([
   "columnheader",
   "rowheader",
 ]);
-const SENSITIVE_AX_INPUT_TYPES = new Set(["password", "credit-card"]);
+const SENSITIVE_AX_INPUT_TYPES = new Set([
+  "password",
+  "credit-card",
+  "one-time-code",
+  "current-password",
+  "new-password",
+]);
 
 function axPropertyString(axNode: CdpAxNode, name: string): string | undefined {
   const prop = axNode.properties?.find((item) => item.name === name);
@@ -1537,14 +1556,13 @@ async function fallbackCapturedViewModel(
   try {
     const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
     throwIfAborted(signal, "observation");
-    const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
+    const source = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
     viewport = {
-      width: vpSrc.clientWidth ?? 0,
-      height: vpSrc.clientHeight ?? 0,
+      width: source.clientWidth ?? 0,
+      height: source.clientHeight ?? 0,
     };
-  } catch (err) {
-    if (isAbortError(err)) throw err;
-    // viewport stays zero-sized
+  } catch (error) {
+    if (isAbortError(error)) throw error;
   }
   const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal);
   throwIfAborted(signal, "observation");
@@ -1554,22 +1572,58 @@ async function fallbackCapturedViewModel(
 async function captureForVom(
   cdp: CdpRunner,
   tabId: number,
-  conditionalSurfaceProbe: boolean,
-  hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>,
-  signal?: AbortSignal,
+  options: CaptureVomObservationOptions,
 ): Promise<CapturedViewModel> {
   try {
     return await captureViewModel(cdp, tabId, {
-      conditionalSurfaceProbe,
-      hoverProbeBypassOverlay,
-      signal,
+      conditionalSurfaceProbe: options.conditionalSurfaceProbe ?? false,
+      hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
+      signal: options.signal,
     });
-  } catch (err) {
-    if (isAbortError(err)) throw err;
-    return fallbackCapturedViewModel(cdp, tabId, signal);
+  } catch (error) {
+    if (isAbortError(error)) throw error;
+    return fallbackCapturedViewModel(cdp, tabId, options.signal);
   }
 }
 
+export interface CaptureVomObservationOptions extends VomOptions {
+  conditionalSurfaceProbe?: boolean;
+  hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
+  signal?: AbortSignal;
+}
+
+export async function captureVomObservation(
+  cdp: CdpRunner,
+  tabId: number,
+  url: string | undefined,
+  options: CaptureVomObservationOptions = {},
+): Promise<CaptureVomObservationResult> {
+  throwIfAborted(options.signal, "observation");
+  await cdp.ensureAttachedToUrl?.(tabId, url);
+  throwIfAborted(options.signal, "observation");
+  const captured = await captureForVom(cdp, tabId, options);
+  throwIfAborted(options.signal, "observation");
+  const documents = await captureFrameData<CdpAxNode>(cdp, tabId, captured, options.signal);
+  throwIfAborted(options.signal, "observation");
+  const scene =
+    captured.rootFrameId && captured.frameNodes?.size
+      ? buildFrameVomScene(documents, captured, { pageUrl: url })
+      : buildVomScene(documents[0]?.axNodes ?? [], captured, { pageUrl: url });
+  const rendered = renderVom(scene, {
+    maxDepth: options.maxDepth,
+    maxTokens: options.maxTokens,
+    redactValues: options.redactValues,
+    activeRegionPolicy: options.activeRegionPolicy,
+  });
+  throwIfAborted(options.signal, "observation");
+  return projectRecordSafeObservation({
+    rootFrameId: captured.rootFrameId ?? documents[0]?.frameId ?? "root",
+    frameDocuments: documents,
+    rendered,
+    surfaceProbes: captured.surfaceProbes,
+  });
+}
+
 async function handleVomObservation(
   manager: SessionManager,
   params: SnapshotParams | ObserveParams,
@@ -1599,40 +1653,22 @@ async function handleVomObservation(
   try {
     throwIfAborted(signal, toolName);
     deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
-    await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
-    throwIfAborted(signal, toolName);
     const effectiveConditionalSurfaceProbe =
       deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
-    const captured = await captureForVom(
-      deps.cdp,
-      target.tabId,
-      effectiveConditionalSurfaceProbe,
-      deps.hoverProbeBypassOverlay,
-      signal,
-    );
-    throwIfAborted(signal, toolName);
-    const frameDocuments = await captureFrameData<CdpAxNode>(
-      deps.cdp,
-      target.tabId,
-      captured,
-      signal,
-    );
-    throwIfAborted(signal, toolName);
-    const scene =
-      captured.rootFrameId && captured.frameNodes?.size
-        ? buildFrameVomScene(frameDocuments, captured, { pageUrl: target.url })
-        : buildVomScene(frameDocuments[0]?.axNodes ?? [], captured, { pageUrl: target.url });
-    const rendered = renderVom(scene, {
+    const observation = await captureVomObservation(deps.cdp, target.tabId, target.url, {
       maxDepth: params.max_depth,
       maxTokens: params.max_tokens,
       activeRegionPolicy: true,
+      conditionalSurfaceProbe: effectiveConditionalSurfaceProbe,
+      hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay,
+      signal,
     });
     throwIfAborted(signal, toolName);
     const targetByFrameId = new Map(
-      frameDocuments.map((document) => [document.frameId, document.target]),
+      observation.frames.map((frame) => [frame.frameId, frame.target]),
     );
     ctx.refStore.replace(
-      rendered.refs.map((ref) => {
+      observation.refs.map((ref) => {
         const refTarget = ref.frameId ? targetByFrameId.get(ref.frameId) : undefined;
         return [
           ref.ref,
@@ -1646,14 +1682,14 @@ async function handleVomObservation(
       }),
     );
     return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
-      text: rendered.text,
-      ref_count: rendered.refs.length,
+      text: observation.text,
+      ref_count: observation.refs.length,
       tab_id: target.tabId,
-      truncated: rendered.truncated,
+      truncated: observation.truncated,
       ...(toolName === "observe" && (params as ObserveParams).debug_surfaces
         ? {
             debug: {
-              surface_probes: (captured.surfaceProbes ?? []).map((probe) => ({
+              surface_probes: (observation.surfaceProbes ?? []).map((probe) => ({
                 trigger_backend_node_id: probe.triggerBackendNodeId,
                 ...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}),
                 trigger_action: probe.triggerAction,

apps/extension/src/tools/vom/frame-document.ts

@@ -77,9 +77,12 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
 ): FrameDocument<T>[] {
   const frames = frameList(graph, batches, captured);
   const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
+  const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId;
+  const domNodesForFrame = (frameId: string): CapturedNode[] =>
+    captured.frameNodes?.get(frameId) ?? (frameId === rootFrameId ? captured.nodes : []);
   const backendOwner = new Map<string, string>();
   for (const frame of frames) {
-    for (const node of captured.frameNodes?.get(frame.frameId) ?? []) {
+    for (const node of domNodesForFrame(frame.frameId)) {
       backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId);
     }
   }
@@ -165,6 +168,6 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
     ...frame,
     contextScopeId: frame.frameId,
     axNodes: axNodesByFrame.get(frame.frameId) ?? [],
-    domNodes: captured.frameNodes?.get(frame.frameId) ?? [],
+    domNodes: domNodesForFrame(frame.frameId),
   }));
 }

apps/extension/src/tools/vom/record-safe-observation.ts

@@ -0,0 +1,77 @@
+import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom";
+import type { CdpTarget } from "@/browser-driver/frame-graph";
+import type { CapturedSurfaceProbe } from "./capture";
+import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture";
+
+/** Frame identity needed to resolve an `@eN` that lives in an iframe. */
+export interface CaptureVomFrame {
+  frameId: string;
+  target: CdpTarget;
+  parentFrameId?: string;
+  ownerBackendNodeId?: number;
+}
+
+/** Allowlisted geometry used to match a recorded action to a rendered ref. */
+export interface CaptureVomMatchNode {
+  frameId: string;
+  backendNodeId: number;
+  tag: string;
+  /** Top-level viewport-relative geometry. */
+  rect: Rect | null;
+  /** Frame-local viewport-relative geometry, when available. */
+  localRect?: Rect | null;
+}
+
+/**
+ * Raw AX/DOM trees are never returned. When `redactValues` is enabled by the
+ * caller, form values in `text` are masked as well.
+ */
+export interface CaptureVomObservationResult {
+  text: string;
+  refs: RenderedRef[];
+  truncated: boolean;
+  rootFrameId: string;
+  frames: CaptureVomFrame[];
+  matchNodes: CaptureVomMatchNode[];
+  surfaceProbes?: CapturedSurfaceProbe[];
+}
+
+function projectFrames(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomFrame[] {
+  return documents.map((document) => ({
+    frameId: document.frameId,
+    target: document.target,
+    ...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
+    ...(document.ownerBackendNodeId !== undefined
+      ? { ownerBackendNodeId: document.ownerBackendNodeId }
+      : {}),
+  }));
+}
+
+function projectMatchNodes(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomMatchNode[] {
+  return documents.flatMap((document) =>
+    document.domNodes.map((node) => ({
+      frameId: document.frameId,
+      backendNodeId: node.backendNodeId,
+      tag: node.tag,
+      rect: node.rect,
+      ...(node.localRect !== undefined ? { localRect: node.localRect } : {}),
+    })),
+  );
+}
+
+export function projectRecordSafeObservation(input: {
+  rootFrameId: string;
+  frameDocuments: CapturedFrameDocument<FrameAxNode>[];
+  rendered: VomResult;
+  surfaceProbes?: CapturedSurfaceProbe[];
+}): CaptureVomObservationResult {
+  return {
+    text: input.rendered.text,
+    refs: input.rendered.refs,
+    truncated: input.rendered.truncated,
+    rootFrameId: input.rootFrameId,
+    frames: projectFrames(input.frameDocuments),
+    matchNodes: projectMatchNodes(input.frameDocuments),
+    surfaceProbes: input.surfaceProbes,
+  };
+}

packages/dsh-plugin-browserskill/src/observation.ts

@@ -12,7 +12,7 @@
  * events, and never move the registry's current pointer.
  */
 
-import { createHash } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
 import { readFile, unlink } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
@@ -78,6 +78,7 @@ const FAILURE_BACKOFF_THRESHOLD = 3;
 const FRAME_RING_SIZE = 2;
 
 export class ObservationService {
+  private readonly scratchNamespace = randomUUID();
   private readonly observations = new Map<string, SessionObservation>();
   private readonly listeners = new Set<(event: ObservationEvent) => void>();
   private readonly captureTimers = new Map<string, unknown>();
@@ -304,9 +305,9 @@ export class ObservationService {
     if (current === undefined || current.dead === true) return;
     if (this.captureInFlight.has(sessionId)) return;
     this.captureInFlight.add(sessionId);
-    // One fixed scratch path per session (overwritten each frame) and always
-    // unlinked after the bytes are read — /tmp must not grow without bound.
-    const outPath = join(tmpdir(), `bsk-obs-${sessionId}.png`);
+    // Stable within this service, isolated from captures owned by other instances.
+    const sessionKey = createHash("sha256").update(sessionId).digest("hex").slice(0, 16);
+    const outPath = join(tmpdir(), `bsk-obs-${this.scratchNamespace}-${sessionKey}.png`);
     let writtenPath = outPath;
     try {
       const result = await this.deps.queue.run(sessionId, () =>

packages/dsh-plugin-browserskill/tests/observation.test.ts

@@ -5,7 +5,7 @@
 import { existsSync, mkdtempSync, writeFileSync } from "node:fs";
 import type { IncomingMessage, ServerResponse } from "node:http";
 import { tmpdir } from "node:os";
-import { join } from "node:path";
+import { basename, dirname } from "node:path";
 import { describe, expect, it } from "vitest";
 import {
   actionForLabel,
@@ -548,8 +548,12 @@ describe("scratch files and thumbnail references", () => {
     );
     const firstOut = runner.calls.filter((c) => c.args[0] === "screenshot").at(-1)?.args;
     const pathOf = (args: string[]) => args[args.indexOf("--out") + 1];
-    expect(pathOf(firstOut ?? [])).toBe(join(tmpdir(), "bsk-obs-s1.png"));
-    expect(existsSync(pathOf(firstOut ?? []))).toBe(false);
+    const firstPath = pathOf(firstOut ?? []);
+    expect(dirname(firstPath)).toBe(tmpdir());
+    expect(basename(firstPath)).toMatch(
+      /^bsk-obs-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-[0-9a-f]{16}\.png$/,
+    );
+    expect(existsSync(firstPath)).toBe(false);
     // second frame: same path, deleted again
     service.endAction("s1");
     scheduler.runNext();
@@ -559,11 +563,33 @@ describe("scratch files and thumbnail references", () => {
         scheduler.pending().length === 1,
     );
     const secondOut = runner.calls.filter((c) => c.args[0] === "screenshot").at(-1)?.args;
-    expect(pathOf(secondOut ?? [])).toBe(join(tmpdir(), "bsk-obs-s1.png"));
-    await waitFor(() => !existsSync(pathOf(secondOut ?? [])));
+    const secondPath = pathOf(secondOut ?? []);
+    expect(secondPath).toBe(firstPath);
+    await waitFor(() => !existsSync(secondPath));
     service.dispose();
   });
 
+  it("isolates scratch paths between service instances", async () => {
+    const first = setup({ scheduler: fakeScheduler(), runner: fakeRunner() });
+    const second = setup({ scheduler: fakeScheduler(), runner: fakeRunner() });
+    first.service.addSession("s1");
+    second.service.addSession("s1");
+    first.scheduler.runNext();
+    second.scheduler.runNext();
+    await waitFor(
+      () =>
+        first.service.getState()[0]?.thumbnailAttachmentId !== undefined &&
+        second.service.getState()[0]?.thumbnailAttachmentId !== undefined,
+    );
+    const pathOf = (runner: FakeRunner) => {
+      const args = runner.calls.find((call) => call.args[0] === "screenshot")?.args ?? [];
+      return args[args.indexOf("--out") + 1];
+    };
+    expect(pathOf(first.runner)).not.toBe(pathOf(second.runner));
+    first.service.dispose();
+    second.service.dispose();
+  });
+
   it("keeps the last two distinct frames and evicts the third", async () => {
     let n = 0;
     const runner = fakeRunner({

packages/vom/src/__tests__/render.test.ts

@@ -24,6 +24,10 @@ function scene(nodes: VomNode[]): VomScene {
   };
 }
 
+function coreRefs(refs: ReturnType<typeof renderVom>["refs"]) {
+  return refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }));
+}
+
 describe("renderVom single-layer page", () => {
   it("does not derive handle context across frame scopes", () => {
     const out = renderVom(
@@ -130,7 +134,7 @@ describe("renderVom single-layer page", () => {
         '      @e2 button "加入购物车"',
       ].join("\n"),
     );
-    expect(out.refs).toEqual([
+    expect(coreRefs(out.refs)).toEqual([
       { ref: "e1", backendNodeId: 3 },
       { ref: "e2", backendNodeId: 6 },
     ]);
@@ -151,7 +155,7 @@ describe("renderVom single-layer page", () => {
     expect(out.text).toContain('    heading "Title"');
     expect(out.text).toContain('    img "Logo"');
     expect(out.text).toContain('    @e1 button "Continue"');
-    expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
+    expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
   });
 
   it("assigns refs to listbox controls", () => {
@@ -163,7 +167,7 @@ describe("renderVom single-layer page", () => {
     );
 
     expect(out.text).toContain('    @e1 listbox "Choices"');
-    expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
+    expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
   });
 
   it("annotates external links regardless of role casing", () => {
@@ -200,7 +204,7 @@ describe("renderVom single-layer page", () => {
     expect(out.text).toContain("    alertdialog");
     expect(out.text).toContain("      section");
     expect(out.text).toContain('        @e1 button "Close"');
-    expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 5 }]);
+    expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 5 }]);
   });
 
   it("skips generic nodes without adding an extra indentation level", () => {
@@ -291,7 +295,7 @@ describe("renderVom single-layer page", () => {
     expect(out.text).toContain("    section");
     expect(out.text).not.toContain("Too deep");
     expect(out.text).toContain('    @e1 button "Later"');
-    expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
+    expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
   });
 
   it("truncates when maxTokens is exceeded while keeping refs in sync with rendered text", () => {
@@ -326,7 +330,7 @@ describe("renderVom single-layer page", () => {
     );
 
     expect(out.text).toContain('@e1 button "Open settings"');
-    expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
+    expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
   });
 
   it("keeps the deepest custom control when wrapper and child both look clickable", () => {
@@ -356,7 +360,7 @@ describe("renderVom single-layer page", () => {
 
     expect(out.text).not.toContain('@e1 button "Card"');
     expect(out.text).toContain('@e1 button "Details"');
-    expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]);
+    expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
   });
 
   it("does not recover clickable wrappers around native controls", () => {
@@ -379,7 +383,7 @@ describe("renderVom single-layer page", () => {
 
     expect(out.text).not.toContain('@e1 button "Checkout"');
     expect(out.text).toContain('@e1 button "Pay"');
-    expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]);
+    expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
   });
 
   it("adds context to duplicate weak action labels", () => {
@@ -442,7 +446,7 @@ describe("renderVom single-layer page", () => {
 
     expect(out.text).toContain("@layers 1 focus=L1");
     expect(out.text).toContain('@e1 button "Background"');
-    expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
+    expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
   });
 
   it("can filter refs blocked by active positioned regions without changing layer output", () => {
@@ -485,7 +489,7 @@ describe("renderVom single-layer page", () => {
     expect(out.text).not.toContain("active-region");
     expect(out.text).not.toContain("Background");
     expect(out.text).toContain('@e1 button "Foreground"');
-    expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
+    expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
   });
 
   it("keeps paint-order comparisons inside their frame document", () => {
@@ -600,7 +604,7 @@ describe("renderVom single-layer page", () => {
     });
 
     expect(out.text).toContain('@e1 button "Products" [hover first: Shoes | Bags | Accessories]');
-    expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
+    expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
   });
 
   it("injects active scope blocks immediately after their trigger without refs", () => {
@@ -626,7 +630,7 @@ describe("renderVom single-layer page", () => {
         "        Bob - great sound",
       ].join("\n"),
     );
-    expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
+    expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
   });
 
   it("does not render redundant children inside named ref controls", () => {
@@ -640,7 +644,7 @@ describe("renderVom single-layer page", () => {
 
     expect(out.text).toContain('@e1 button "Save"');
     expect(out.text).not.toContain("disk icon");
-    expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
+    expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
   });
 
   it("does not render redundant children inside native text inputs", () => {
@@ -807,7 +811,7 @@ describe("renderVom double-layer page", () => {
       ].join("\n"),
     );
     expect(out.text).not.toContain("底层按钮");
-    expect(out.refs).toEqual([
+    expect(coreRefs(out.refs)).toEqual([
       { ref: "e1", backendNodeId: 4 },
       { ref: "e2", backendNodeId: 5 },
     ]);
@@ -859,4 +863,36 @@ describe("renderVom double-layer page", () => {
     expect(out.text).toContain('    @e2 textbox "密码" [filled] ="•••"');
     expect(out.refs.map((r) => r.backendNodeId)).toEqual([101, 102]);
   });
+
+  it("redacts form values and returns semantic metadata for rendered refs", () => {
+    const out = renderVom(
+      scene([
+        node({ id: 1, role: "RootWebArea", name: "Form" }),
+        node({
+          id: 2,
+          backendNodeId: 42,
+          frameId: "child-frame",
+          parentId: 1,
+          role: "textbox",
+          name: "Email",
+          value: "user@example.com",
+          inputState: "filled",
+          tag: "input",
+        }),
+      ]),
+      { redactValues: true },
+    );
+
+    expect(out.text).toContain('[filled] ="•••"');
+    expect(out.text).not.toContain("user@example.com");
+    expect(out.refs[0]).toMatchObject({
+      ref: "e1",
+      backendNodeId: 42,
+      frameId: "child-frame",
+      role: "textbox",
+      name: "Email",
+      line: expect.any(Number),
+    });
+    expect(out.text.split("\n")[out.refs[0]?.line ?? -1]).toContain('@e1 textbox "Email"');
+  });
 });

packages/vom/src/index.ts

@@ -5,6 +5,7 @@ export type {
   CondSurface,
   LayerKind,
   Rect,
+  RenderedRef,
   Viewport,
   VomNode,
   VomOptions,

packages/vom/src/render.ts

@@ -4,9 +4,9 @@ import type {
   BlockingLayer,
   CondSurface,
   Rect,
+  RenderedRef,
   VomNode,
   VomOptions,
-  VomRef,
   VomResult,
   VomScene,
 } from "./types";
@@ -615,6 +615,7 @@ function renderNodeLine(
   ref: string | undefined,
   context: string[] = [],
   surface: CondSurface | undefined = undefined,
+  redactValues = false,
 ): string {
   let line = `${"  ".repeat(depth)}${ref ? `@${ref} ` : ""}${node.role}`;
 
@@ -636,13 +637,14 @@ function renderNodeLine(
   }
   const placeholder = cleaned(node.placeholder);
   if (placeholder) line += ` placeholder=${JSON.stringify(placeholder)}`;
-  const sensitiveHasValue =
+  const hasValue =
     rawValue !== undefined || node.inputState === "filled" || node.inputState === "default";
-  const value = node.sensitive
-    ? sensitiveHasValue
-      ? SENSITIVE_MASK
-      : undefined
-    : rawValue?.slice(0, MAX_VALUE_LENGTH);
+  const value =
+    node.sensitive || redactValues
+      ? hasValue
+        ? SENSITIVE_MASK
+        : undefined
+      : rawValue?.slice(0, MAX_VALUE_LENGTH);
   if (value !== undefined) line += ` =${JSON.stringify(value)}`;
 
   if (surface && surface.subItems.length > 0) {
@@ -695,11 +697,12 @@ function shouldSkipRedundantChildren(node: VomNode, state: RenderState): boolean
 
 interface RenderState {
   lines: string[];
-  refs: VomRef[];
+  refs: RenderedRef[];
   nextRef: number;
   tokens: number;
   maxDepth: number;
   maxTokens: number;
+  redactValues: boolean;
   truncated: boolean;
   stopped: boolean;
   children: Map<number | null, VomNode[]>;
@@ -764,7 +767,7 @@ function renderTree(
     const ref = isVomReferenceNode(node) ? `e${state.nextRef}` : undefined;
     const context = ref ? handleContext(node, state) : [];
     const surface = state.surfaceMap.get(node.id);
-    const line = renderNodeLine(node, depth, ref, context, surface);
+    const line = renderNodeLine(node, depth, ref, context, surface, state.redactValues);
     const nextTokens = state.tokens + estimateTokens(line);
     if (nextTokens > state.maxTokens) {
       state.truncated = true;
@@ -775,10 +778,15 @@ function renderTree(
     state.lines.push(line);
     state.tokens = nextTokens;
     if (ref) {
+      const name = cleaned(node.name);
       state.refs.push({
         ref,
         backendNodeId: node.backendNodeId ?? node.id,
         ...(node.frameId ? { frameId: node.frameId } : {}),
+        ...(node.role ? { role: node.role } : {}),
+        ...(name ? { name } : {}),
+        ...(context.length > 0 ? { ctx: context.join(" > ") } : {}),
+        line: state.lines.length - 1,
       });
       state.nextRef += 1;
     }
@@ -808,6 +816,7 @@ function renderNodes(
     tokens: estimateTokens(initialLines.join("\n")),
     maxDepth: options.maxDepth ?? DEFAULT_MAX_DEPTH,
     maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
+    redactValues: options.redactValues ?? false,
     truncated: false,
     stopped: false,
     children,

packages/vom/src/types.ts

@@ -64,6 +64,10 @@ export interface VomScene {
 export interface VomOptions {
   maxDepth?: number;
   maxTokens?: number;
+  /**
+   * When true, form values are rendered as masks instead of literal values.
+   */
+  redactValues?: boolean;
   /**
    * Experimental: filter refs that are geometrically blocked by foreground
    * fixed/absolute/sticky regions. Disabled by default and does not alter the
@@ -90,9 +94,17 @@ export interface VomRef {
   frameId?: string;
 }
 
+export interface RenderedRef extends VomRef {
+  role?: string;
+  name?: string;
+  ctx?: string;
+  /** Zero-based line index in `VomResult.text`. */
+  line: number;
+}
+
 export interface VomResult {
   text: string;
-  refs: VomRef[];
+  refs: RenderedRef[];
   truncated: boolean;
 }