feat(vom): expose record-safe semantic observations

Share the VOM capture pipeline with recording, enrich rendered refs, and support value redaction without regressing cooperative cancellation.
This commit is contained in:
“haonan”
2026-08-19 15:05:55 +08:00
committed by GitHub
parent 6b78211e79
commit 45a246c41b
7 changed files with 217 additions and 69 deletions
@@ -8,6 +8,7 @@ import {
buildFrameVomScene,
buildVomScene,
type CdpAxNode,
captureVomObservation,
handleGetHtml,
handleObserve,
handleScreenshot,
@@ -1329,7 +1330,7 @@ describe("buildVomScene", () => {
expect(rendered).toContain(' @e2 textbox "验证码"');
});
it("does not mark non-password inputs sensitive from password-like labels", () => {
it("marks current-password autocomplete as sensitive even for text inputs", () => {
const axNodes: CdpAxNode[] = [
{
nodeId: "1",
@@ -1364,7 +1365,7 @@ describe("buildVomScene", () => {
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
expect.objectContaining({
id: 10,
sensitive: false,
sensitive: true,
}),
);
});
@@ -1659,7 +1660,9 @@ describe("buildVomScene", () => {
);
const rendered = renderVom(scene);
expect(rendered.text).toContain('@e1 button "close"');
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
{ ref: "e1", backendNodeId: 20 },
]);
});
it("does not promote a clickable container that wraps a real interactive control", () => {
@@ -1803,7 +1806,9 @@ describe("buildVomScene", () => {
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic");
const rendered = renderVom(scene);
expect(rendered.text).toContain('@e1 button "收藏"');
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
{ ref: "e1", backendNodeId: 20 },
]);
});
it("builds active scope blocks from active aria-controls relationships", () => {
@@ -2908,6 +2913,41 @@ describe("handleSnapshot", () => {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
};
it("exposes the frame-aware observation result for recording", async () => {
const ax: CdpAxNode[] = [
{
nodeId: "root",
role: { type: "role", value: "RootWebArea" },
backendDOMNodeId: 11,
childIds: ["password"],
},
{
nodeId: "password",
parentId: "root",
role: { type: "role", value: "textbox" },
name: { type: "computedString", value: "Password" },
value: { value: "hunter2" },
backendDOMNodeId: 13,
},
];
const deps = makeOverlayDeps(ax, loginSnapshotReply(), VP_METRICS);
const result = await captureVomObservation(deps.cdp, 4, "https://example.com", {
redactValues: true,
});
expect(result.text).not.toContain("hunter2");
expect(result.text).toContain("•••");
expect(result.refs[0]).toMatchObject({
backendNodeId: 13,
role: "textbox",
name: "Password",
line: expect.any(Number),
});
expect(result.captured.some((node) => node.backendNodeId === 13)).toBe(true);
expect(result.documents).toHaveLength(1);
});
it("renders a blocking login overlay as the focused top layer", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11");
@@ -0,0 +1,5 @@
export {
type CaptureVomObservationOptions,
type CaptureVomObservationResult,
captureVomObservation,
} from "./observation";
+86 -41
View File
@@ -8,8 +8,10 @@ import {
applyVomInteractionRecovery,
type CondSurface,
isVomReferenceNode,
type RenderedRef,
renderVom,
type VomNode,
type VomOptions,
type VomScene,
} from "@browser-skill/vom";
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
@@ -48,6 +50,7 @@ import {
import { resolveSnapshotRef } from "./snapshot-ref";
import {
type CapturedNode,
type CapturedSurfaceProbe,
type CapturedViewModel,
captureViewModel,
collectOverlayExcludedBackendIds,
@@ -411,7 +414,15 @@ function isModalSignal(
function isSensitive(capturedNode: CapturedNode | undefined): boolean {
const attrs = capturedNode?.attrs ?? {};
return (attrs.type ?? "").toLowerCase() === "password";
const type = (attrs.type ?? "").toLowerCase();
if (type === "password") return true;
const autocomplete = (attrs.autocomplete ?? "").toLowerCase();
return (
autocomplete.startsWith("cc-") ||
autocomplete === "one-time-code" ||
autocomplete === "current-password" ||
autocomplete === "new-password"
);
}
interface AxNodeSignals {
@@ -461,7 +472,13 @@ const AX_TEXT_STOP_ROLES = new Set([
"columnheader",
"rowheader",
]);
const SENSITIVE_AX_INPUT_TYPES = new Set(["password", "credit-card"]);
const SENSITIVE_AX_INPUT_TYPES = new Set([
"password",
"credit-card",
"one-time-code",
"current-password",
"new-password",
]);
function axPropertyString(axNode: CdpAxNode, name: string): string | undefined {
const prop = axNode.properties?.find((item) => item.name === name);
@@ -1537,14 +1554,13 @@ async function fallbackCapturedViewModel(
try {
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
throwIfAborted(signal, "observation");
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
const source = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
viewport = {
width: vpSrc.clientWidth ?? 0,
height: vpSrc.clientHeight ?? 0,
width: source.clientWidth ?? 0,
height: source.clientHeight ?? 0,
};
} catch (err) {
if (isAbortError(err)) throw err;
// viewport stays zero-sized
} catch (error) {
if (isAbortError(error)) throw error;
}
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal);
throwIfAborted(signal, "observation");
@@ -1554,22 +1570,69 @@ async function fallbackCapturedViewModel(
async function captureForVom(
cdp: CdpRunner,
tabId: number,
conditionalSurfaceProbe: boolean,
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>,
signal?: AbortSignal,
options: CaptureVomObservationOptions,
): Promise<CapturedViewModel> {
try {
return await captureViewModel(cdp, tabId, {
conditionalSurfaceProbe,
hoverProbeBypassOverlay,
signal,
conditionalSurfaceProbe: options.conditionalSurfaceProbe ?? false,
hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
signal: options.signal,
});
} catch (err) {
if (isAbortError(err)) throw err;
return fallbackCapturedViewModel(cdp, tabId, signal);
} catch (error) {
if (isAbortError(error)) throw error;
return fallbackCapturedViewModel(cdp, tabId, options.signal);
}
}
export interface CaptureVomObservationOptions extends VomOptions {
conditionalSurfaceProbe?: boolean;
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
signal?: AbortSignal;
}
export interface CaptureVomObservationResult {
text: string;
refs: RenderedRef[];
truncated: boolean;
captured: CapturedNode[];
documents: CapturedFrameDocument<CdpAxNode>[];
surfaceProbes?: CapturedSurfaceProbe[];
}
export async function captureVomObservation(
cdp: CdpRunner,
tabId: number,
url: string | undefined,
options: CaptureVomObservationOptions = {},
): Promise<CaptureVomObservationResult> {
throwIfAborted(options.signal, "observation");
await cdp.ensureAttachedToUrl?.(tabId, url);
throwIfAborted(options.signal, "observation");
const captured = await captureForVom(cdp, tabId, options);
throwIfAborted(options.signal, "observation");
const documents = await captureFrameData<CdpAxNode>(cdp, tabId, captured, options.signal);
throwIfAborted(options.signal, "observation");
const scene =
captured.rootFrameId && captured.frameNodes?.size
? buildFrameVomScene(documents, captured, { pageUrl: url })
: buildVomScene(documents[0]?.axNodes ?? [], captured, { pageUrl: url });
const rendered = renderVom(scene, {
maxDepth: options.maxDepth,
maxTokens: options.maxTokens,
redactValues: options.redactValues,
activeRegionPolicy: options.activeRegionPolicy,
});
throwIfAborted(options.signal, "observation");
return {
text: rendered.text,
refs: rendered.refs,
truncated: rendered.truncated,
captured: captured.nodes,
documents,
surfaceProbes: captured.surfaceProbes,
};
}
async function handleVomObservation(
manager: SessionManager,
params: SnapshotParams | ObserveParams,
@@ -1599,37 +1662,19 @@ async function handleVomObservation(
try {
throwIfAborted(signal, toolName);
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
throwIfAborted(signal, toolName);
const effectiveConditionalSurfaceProbe =
deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
const captured = await captureForVom(
deps.cdp,
target.tabId,
effectiveConditionalSurfaceProbe,
deps.hoverProbeBypassOverlay,
signal,
);
throwIfAborted(signal, toolName);
const frameDocuments = await captureFrameData<CdpAxNode>(
deps.cdp,
target.tabId,
captured,
signal,
);
throwIfAborted(signal, toolName);
const scene =
captured.rootFrameId && captured.frameNodes?.size
? buildFrameVomScene(frameDocuments, captured, { pageUrl: target.url })
: buildVomScene(frameDocuments[0]?.axNodes ?? [], captured, { pageUrl: target.url });
const rendered = renderVom(scene, {
const rendered = await captureVomObservation(deps.cdp, target.tabId, target.url, {
maxDepth: params.max_depth,
maxTokens: params.max_tokens,
activeRegionPolicy: true,
conditionalSurfaceProbe: effectiveConditionalSurfaceProbe,
hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay,
signal,
});
throwIfAborted(signal, toolName);
const targetByFrameId = new Map(
frameDocuments.map((document) => [document.frameId, document.target]),
rendered.documents.map((document) => [document.frameId, document.target]),
);
ctx.refStore.replace(
rendered.refs.map((ref) => {
@@ -1653,7 +1698,7 @@ async function handleVomObservation(
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
? {
debug: {
surface_probes: (captured.surfaceProbes ?? []).map((probe) => ({
surface_probes: (rendered.surfaceProbes ?? []).map((probe) => ({
trigger_backend_node_id: probe.triggerBackendNodeId,
...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}),
trigger_action: probe.triggerAction,
+50 -14
View File
@@ -24,6 +24,10 @@ function scene(nodes: VomNode[]): VomScene {
};
}
function coreRefs(refs: ReturnType<typeof renderVom>["refs"]) {
return refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }));
}
describe("renderVom single-layer page", () => {
it("does not derive handle context across frame scopes", () => {
const out = renderVom(
@@ -130,7 +134,7 @@ describe("renderVom single-layer page", () => {
' @e2 button "加入购物车"',
].join("\n"),
);
expect(out.refs).toEqual([
expect(coreRefs(out.refs)).toEqual([
{ ref: "e1", backendNodeId: 3 },
{ ref: "e2", backendNodeId: 6 },
]);
@@ -151,7 +155,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).toContain(' heading "Title"');
expect(out.text).toContain(' img "Logo"');
expect(out.text).toContain(' @e1 button "Continue"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
});
it("assigns refs to listbox controls", () => {
@@ -163,7 +167,7 @@ describe("renderVom single-layer page", () => {
);
expect(out.text).toContain(' @e1 listbox "Choices"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
});
it("annotates external links regardless of role casing", () => {
@@ -200,7 +204,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).toContain(" alertdialog");
expect(out.text).toContain(" section");
expect(out.text).toContain(' @e1 button "Close"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 5 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 5 }]);
});
it("skips generic nodes without adding an extra indentation level", () => {
@@ -291,7 +295,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).toContain(" section");
expect(out.text).not.toContain("Too deep");
expect(out.text).toContain(' @e1 button "Later"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
});
it("truncates when maxTokens is exceeded while keeping refs in sync with rendered text", () => {
@@ -326,7 +330,7 @@ describe("renderVom single-layer page", () => {
);
expect(out.text).toContain('@e1 button "Open settings"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
});
it("keeps the deepest custom control when wrapper and child both look clickable", () => {
@@ -356,7 +360,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).not.toContain('@e1 button "Card"');
expect(out.text).toContain('@e1 button "Details"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
});
it("does not recover clickable wrappers around native controls", () => {
@@ -379,7 +383,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).not.toContain('@e1 button "Checkout"');
expect(out.text).toContain('@e1 button "Pay"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
});
it("adds context to duplicate weak action labels", () => {
@@ -442,7 +446,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).toContain("@layers 1 focus=L1");
expect(out.text).toContain('@e1 button "Background"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
});
it("can filter refs blocked by active positioned regions without changing layer output", () => {
@@ -485,7 +489,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).not.toContain("active-region");
expect(out.text).not.toContain("Background");
expect(out.text).toContain('@e1 button "Foreground"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
});
it("keeps paint-order comparisons inside their frame document", () => {
@@ -600,7 +604,7 @@ describe("renderVom single-layer page", () => {
});
expect(out.text).toContain('@e1 button "Products" [hover first: Shoes | Bags | Accessories]');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
});
it("injects active scope blocks immediately after their trigger without refs", () => {
@@ -626,7 +630,7 @@ describe("renderVom single-layer page", () => {
" Bob - great sound",
].join("\n"),
);
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
});
it("does not render redundant children inside named ref controls", () => {
@@ -640,7 +644,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).toContain('@e1 button "Save"');
expect(out.text).not.toContain("disk icon");
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
});
it("does not render redundant children inside native text inputs", () => {
@@ -807,7 +811,7 @@ describe("renderVom double-layer page", () => {
].join("\n"),
);
expect(out.text).not.toContain("底层按钮");
expect(out.refs).toEqual([
expect(coreRefs(out.refs)).toEqual([
{ ref: "e1", backendNodeId: 4 },
{ ref: "e2", backendNodeId: 5 },
]);
@@ -859,4 +863,36 @@ describe("renderVom double-layer page", () => {
expect(out.text).toContain(' @e2 textbox "密码" [filled] ="•••"');
expect(out.refs.map((r) => r.backendNodeId)).toEqual([101, 102]);
});
it("redacts form values and returns semantic metadata for rendered refs", () => {
const out = renderVom(
scene([
node({ id: 1, role: "RootWebArea", name: "Form" }),
node({
id: 2,
backendNodeId: 42,
frameId: "child-frame",
parentId: 1,
role: "textbox",
name: "Email",
value: "user@example.com",
inputState: "filled",
tag: "input",
}),
]),
{ redactValues: true },
);
expect(out.text).toContain('[filled] ="•••"');
expect(out.text).not.toContain("user@example.com");
expect(out.refs[0]).toMatchObject({
ref: "e1",
backendNodeId: 42,
frameId: "child-frame",
role: "textbox",
name: "Email",
line: expect.any(Number),
});
expect(out.text.split("\n")[out.refs[0]?.line ?? -1]).toContain('@e1 textbox "Email"');
});
});
+1
View File
@@ -5,6 +5,7 @@ export type {
CondSurface,
LayerKind,
Rect,
RenderedRef,
Viewport,
VomNode,
VomOptions,
+18 -9
View File
@@ -4,9 +4,9 @@ import type {
BlockingLayer,
CondSurface,
Rect,
RenderedRef,
VomNode,
VomOptions,
VomRef,
VomResult,
VomScene,
} from "./types";
@@ -615,6 +615,7 @@ function renderNodeLine(
ref: string | undefined,
context: string[] = [],
surface: CondSurface | undefined = undefined,
redactValues = false,
): string {
let line = `${" ".repeat(depth)}${ref ? `@${ref} ` : ""}${node.role}`;
@@ -636,13 +637,14 @@ function renderNodeLine(
}
const placeholder = cleaned(node.placeholder);
if (placeholder) line += ` placeholder=${JSON.stringify(placeholder)}`;
const sensitiveHasValue =
const hasValue =
rawValue !== undefined || node.inputState === "filled" || node.inputState === "default";
const value = node.sensitive
? sensitiveHasValue
? SENSITIVE_MASK
: undefined
: rawValue?.slice(0, MAX_VALUE_LENGTH);
const value =
node.sensitive || redactValues
? hasValue
? SENSITIVE_MASK
: undefined
: rawValue?.slice(0, MAX_VALUE_LENGTH);
if (value !== undefined) line += ` =${JSON.stringify(value)}`;
if (surface && surface.subItems.length > 0) {
@@ -695,11 +697,12 @@ function shouldSkipRedundantChildren(node: VomNode, state: RenderState): boolean
interface RenderState {
lines: string[];
refs: VomRef[];
refs: RenderedRef[];
nextRef: number;
tokens: number;
maxDepth: number;
maxTokens: number;
redactValues: boolean;
truncated: boolean;
stopped: boolean;
children: Map<number | null, VomNode[]>;
@@ -764,7 +767,7 @@ function renderTree(
const ref = isVomReferenceNode(node) ? `e${state.nextRef}` : undefined;
const context = ref ? handleContext(node, state) : [];
const surface = state.surfaceMap.get(node.id);
const line = renderNodeLine(node, depth, ref, context, surface);
const line = renderNodeLine(node, depth, ref, context, surface, state.redactValues);
const nextTokens = state.tokens + estimateTokens(line);
if (nextTokens > state.maxTokens) {
state.truncated = true;
@@ -775,10 +778,15 @@ function renderTree(
state.lines.push(line);
state.tokens = nextTokens;
if (ref) {
const name = cleaned(node.name);
state.refs.push({
ref,
backendNodeId: node.backendNodeId ?? node.id,
...(node.frameId ? { frameId: node.frameId } : {}),
...(node.role ? { role: node.role } : {}),
...(name ? { name } : {}),
...(context.length > 0 ? { ctx: context.join(" > ") } : {}),
line: state.lines.length - 1,
});
state.nextRef += 1;
}
@@ -808,6 +816,7 @@ function renderNodes(
tokens: estimateTokens(initialLines.join("\n")),
maxDepth: options.maxDepth ?? DEFAULT_MAX_DEPTH,
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
redactValues: options.redactValues ?? false,
truncated: false,
stopped: false,
children,
+13 -1
View File
@@ -64,6 +64,10 @@ export interface VomScene {
export interface VomOptions {
maxDepth?: number;
maxTokens?: number;
/**
* When true, form values are rendered as masks instead of literal values.
*/
redactValues?: boolean;
/**
* Experimental: filter refs that are geometrically blocked by foreground
* fixed/absolute/sticky regions. Disabled by default and does not alter the
@@ -90,9 +94,17 @@ export interface VomRef {
frameId?: string;
}
export interface RenderedRef extends VomRef {
role?: string;
name?: string;
ctx?: string;
/** Zero-based line index in `VomResult.text`. */
line: number;
}
export interface VomResult {
text: string;
refs: VomRef[];
refs: RenderedRef[];
truncated: boolean;
}