feat(vom): expose record-safe semantic observations

Share the VOM capture pipeline with recording, enrich rendered refs, and support value redaction without regressing cooperative cancellation.
This commit is contained in:
“haonan”
2026-08-19 15:05:55 +08:00
committed by GitHub
parent 6b78211e79
commit 45a246c41b
7 changed files with 217 additions and 69 deletions
@@ -8,6 +8,7 @@ import {
buildFrameVomScene, buildFrameVomScene,
buildVomScene, buildVomScene,
type CdpAxNode, type CdpAxNode,
captureVomObservation,
handleGetHtml, handleGetHtml,
handleObserve, handleObserve,
handleScreenshot, handleScreenshot,
@@ -1329,7 +1330,7 @@ describe("buildVomScene", () => {
expect(rendered).toContain(' @e2 textbox "验证码"'); expect(rendered).toContain(' @e2 textbox "验证码"');
}); });
it("does not mark non-password inputs sensitive from password-like labels", () => { it("marks current-password autocomplete as sensitive even for text inputs", () => {
const axNodes: CdpAxNode[] = [ const axNodes: CdpAxNode[] = [
{ {
nodeId: "1", nodeId: "1",
@@ -1364,7 +1365,7 @@ describe("buildVomScene", () => {
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual( expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
expect.objectContaining({ expect.objectContaining({
id: 10, id: 10,
sensitive: false, sensitive: true,
}), }),
); );
}); });
@@ -1659,7 +1660,9 @@ describe("buildVomScene", () => {
); );
const rendered = renderVom(scene); const rendered = renderVom(scene);
expect(rendered.text).toContain('@e1 button "close"'); expect(rendered.text).toContain('@e1 button "close"');
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]); expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
{ ref: "e1", backendNodeId: 20 },
]);
}); });
it("does not promote a clickable container that wraps a real interactive control", () => { it("does not promote a clickable container that wraps a real interactive control", () => {
@@ -1803,7 +1806,9 @@ describe("buildVomScene", () => {
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic"); expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic");
const rendered = renderVom(scene); const rendered = renderVom(scene);
expect(rendered.text).toContain('@e1 button "收藏"'); expect(rendered.text).toContain('@e1 button "收藏"');
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]); expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
{ ref: "e1", backendNodeId: 20 },
]);
}); });
it("builds active scope blocks from active aria-controls relationships", () => { it("builds active scope blocks from active aria-controls relationships", () => {
@@ -2908,6 +2913,41 @@ describe("handleSnapshot", () => {
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 }, cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
}; };
it("exposes the frame-aware observation result for recording", async () => {
const ax: CdpAxNode[] = [
{
nodeId: "root",
role: { type: "role", value: "RootWebArea" },
backendDOMNodeId: 11,
childIds: ["password"],
},
{
nodeId: "password",
parentId: "root",
role: { type: "role", value: "textbox" },
name: { type: "computedString", value: "Password" },
value: { value: "hunter2" },
backendDOMNodeId: 13,
},
];
const deps = makeOverlayDeps(ax, loginSnapshotReply(), VP_METRICS);
const result = await captureVomObservation(deps.cdp, 4, "https://example.com", {
redactValues: true,
});
expect(result.text).not.toContain("hunter2");
expect(result.text).toContain("•••");
expect(result.refs[0]).toMatchObject({
backendNodeId: 13,
role: "textbox",
name: "Password",
line: expect.any(Number),
});
expect(result.captured.some((node) => node.backendNodeId === 13)).toBe(true);
expect(result.documents).toHaveLength(1);
});
it("renders a blocking login overlay as the focused top layer", async () => { it("renders a blocking login overlay as the focused top layer", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) }); const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
await sm.start("aa11"); await sm.start("aa11");
@@ -0,0 +1,5 @@
export {
type CaptureVomObservationOptions,
type CaptureVomObservationResult,
captureVomObservation,
} from "./observation";
+86 -41
View File
@@ -8,8 +8,10 @@ import {
applyVomInteractionRecovery, applyVomInteractionRecovery,
type CondSurface, type CondSurface,
isVomReferenceNode, isVomReferenceNode,
type RenderedRef,
renderVom, renderVom,
type VomNode, type VomNode,
type VomOptions,
type VomScene, type VomScene,
} from "@browser-skill/vom"; } from "@browser-skill/vom";
import { ChromiumCdp } from "@/browser-driver/chromium-cdp"; import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
@@ -48,6 +50,7 @@ import {
import { resolveSnapshotRef } from "./snapshot-ref"; import { resolveSnapshotRef } from "./snapshot-ref";
import { import {
type CapturedNode, type CapturedNode,
type CapturedSurfaceProbe,
type CapturedViewModel, type CapturedViewModel,
captureViewModel, captureViewModel,
collectOverlayExcludedBackendIds, collectOverlayExcludedBackendIds,
@@ -411,7 +414,15 @@ function isModalSignal(
function isSensitive(capturedNode: CapturedNode | undefined): boolean { function isSensitive(capturedNode: CapturedNode | undefined): boolean {
const attrs = capturedNode?.attrs ?? {}; const attrs = capturedNode?.attrs ?? {};
return (attrs.type ?? "").toLowerCase() === "password"; const type = (attrs.type ?? "").toLowerCase();
if (type === "password") return true;
const autocomplete = (attrs.autocomplete ?? "").toLowerCase();
return (
autocomplete.startsWith("cc-") ||
autocomplete === "one-time-code" ||
autocomplete === "current-password" ||
autocomplete === "new-password"
);
} }
interface AxNodeSignals { interface AxNodeSignals {
@@ -461,7 +472,13 @@ const AX_TEXT_STOP_ROLES = new Set([
"columnheader", "columnheader",
"rowheader", "rowheader",
]); ]);
const SENSITIVE_AX_INPUT_TYPES = new Set(["password", "credit-card"]); const SENSITIVE_AX_INPUT_TYPES = new Set([
"password",
"credit-card",
"one-time-code",
"current-password",
"new-password",
]);
function axPropertyString(axNode: CdpAxNode, name: string): string | undefined { function axPropertyString(axNode: CdpAxNode, name: string): string | undefined {
const prop = axNode.properties?.find((item) => item.name === name); const prop = axNode.properties?.find((item) => item.name === name);
@@ -1537,14 +1554,13 @@ async function fallbackCapturedViewModel(
try { try {
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {}); const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
throwIfAborted(signal, "observation"); throwIfAborted(signal, "observation");
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {}; const source = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
viewport = { viewport = {
width: vpSrc.clientWidth ?? 0, width: source.clientWidth ?? 0,
height: vpSrc.clientHeight ?? 0, height: source.clientHeight ?? 0,
}; };
} catch (err) { } catch (error) {
if (isAbortError(err)) throw err; if (isAbortError(error)) throw error;
// viewport stays zero-sized
} }
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal); const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal);
throwIfAborted(signal, "observation"); throwIfAborted(signal, "observation");
@@ -1554,22 +1570,69 @@ async function fallbackCapturedViewModel(
async function captureForVom( async function captureForVom(
cdp: CdpRunner, cdp: CdpRunner,
tabId: number, tabId: number,
conditionalSurfaceProbe: boolean, options: CaptureVomObservationOptions,
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>,
signal?: AbortSignal,
): Promise<CapturedViewModel> { ): Promise<CapturedViewModel> {
try { try {
return await captureViewModel(cdp, tabId, { return await captureViewModel(cdp, tabId, {
conditionalSurfaceProbe, conditionalSurfaceProbe: options.conditionalSurfaceProbe ?? false,
hoverProbeBypassOverlay, hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
signal, signal: options.signal,
}); });
} catch (err) { } catch (error) {
if (isAbortError(err)) throw err; if (isAbortError(error)) throw error;
return fallbackCapturedViewModel(cdp, tabId, signal); return fallbackCapturedViewModel(cdp, tabId, options.signal);
} }
} }
export interface CaptureVomObservationOptions extends VomOptions {
conditionalSurfaceProbe?: boolean;
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
signal?: AbortSignal;
}
export interface CaptureVomObservationResult {
text: string;
refs: RenderedRef[];
truncated: boolean;
captured: CapturedNode[];
documents: CapturedFrameDocument<CdpAxNode>[];
surfaceProbes?: CapturedSurfaceProbe[];
}
export async function captureVomObservation(
cdp: CdpRunner,
tabId: number,
url: string | undefined,
options: CaptureVomObservationOptions = {},
): Promise<CaptureVomObservationResult> {
throwIfAborted(options.signal, "observation");
await cdp.ensureAttachedToUrl?.(tabId, url);
throwIfAborted(options.signal, "observation");
const captured = await captureForVom(cdp, tabId, options);
throwIfAborted(options.signal, "observation");
const documents = await captureFrameData<CdpAxNode>(cdp, tabId, captured, options.signal);
throwIfAborted(options.signal, "observation");
const scene =
captured.rootFrameId && captured.frameNodes?.size
? buildFrameVomScene(documents, captured, { pageUrl: url })
: buildVomScene(documents[0]?.axNodes ?? [], captured, { pageUrl: url });
const rendered = renderVom(scene, {
maxDepth: options.maxDepth,
maxTokens: options.maxTokens,
redactValues: options.redactValues,
activeRegionPolicy: options.activeRegionPolicy,
});
throwIfAborted(options.signal, "observation");
return {
text: rendered.text,
refs: rendered.refs,
truncated: rendered.truncated,
captured: captured.nodes,
documents,
surfaceProbes: captured.surfaceProbes,
};
}
async function handleVomObservation( async function handleVomObservation(
manager: SessionManager, manager: SessionManager,
params: SnapshotParams | ObserveParams, params: SnapshotParams | ObserveParams,
@@ -1599,37 +1662,19 @@ async function handleVomObservation(
try { try {
throwIfAborted(signal, toolName); throwIfAborted(signal, toolName);
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
throwIfAborted(signal, toolName);
const effectiveConditionalSurfaceProbe = const effectiveConditionalSurfaceProbe =
deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe; deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
const captured = await captureForVom( const rendered = await captureVomObservation(deps.cdp, target.tabId, target.url, {
deps.cdp,
target.tabId,
effectiveConditionalSurfaceProbe,
deps.hoverProbeBypassOverlay,
signal,
);
throwIfAborted(signal, toolName);
const frameDocuments = await captureFrameData<CdpAxNode>(
deps.cdp,
target.tabId,
captured,
signal,
);
throwIfAborted(signal, toolName);
const scene =
captured.rootFrameId && captured.frameNodes?.size
? buildFrameVomScene(frameDocuments, captured, { pageUrl: target.url })
: buildVomScene(frameDocuments[0]?.axNodes ?? [], captured, { pageUrl: target.url });
const rendered = renderVom(scene, {
maxDepth: params.max_depth, maxDepth: params.max_depth,
maxTokens: params.max_tokens, maxTokens: params.max_tokens,
activeRegionPolicy: true, activeRegionPolicy: true,
conditionalSurfaceProbe: effectiveConditionalSurfaceProbe,
hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay,
signal,
}); });
throwIfAborted(signal, toolName); throwIfAborted(signal, toolName);
const targetByFrameId = new Map( const targetByFrameId = new Map(
frameDocuments.map((document) => [document.frameId, document.target]), rendered.documents.map((document) => [document.frameId, document.target]),
); );
ctx.refStore.replace( ctx.refStore.replace(
rendered.refs.map((ref) => { rendered.refs.map((ref) => {
@@ -1653,7 +1698,7 @@ async function handleVomObservation(
...(toolName === "observe" && (params as ObserveParams).debug_surfaces ...(toolName === "observe" && (params as ObserveParams).debug_surfaces
? { ? {
debug: { debug: {
surface_probes: (captured.surfaceProbes ?? []).map((probe) => ({ surface_probes: (rendered.surfaceProbes ?? []).map((probe) => ({
trigger_backend_node_id: probe.triggerBackendNodeId, trigger_backend_node_id: probe.triggerBackendNodeId,
...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}), ...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}),
trigger_action: probe.triggerAction, trigger_action: probe.triggerAction,
+50 -14
View File
@@ -24,6 +24,10 @@ function scene(nodes: VomNode[]): VomScene {
}; };
} }
function coreRefs(refs: ReturnType<typeof renderVom>["refs"]) {
return refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }));
}
describe("renderVom single-layer page", () => { describe("renderVom single-layer page", () => {
it("does not derive handle context across frame scopes", () => { it("does not derive handle context across frame scopes", () => {
const out = renderVom( const out = renderVom(
@@ -130,7 +134,7 @@ describe("renderVom single-layer page", () => {
' @e2 button "加入购物车"', ' @e2 button "加入购物车"',
].join("\n"), ].join("\n"),
); );
expect(out.refs).toEqual([ expect(coreRefs(out.refs)).toEqual([
{ ref: "e1", backendNodeId: 3 }, { ref: "e1", backendNodeId: 3 },
{ ref: "e2", backendNodeId: 6 }, { ref: "e2", backendNodeId: 6 },
]); ]);
@@ -151,7 +155,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).toContain(' heading "Title"'); expect(out.text).toContain(' heading "Title"');
expect(out.text).toContain(' img "Logo"'); expect(out.text).toContain(' img "Logo"');
expect(out.text).toContain(' @e1 button "Continue"'); expect(out.text).toContain(' @e1 button "Continue"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]); expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
}); });
it("assigns refs to listbox controls", () => { it("assigns refs to listbox controls", () => {
@@ -163,7 +167,7 @@ describe("renderVom single-layer page", () => {
); );
expect(out.text).toContain(' @e1 listbox "Choices"'); expect(out.text).toContain(' @e1 listbox "Choices"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
}); });
it("annotates external links regardless of role casing", () => { it("annotates external links regardless of role casing", () => {
@@ -200,7 +204,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).toContain(" alertdialog"); expect(out.text).toContain(" alertdialog");
expect(out.text).toContain(" section"); expect(out.text).toContain(" section");
expect(out.text).toContain(' @e1 button "Close"'); expect(out.text).toContain(' @e1 button "Close"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 5 }]); expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 5 }]);
}); });
it("skips generic nodes without adding an extra indentation level", () => { it("skips generic nodes without adding an extra indentation level", () => {
@@ -291,7 +295,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).toContain(" section"); expect(out.text).toContain(" section");
expect(out.text).not.toContain("Too deep"); expect(out.text).not.toContain("Too deep");
expect(out.text).toContain(' @e1 button "Later"'); expect(out.text).toContain(' @e1 button "Later"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]); expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
}); });
it("truncates when maxTokens is exceeded while keeping refs in sync with rendered text", () => { it("truncates when maxTokens is exceeded while keeping refs in sync with rendered text", () => {
@@ -326,7 +330,7 @@ describe("renderVom single-layer page", () => {
); );
expect(out.text).toContain('@e1 button "Open settings"'); expect(out.text).toContain('@e1 button "Open settings"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
}); });
it("keeps the deepest custom control when wrapper and child both look clickable", () => { it("keeps the deepest custom control when wrapper and child both look clickable", () => {
@@ -356,7 +360,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).not.toContain('@e1 button "Card"'); expect(out.text).not.toContain('@e1 button "Card"');
expect(out.text).toContain('@e1 button "Details"'); expect(out.text).toContain('@e1 button "Details"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]); expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
}); });
it("does not recover clickable wrappers around native controls", () => { it("does not recover clickable wrappers around native controls", () => {
@@ -379,7 +383,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).not.toContain('@e1 button "Checkout"'); expect(out.text).not.toContain('@e1 button "Checkout"');
expect(out.text).toContain('@e1 button "Pay"'); expect(out.text).toContain('@e1 button "Pay"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]); expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
}); });
it("adds context to duplicate weak action labels", () => { it("adds context to duplicate weak action labels", () => {
@@ -442,7 +446,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).toContain("@layers 1 focus=L1"); expect(out.text).toContain("@layers 1 focus=L1");
expect(out.text).toContain('@e1 button "Background"'); expect(out.text).toContain('@e1 button "Background"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
}); });
it("can filter refs blocked by active positioned regions without changing layer output", () => { it("can filter refs blocked by active positioned regions without changing layer output", () => {
@@ -485,7 +489,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).not.toContain("active-region"); expect(out.text).not.toContain("active-region");
expect(out.text).not.toContain("Background"); expect(out.text).not.toContain("Background");
expect(out.text).toContain('@e1 button "Foreground"'); expect(out.text).toContain('@e1 button "Foreground"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]); expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
}); });
it("keeps paint-order comparisons inside their frame document", () => { it("keeps paint-order comparisons inside their frame document", () => {
@@ -600,7 +604,7 @@ describe("renderVom single-layer page", () => {
}); });
expect(out.text).toContain('@e1 button "Products" [hover first: Shoes | Bags | Accessories]'); expect(out.text).toContain('@e1 button "Products" [hover first: Shoes | Bags | Accessories]');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
}); });
it("injects active scope blocks immediately after their trigger without refs", () => { it("injects active scope blocks immediately after their trigger without refs", () => {
@@ -626,7 +630,7 @@ describe("renderVom single-layer page", () => {
" Bob - great sound", " Bob - great sound",
].join("\n"), ].join("\n"),
); );
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
}); });
it("does not render redundant children inside named ref controls", () => { it("does not render redundant children inside named ref controls", () => {
@@ -640,7 +644,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).toContain('@e1 button "Save"'); expect(out.text).toContain('@e1 button "Save"');
expect(out.text).not.toContain("disk icon"); expect(out.text).not.toContain("disk icon");
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]); expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
}); });
it("does not render redundant children inside native text inputs", () => { it("does not render redundant children inside native text inputs", () => {
@@ -807,7 +811,7 @@ describe("renderVom double-layer page", () => {
].join("\n"), ].join("\n"),
); );
expect(out.text).not.toContain("底层按钮"); expect(out.text).not.toContain("底层按钮");
expect(out.refs).toEqual([ expect(coreRefs(out.refs)).toEqual([
{ ref: "e1", backendNodeId: 4 }, { ref: "e1", backendNodeId: 4 },
{ ref: "e2", backendNodeId: 5 }, { ref: "e2", backendNodeId: 5 },
]); ]);
@@ -859,4 +863,36 @@ describe("renderVom double-layer page", () => {
expect(out.text).toContain(' @e2 textbox "密码" [filled] ="•••"'); expect(out.text).toContain(' @e2 textbox "密码" [filled] ="•••"');
expect(out.refs.map((r) => r.backendNodeId)).toEqual([101, 102]); expect(out.refs.map((r) => r.backendNodeId)).toEqual([101, 102]);
}); });
it("redacts form values and returns semantic metadata for rendered refs", () => {
const out = renderVom(
scene([
node({ id: 1, role: "RootWebArea", name: "Form" }),
node({
id: 2,
backendNodeId: 42,
frameId: "child-frame",
parentId: 1,
role: "textbox",
name: "Email",
value: "user@example.com",
inputState: "filled",
tag: "input",
}),
]),
{ redactValues: true },
);
expect(out.text).toContain('[filled] ="•••"');
expect(out.text).not.toContain("user@example.com");
expect(out.refs[0]).toMatchObject({
ref: "e1",
backendNodeId: 42,
frameId: "child-frame",
role: "textbox",
name: "Email",
line: expect.any(Number),
});
expect(out.text.split("\n")[out.refs[0]?.line ?? -1]).toContain('@e1 textbox "Email"');
});
}); });
+1
View File
@@ -5,6 +5,7 @@ export type {
CondSurface, CondSurface,
LayerKind, LayerKind,
Rect, Rect,
RenderedRef,
Viewport, Viewport,
VomNode, VomNode,
VomOptions, VomOptions,
+18 -9
View File
@@ -4,9 +4,9 @@ import type {
BlockingLayer, BlockingLayer,
CondSurface, CondSurface,
Rect, Rect,
RenderedRef,
VomNode, VomNode,
VomOptions, VomOptions,
VomRef,
VomResult, VomResult,
VomScene, VomScene,
} from "./types"; } from "./types";
@@ -615,6 +615,7 @@ function renderNodeLine(
ref: string | undefined, ref: string | undefined,
context: string[] = [], context: string[] = [],
surface: CondSurface | undefined = undefined, surface: CondSurface | undefined = undefined,
redactValues = false,
): string { ): string {
let line = `${" ".repeat(depth)}${ref ? `@${ref} ` : ""}${node.role}`; let line = `${" ".repeat(depth)}${ref ? `@${ref} ` : ""}${node.role}`;
@@ -636,13 +637,14 @@ function renderNodeLine(
} }
const placeholder = cleaned(node.placeholder); const placeholder = cleaned(node.placeholder);
if (placeholder) line += ` placeholder=${JSON.stringify(placeholder)}`; if (placeholder) line += ` placeholder=${JSON.stringify(placeholder)}`;
const sensitiveHasValue = const hasValue =
rawValue !== undefined || node.inputState === "filled" || node.inputState === "default"; rawValue !== undefined || node.inputState === "filled" || node.inputState === "default";
const value = node.sensitive const value =
? sensitiveHasValue node.sensitive || redactValues
? SENSITIVE_MASK ? hasValue
: undefined ? SENSITIVE_MASK
: rawValue?.slice(0, MAX_VALUE_LENGTH); : undefined
: rawValue?.slice(0, MAX_VALUE_LENGTH);
if (value !== undefined) line += ` =${JSON.stringify(value)}`; if (value !== undefined) line += ` =${JSON.stringify(value)}`;
if (surface && surface.subItems.length > 0) { if (surface && surface.subItems.length > 0) {
@@ -695,11 +697,12 @@ function shouldSkipRedundantChildren(node: VomNode, state: RenderState): boolean
interface RenderState { interface RenderState {
lines: string[]; lines: string[];
refs: VomRef[]; refs: RenderedRef[];
nextRef: number; nextRef: number;
tokens: number; tokens: number;
maxDepth: number; maxDepth: number;
maxTokens: number; maxTokens: number;
redactValues: boolean;
truncated: boolean; truncated: boolean;
stopped: boolean; stopped: boolean;
children: Map<number | null, VomNode[]>; children: Map<number | null, VomNode[]>;
@@ -764,7 +767,7 @@ function renderTree(
const ref = isVomReferenceNode(node) ? `e${state.nextRef}` : undefined; const ref = isVomReferenceNode(node) ? `e${state.nextRef}` : undefined;
const context = ref ? handleContext(node, state) : []; const context = ref ? handleContext(node, state) : [];
const surface = state.surfaceMap.get(node.id); const surface = state.surfaceMap.get(node.id);
const line = renderNodeLine(node, depth, ref, context, surface); const line = renderNodeLine(node, depth, ref, context, surface, state.redactValues);
const nextTokens = state.tokens + estimateTokens(line); const nextTokens = state.tokens + estimateTokens(line);
if (nextTokens > state.maxTokens) { if (nextTokens > state.maxTokens) {
state.truncated = true; state.truncated = true;
@@ -775,10 +778,15 @@ function renderTree(
state.lines.push(line); state.lines.push(line);
state.tokens = nextTokens; state.tokens = nextTokens;
if (ref) { if (ref) {
const name = cleaned(node.name);
state.refs.push({ state.refs.push({
ref, ref,
backendNodeId: node.backendNodeId ?? node.id, backendNodeId: node.backendNodeId ?? node.id,
...(node.frameId ? { frameId: node.frameId } : {}), ...(node.frameId ? { frameId: node.frameId } : {}),
...(node.role ? { role: node.role } : {}),
...(name ? { name } : {}),
...(context.length > 0 ? { ctx: context.join(" > ") } : {}),
line: state.lines.length - 1,
}); });
state.nextRef += 1; state.nextRef += 1;
} }
@@ -808,6 +816,7 @@ function renderNodes(
tokens: estimateTokens(initialLines.join("\n")), tokens: estimateTokens(initialLines.join("\n")),
maxDepth: options.maxDepth ?? DEFAULT_MAX_DEPTH, maxDepth: options.maxDepth ?? DEFAULT_MAX_DEPTH,
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS, maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
redactValues: options.redactValues ?? false,
truncated: false, truncated: false,
stopped: false, stopped: false,
children, children,
+13 -1
View File
@@ -64,6 +64,10 @@ export interface VomScene {
export interface VomOptions { export interface VomOptions {
maxDepth?: number; maxDepth?: number;
maxTokens?: number; maxTokens?: number;
/**
* When true, form values are rendered as masks instead of literal values.
*/
redactValues?: boolean;
/** /**
* Experimental: filter refs that are geometrically blocked by foreground * Experimental: filter refs that are geometrically blocked by foreground
* fixed/absolute/sticky regions. Disabled by default and does not alter the * fixed/absolute/sticky regions. Disabled by default and does not alter the
@@ -90,9 +94,17 @@ export interface VomRef {
frameId?: string; frameId?: string;
} }
export interface RenderedRef extends VomRef {
role?: string;
name?: string;
ctx?: string;
/** Zero-based line index in `VomResult.text`. */
line: number;
}
export interface VomResult { export interface VomResult {
text: string; text: string;
refs: VomRef[]; refs: RenderedRef[];
truncated: boolean; truncated: boolean;
} }