fix(vom): add record safe observations, unify definition and construction of safe DTO

This commit is contained in:
Ljy-0827
2026-08-21 11:26:52 +08:00
parent 20554508db
commit 005a6fd534
4 changed files with 97 additions and 67 deletions
@@ -3146,11 +3146,12 @@ describe("handleSnapshot", () => {
expect(result.rootFrameId).toBe("root");
expect(result).not.toHaveProperty("frameDocuments");
expect(result.frames).toEqual([{ frameId: "root", target: { tabId: 4 } }]);
expect(result.nodes.find((node) => node.backendNodeId === 13)).toMatchObject({
expect(result.matchNodes.find((node) => node.backendNodeId === 13)).toMatchObject({
frameId: "root",
backendNodeId: 13,
tag: "input",
rect: { x: 400, y: 300, w: 200, h: 40 },
localRect: { x: 400, y: 300, w: 200, h: 40 },
});
});
@@ -3214,7 +3215,10 @@ describe("handleSnapshot", () => {
expect(dumped).not.toContain(secrets.otp);
expect(dumped).not.toContain(secrets.card);
expect(dumped).not.toContain("formValue");
expect(dumped).not.toContain("formDefaultValue");
expect(dumped).not.toContain("axNodes");
expect(dumped).not.toContain("domNodes");
expect(dumped).not.toContain("attrs");
expect(result.text).toContain("•••");
});
@@ -3231,9 +3235,10 @@ describe("handleSnapshot", () => {
ownerBackendNodeId: 12,
target: { tabId: 4, sessionId: "child-session" },
});
expect(result.nodes.find((node) => node.backendNodeId === 22)).toMatchObject({
expect(result.matchNodes.find((node) => node.backendNodeId === 22)).toMatchObject({
frameId: "child",
tag: "button",
localRect: { x: 20, y: 30, w: 120, h: 40 },
});
expect(result.refs.find((ref) => ref.backendNodeId === 22)).toMatchObject({
frameId: "child",
@@ -1,7 +1,9 @@
export {
type CaptureVomFrame,
type CaptureVomNode,
type CaptureVomObservationOptions,
type CaptureVomObservationResult,
captureVomObservation,
} from "./observation";
export {
type CaptureVomFrame,
type CaptureVomMatchNode,
type CaptureVomObservationResult,
} from "./vom/record-safe-observation";
+8 -62
View File
@@ -8,8 +8,6 @@ import {
applyVomInteractionRecovery,
type CondSurface,
isVomReferenceNode,
type Rect,
type RenderedRef,
renderVom,
type VomNode,
type VomOptions,
@@ -51,12 +49,15 @@ import {
import { resolveSnapshotRef } from "./snapshot-ref";
import {
type CapturedNode,
type CapturedSurfaceProbe,
type CapturedViewModel,
captureViewModel,
collectOverlayExcludedBackendIds,
} from "./vom/capture";
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
import {
type CaptureVomObservationResult,
projectRecordSafeObservation,
} from "./vom/record-safe-observation";
// ---------------------------------------------------------------------------
// Shared helpers (legacy aliases — observation.ts kept exporting these
@@ -1591,58 +1592,6 @@ export interface CaptureVomObservationOptions extends VomOptions {
signal?: AbortSignal;
}
/** Frame identity needed to click an `@eN` that lives in an iframe. */
export interface CaptureVomFrame {
frameId: string;
target: CdpTarget;
parentFrameId?: string;
ownerBackendNodeId?: number;
}
/** Geometry index for matching a recorded click to a captured node. */
export interface CaptureVomNode {
frameId: string;
backendNodeId: number;
tag: string;
rect: Rect | null;
}
/**
* Record-safe observation: rendered VOM text plus the matching index.
* Raw AX/DOM trees are not returned — they carry form values.
*/
export interface CaptureVomObservationResult {
text: string;
refs: RenderedRef[];
truncated: boolean;
rootFrameId: string;
frames: CaptureVomFrame[];
nodes: CaptureVomNode[];
surfaceProbes?: CapturedSurfaceProbe[];
}
function recordSafeFrames(documents: CapturedFrameDocument<CdpAxNode>[]): CaptureVomFrame[] {
return documents.map((document) => ({
frameId: document.frameId,
target: document.target,
...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
...(document.ownerBackendNodeId !== undefined
? { ownerBackendNodeId: document.ownerBackendNodeId }
: {}),
}));
}
function recordSafeNodes(documents: CapturedFrameDocument<CdpAxNode>[]): CaptureVomNode[] {
return documents.flatMap((document) =>
document.domNodes.map((node) => ({
frameId: document.frameId,
backendNodeId: node.backendNodeId,
tag: node.tag,
rect: node.rect,
})),
);
}
export async function captureVomObservation(
cdp: CdpRunner,
tabId: number,
@@ -1667,15 +1616,12 @@ export async function captureVomObservation(
activeRegionPolicy: options.activeRegionPolicy,
});
throwIfAborted(options.signal, "observation");
return {
text: rendered.text,
refs: rendered.refs,
truncated: rendered.truncated,
return projectRecordSafeObservation({
rootFrameId: captured.rootFrameId ?? documents[0]?.frameId ?? "root",
frames: recordSafeFrames(documents),
nodes: recordSafeNodes(documents),
frameDocuments: documents,
rendered,
surfaceProbes: captured.surfaceProbes,
};
});
}
async function handleVomObservation(
@@ -0,0 +1,77 @@
import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import type { CapturedSurfaceProbe } from "./capture";
import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture";
/** Frame identity needed to resolve an `@eN` that lives in an iframe. */
export interface CaptureVomFrame {
frameId: string;
target: CdpTarget;
parentFrameId?: string;
ownerBackendNodeId?: number;
}
/** Allowlisted geometry used to match a recorded action to a rendered ref. */
export interface CaptureVomMatchNode {
frameId: string;
backendNodeId: number;
tag: string;
/** Top-level viewport-relative geometry. */
rect: Rect | null;
/** Frame-local viewport-relative geometry, when available. */
localRect?: Rect | null;
}
/**
* Raw AX/DOM trees are never returned. When `redactValues` is enabled by the
* caller, form values in `text` are masked as well.
*/
export interface CaptureVomObservationResult {
text: string;
refs: RenderedRef[];
truncated: boolean;
rootFrameId: string;
frames: CaptureVomFrame[];
matchNodes: CaptureVomMatchNode[];
surfaceProbes?: CapturedSurfaceProbe[];
}
function projectFrames(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomFrame[] {
return documents.map((document) => ({
frameId: document.frameId,
target: document.target,
...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
...(document.ownerBackendNodeId !== undefined
? { ownerBackendNodeId: document.ownerBackendNodeId }
: {}),
}));
}
function projectMatchNodes(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomMatchNode[] {
return documents.flatMap((document) =>
document.domNodes.map((node) => ({
frameId: document.frameId,
backendNodeId: node.backendNodeId,
tag: node.tag,
rect: node.rect,
...(node.localRect !== undefined ? { localRect: node.localRect } : {}),
})),
);
}
export function projectRecordSafeObservation(input: {
rootFrameId: string;
frameDocuments: CapturedFrameDocument<FrameAxNode>[];
rendered: VomResult;
surfaceProbes?: CapturedSurfaceProbe[];
}): CaptureVomObservationResult {
return {
text: input.rendered.text,
refs: input.rendered.refs,
truncated: input.rendered.truncated,
rootFrameId: input.rootFrameId,
frames: projectFrames(input.frameDocuments),
matchNodes: projectMatchNodes(input.frameDocuments),
surfaceProbes: input.surfaceProbes,
};
}