fix(vom): add record safe observations, unify definition and construction of safe DTO

This commit is contained in:
Ljy-0827
2026-08-21 11:26:52 +08:00
parent 20554508db
commit 005a6fd534
4 changed files with 97 additions and 67 deletions
+8 -62
View File
@@ -8,8 +8,6 @@ import {
applyVomInteractionRecovery,
type CondSurface,
isVomReferenceNode,
type Rect,
type RenderedRef,
renderVom,
type VomNode,
type VomOptions,
@@ -51,12 +49,15 @@ import {
import { resolveSnapshotRef } from "./snapshot-ref";
import {
type CapturedNode,
type CapturedSurfaceProbe,
type CapturedViewModel,
captureViewModel,
collectOverlayExcludedBackendIds,
} from "./vom/capture";
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
import {
type CaptureVomObservationResult,
projectRecordSafeObservation,
} from "./vom/record-safe-observation";
// ---------------------------------------------------------------------------
// Shared helpers (legacy aliases — observation.ts kept exporting these
@@ -1591,58 +1592,6 @@ export interface CaptureVomObservationOptions extends VomOptions {
signal?: AbortSignal;
}
/** Frame identity needed to click an `@eN` that lives in an iframe. */
export interface CaptureVomFrame {
frameId: string;
target: CdpTarget;
parentFrameId?: string;
ownerBackendNodeId?: number;
}
/** Geometry index for matching a recorded click to a captured node. */
export interface CaptureVomNode {
frameId: string;
backendNodeId: number;
tag: string;
rect: Rect | null;
}
/**
* Record-safe observation: rendered VOM text plus the matching index.
* Raw AX/DOM trees are not returned — they carry form values.
*/
export interface CaptureVomObservationResult {
text: string;
refs: RenderedRef[];
truncated: boolean;
rootFrameId: string;
frames: CaptureVomFrame[];
nodes: CaptureVomNode[];
surfaceProbes?: CapturedSurfaceProbe[];
}
function recordSafeFrames(documents: CapturedFrameDocument<CdpAxNode>[]): CaptureVomFrame[] {
return documents.map((document) => ({
frameId: document.frameId,
target: document.target,
...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
...(document.ownerBackendNodeId !== undefined
? { ownerBackendNodeId: document.ownerBackendNodeId }
: {}),
}));
}
function recordSafeNodes(documents: CapturedFrameDocument<CdpAxNode>[]): CaptureVomNode[] {
return documents.flatMap((document) =>
document.domNodes.map((node) => ({
frameId: document.frameId,
backendNodeId: node.backendNodeId,
tag: node.tag,
rect: node.rect,
})),
);
}
export async function captureVomObservation(
cdp: CdpRunner,
tabId: number,
@@ -1667,15 +1616,12 @@ export async function captureVomObservation(
activeRegionPolicy: options.activeRegionPolicy,
});
throwIfAborted(options.signal, "observation");
return {
text: rendered.text,
refs: rendered.refs,
truncated: rendered.truncated,
return projectRecordSafeObservation({
rootFrameId: captured.rootFrameId ?? documents[0]?.frameId ?? "root",
frames: recordSafeFrames(documents),
nodes: recordSafeNodes(documents),
frameDocuments: documents,
rendered,
surfaceProbes: captured.surfaceProbes,
};
});
}
async function handleVomObservation(