mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
fix(vom): add record safe observations, unify definition and construction of safe DTO
This commit is contained in:
@@ -8,8 +8,6 @@ import {
|
||||
applyVomInteractionRecovery,
|
||||
type CondSurface,
|
||||
isVomReferenceNode,
|
||||
type Rect,
|
||||
type RenderedRef,
|
||||
renderVom,
|
||||
type VomNode,
|
||||
type VomOptions,
|
||||
@@ -51,12 +49,15 @@ import {
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import {
|
||||
type CapturedNode,
|
||||
type CapturedSurfaceProbe,
|
||||
type CapturedViewModel,
|
||||
captureViewModel,
|
||||
collectOverlayExcludedBackendIds,
|
||||
} from "./vom/capture";
|
||||
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
|
||||
import {
|
||||
type CaptureVomObservationResult,
|
||||
projectRecordSafeObservation,
|
||||
} from "./vom/record-safe-observation";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared helpers (legacy aliases — observation.ts kept exporting these
|
||||
@@ -1591,58 +1592,6 @@ export interface CaptureVomObservationOptions extends VomOptions {
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
/** Frame identity needed to click an `@eN` that lives in an iframe. */
|
||||
export interface CaptureVomFrame {
|
||||
frameId: string;
|
||||
target: CdpTarget;
|
||||
parentFrameId?: string;
|
||||
ownerBackendNodeId?: number;
|
||||
}
|
||||
|
||||
/** Geometry index for matching a recorded click to a captured node. */
|
||||
export interface CaptureVomNode {
|
||||
frameId: string;
|
||||
backendNodeId: number;
|
||||
tag: string;
|
||||
rect: Rect | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record-safe observation: rendered VOM text plus the matching index.
|
||||
* Raw AX/DOM trees are not returned — they carry form values.
|
||||
*/
|
||||
export interface CaptureVomObservationResult {
|
||||
text: string;
|
||||
refs: RenderedRef[];
|
||||
truncated: boolean;
|
||||
rootFrameId: string;
|
||||
frames: CaptureVomFrame[];
|
||||
nodes: CaptureVomNode[];
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
}
|
||||
|
||||
function recordSafeFrames(documents: CapturedFrameDocument<CdpAxNode>[]): CaptureVomFrame[] {
|
||||
return documents.map((document) => ({
|
||||
frameId: document.frameId,
|
||||
target: document.target,
|
||||
...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
|
||||
...(document.ownerBackendNodeId !== undefined
|
||||
? { ownerBackendNodeId: document.ownerBackendNodeId }
|
||||
: {}),
|
||||
}));
|
||||
}
|
||||
|
||||
function recordSafeNodes(documents: CapturedFrameDocument<CdpAxNode>[]): CaptureVomNode[] {
|
||||
return documents.flatMap((document) =>
|
||||
document.domNodes.map((node) => ({
|
||||
frameId: document.frameId,
|
||||
backendNodeId: node.backendNodeId,
|
||||
tag: node.tag,
|
||||
rect: node.rect,
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
export async function captureVomObservation(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
@@ -1667,15 +1616,12 @@ export async function captureVomObservation(
|
||||
activeRegionPolicy: options.activeRegionPolicy,
|
||||
});
|
||||
throwIfAborted(options.signal, "observation");
|
||||
return {
|
||||
text: rendered.text,
|
||||
refs: rendered.refs,
|
||||
truncated: rendered.truncated,
|
||||
return projectRecordSafeObservation({
|
||||
rootFrameId: captured.rootFrameId ?? documents[0]?.frameId ?? "root",
|
||||
frames: recordSafeFrames(documents),
|
||||
nodes: recordSafeNodes(documents),
|
||||
frameDocuments: documents,
|
||||
rendered,
|
||||
surfaceProbes: captured.surfaceProbes,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function handleVomObservation(
|
||||
|
||||
Reference in New Issue
Block a user