From 6300eda0aac0f5e1818eec2bd8e5af7724c296c5 Mon Sep 17 00:00:00 2001 From: Ljy-0827 Date: Fri, 21 Aug 2026 11:45:51 +0800 Subject: [PATCH] fix(record): rebase and dto policy --- .../__tests__/recording-observation.test.ts | 19 ++----- .../lib/__tests__/settle-controller.test.ts | 2 +- .../src/lib/__tests__/target-matcher.test.ts | 49 ++++++++++--------- .../src/lib/recording/observation-capture.ts | 35 ++++++------- .../src/lib/recording/target-matcher.ts | 2 +- 5 files changed, 50 insertions(+), 57 deletions(-) diff --git a/apps/extension/src/lib/__tests__/recording-observation.test.ts b/apps/extension/src/lib/__tests__/recording-observation.test.ts index dd277f4..18b4959 100644 --- a/apps/extension/src/lib/__tests__/recording-observation.test.ts +++ b/apps/extension/src/lib/__tests__/recording-observation.test.ts @@ -18,22 +18,13 @@ function sessionWithInput(redactValues = false): RecordingObservationSession { rootFrameId: "root", index: new ObservationNodeIndex({ rootFrameId: "root", - frameDocuments: [ + matchNodes: [ { frameId: "root", - domNodes: [ - { - backendNodeId: 42, - parentBackendNodeId: null, - frameId: "root", - tag: "input", - attrs: {}, - rect: { x: 20, y: 40, w: 200, h: 30 }, - paintOrder: 1, - position: "static", - pointerEvents: "auto", - }, - ], + backendNodeId: 42, + tag: "input", + rect: { x: 20, y: 40, w: 200, h: 30 }, + localRect: { x: 20, y: 40, w: 200, h: 30 }, }, ], refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }], diff --git a/apps/extension/src/lib/__tests__/settle-controller.test.ts b/apps/extension/src/lib/__tests__/settle-controller.test.ts index b72e243..5ec5043 100644 --- a/apps/extension/src/lib/__tests__/settle-controller.test.ts +++ b/apps/extension/src/lib/__tests__/settle-controller.test.ts @@ -8,7 +8,7 @@ import type { RecordingDraftStep } from "../recording/types"; const OBSERVATION: RegisteredObservation = { stateId: "s-next", rootFrameId: "root", - index: new ObservationNodeIndex({ rootFrameId: "root", frameDocuments: [], refs: [] }), + index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }), url: "https://example.com/next", }; diff --git a/apps/extension/src/lib/__tests__/target-matcher.test.ts b/apps/extension/src/lib/__tests__/target-matcher.test.ts index b708118..d3efae2 100644 --- a/apps/extension/src/lib/__tests__/target-matcher.test.ts +++ b/apps/extension/src/lib/__tests__/target-matcher.test.ts @@ -1,40 +1,50 @@ import type { RenderedRef } from "@browser-skill/vom"; import { describe, expect, it } from "vitest"; -import type { CapturedNode } from "@/tools/vom/capture"; +import type { CaptureVomMatchNode } from "@/tools/capture-vom-observation"; import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture"; import { matchObservationTarget } from "../recording/target-matcher"; -function node(backendNodeId: number, frameId: string, x = 10): CapturedNode { +function node(backendNodeId: number, frameId: string, x = 10): CaptureVomMatchNode { return { backendNodeId, - parentBackendNodeId: null, frameId, tag: "button", - attrs: {}, rect: { x, y: 20, w: 100, h: 30 }, - paintOrder: 1, - position: "static", - pointerEvents: "auto", + localRect: { x, y: 20, w: 100, h: 30 }, }; } function observation( - documents: Array<{ frameId: string; domNodes: CapturedNode[] }>, + matchNodes: CaptureVomMatchNode[], refs: RenderedRef[], ): RegisteredObservation { return { stateId: "s1", rootFrameId: "root", - index: new ObservationNodeIndex({ rootFrameId: "root", frameDocuments: documents, refs }), + index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes, refs }), url: "https://example.com", }; } describe("matchObservationTarget", () => { + it("indexes the safe geometry DTO without discarding frame-local geometry", () => { + const geometry = node(42, "child"); + geometry.localRect = { x: 5, y: 6, w: 100, h: 30 }; + const ref: RenderedRef = { + ref: "e1", + backendNodeId: 42, + frameId: "child", + line: 1, + }; + const candidate = observation([geometry], [ref]).index.candidates("child", "button")[0]; + + expect(candidate).toEqual({ frameId: "child", geometry, ref }); + }); + it("matches a unique node using canonical top-level viewport geometry", () => { const target = matchObservationTarget({ observation: observation( - [{ frameId: "root", domNodes: [node(42, "root")] }], + [node(42, "root")], [{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }], ), hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } }, @@ -45,10 +55,7 @@ describe("matchObservationTarget", () => { it("uses frame id with backend node id so sibling frames cannot collide", () => { const target = matchObservationTarget({ observation: observation( - [ - { frameId: "left", domNodes: [node(42, "left")] }, - { frameId: "right", domNodes: [node(42, "right")] }, - ], + [node(42, "left"), node(42, "right")], [ { ref: "e1", backendNodeId: 42, frameId: "left", line: 1 }, { ref: "e2", backendNodeId: 42, frameId: "right", line: 2 }, @@ -65,10 +72,7 @@ describe("matchObservationTarget", () => { it("restricts a missing frame hint to the root frame", () => { const target = matchObservationTarget({ observation: observation( - [ - { frameId: "root", domNodes: [] }, - { frameId: "child", domNodes: [node(42, "child")] }, - ], + [node(42, "child")], [{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }], ), hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } }, @@ -80,7 +84,7 @@ describe("matchObservationTarget", () => { it("returns unmatched for ambiguous geometry", () => { const target = matchObservationTarget({ observation: observation( - [{ frameId: "root", domNodes: [node(42, "root"), node(43, "root")] }], + [node(42, "root"), node(43, "root")], [ { ref: "e1", backendNodeId: 42, line: 1 }, { ref: "e2", backendNodeId: 43, line: 2 }, @@ -94,10 +98,7 @@ describe("matchObservationTarget", () => { it("uses semantics when geometry is unavailable without crossing frame boundaries", () => { const target = matchObservationTarget({ observation: observation( - [ - { frameId: "root", domNodes: [node(41, "root")] }, - { frameId: "child", domNodes: [node(42, "child")] }, - ], + [node(41, "root"), node(42, "child")], [ { ref: "e1", backendNodeId: 41, role: "button", name: "保存", line: 1 }, { ref: "e2", backendNodeId: 42, frameId: "child", role: "button", name: "保存", line: 2 }, @@ -112,7 +113,7 @@ describe("matchObservationTarget", () => { it("uses semantics to disambiguate equal geometry in one frame", () => { const target = matchObservationTarget({ observation: observation( - [{ frameId: "root", domNodes: [node(42, "root"), node(43, "root")] }], + [node(42, "root"), node(43, "root")], [ { ref: "e1", backendNodeId: 42, role: "button", name: "保存", line: 1 }, { ref: "e2", backendNodeId: 43, role: "button", name: "取消", line: 2 }, diff --git a/apps/extension/src/lib/recording/observation-capture.ts b/apps/extension/src/lib/recording/observation-capture.ts index 8006f74..9c1f459 100644 --- a/apps/extension/src/lib/recording/observation-capture.ts +++ b/apps/extension/src/lib/recording/observation-capture.ts @@ -1,11 +1,14 @@ import type { RenderedRef } from "@browser-skill/vom"; -import { captureVomObservation } from "@/tools/capture-vom-observation"; +import { + type CaptureVomMatchNode, + type CaptureVomObservationResult, + captureVomObservation, +} from "@/tools/capture-vom-observation"; import type { CdpRunner, ChromeTabsApi } from "@/tools/shared"; -import type { CapturedNode } from "@/tools/vom/capture"; export interface IndexedObservationNode { frameId: string; - node: CapturedNode; + geometry: CaptureVomMatchNode; ref?: RenderedRef; } @@ -38,11 +41,7 @@ export class ObservationNodeIndex { readonly #refById = new Map(); readonly #refsByFrame = new Map(); - constructor(input: { - rootFrameId: string; - frameDocuments: Array<{ frameId: string; domNodes: CapturedNode[] }>; - refs: RenderedRef[]; - }) { + constructor(input: Pick) { const refByNode = new Map(); for (const ref of input.refs) { const frameId = ref.frameId ?? input.rootFrameId; @@ -52,15 +51,17 @@ export class ObservationNodeIndex { frameRefs.push(ref); this.#refsByFrame.set(frameId, frameRefs); } - for (const document of input.frameDocuments) { - for (const node of document.domNodes) { - const frameId = node.frameId ?? document.frameId; - const entry = { frameId, node, ref: refByNode.get(nodeKey(frameId, node.backendNodeId)) }; - const key = frameTagKey(frameId, node.tag); - const bucket = this.#nodesByFrameTag.get(key) ?? []; - bucket.push(entry); - this.#nodesByFrameTag.set(key, bucket); - } + for (const geometry of input.matchNodes) { + const { frameId } = geometry; + const entry = { + frameId, + geometry, + ref: refByNode.get(nodeKey(frameId, geometry.backendNodeId)), + }; + const key = frameTagKey(frameId, geometry.tag); + const bucket = this.#nodesByFrameTag.get(key) ?? []; + bucket.push(entry); + this.#nodesByFrameTag.set(key, bucket); } } diff --git a/apps/extension/src/lib/recording/target-matcher.ts b/apps/extension/src/lib/recording/target-matcher.ts index 0df1abe..5cf21aa 100644 --- a/apps/extension/src/lib/recording/target-matcher.ts +++ b/apps/extension/src/lib/recording/target-matcher.ts @@ -15,7 +15,7 @@ function rectMatches(a: TargetGeometry["rect"], b: TargetGeometry["rect"]): bool } function candidateMatches(candidate: IndexedObservationNode, geometry: TargetGeometry): boolean { - return candidate.node.rect !== null && rectMatches(geometry.rect, candidate.node.rect); + return candidate.geometry.rect !== null && rectMatches(geometry.rect, candidate.geometry.rect); } export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 {