fix(recording): using unified geometry

This commit is contained in:
Ljy-0827
2026-08-23 21:26:40 +08:00
committed by GitHub
parent 21476c3884
commit 87cdcfe5ca
35 changed files with 1952 additions and 2087 deletions
@@ -0,0 +1,101 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpRunner } from "@/tools/shared";
import { waitForDocumentSettled } from "../recording/document-settle";
function quietCdp(): { cdp: CdpRunner; send: ReturnType<typeof vi.fn> } {
const send = vi.fn(async () => ({
result: { value: { idleMs: 1_000, readyState: "complete" } },
}));
return {
cdp: { send: send as unknown as CdpRunner["send"] },
send,
};
}
describe("waitForDocumentSettled", () => {
afterEach(() => {
vi.useRealTimers();
});
it("waits through the minimum observation floor before accepting a quiet page", async () => {
vi.useFakeTimers();
const { cdp, send } = quietCdp();
const settled = waitForDocumentSettled(cdp, { target: { tabId: 7 } });
await vi.advanceTimersByTimeAsync(180);
await expect(settled).resolves.toBe("quiet");
expect(send).toHaveBeenCalledTimes(3);
});
it("cancels before probing a superseded observation", async () => {
vi.useFakeTimers();
const { cdp, send } = quietCdp();
const controller = new AbortController();
const settled = waitForDocumentSettled(
cdp,
{ target: { tabId: 7 } },
{
signal: controller.signal,
},
);
controller.abort();
await vi.advanceTimersByTimeAsync(60);
await expect(settled).resolves.toBe("cancelled");
expect(send).not.toHaveBeenCalled();
});
it("probes a same-process iframe through its isolated execution context", async () => {
vi.useFakeTimers();
const send = vi.fn(async (_tabId: number, method: string) => {
if (method === "Page.createIsolatedWorld") return { executionContextId: 91 };
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
});
const cdp: CdpRunner = { send: send as unknown as CdpRunner["send"] };
const settled = waitForDocumentSettled(cdp, {
target: { tabId: 7 },
frameId: "child-frame",
});
await vi.advanceTimersByTimeAsync(180);
await expect(settled).resolves.toBe("quiet");
expect(send).toHaveBeenCalledWith(
7,
"Page.createIsolatedWorld",
expect.objectContaining({ frameId: "child-frame" }),
);
expect(send).toHaveBeenCalledWith(
7,
"Runtime.evaluate",
expect.objectContaining({ contextId: 91 }),
);
});
it("routes an OOPIF probe through its CDP target session", async () => {
vi.useFakeTimers();
const send = vi.fn();
const sendToTarget = vi.fn(async (_target, method: string) => {
if (method === "Page.createIsolatedWorld") return { executionContextId: 27 };
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
});
const cdp: CdpRunner = {
send: send as unknown as CdpRunner["send"],
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
};
const target = { tabId: 7, sessionId: "oopif-session" };
const settled = waitForDocumentSettled(cdp, { target, frameId: "oopif-frame" });
await vi.advanceTimersByTimeAsync(180);
await expect(settled).resolves.toBe("quiet");
expect(send).not.toHaveBeenCalled();
expect(sendToTarget).toHaveBeenCalledWith(
target,
"Runtime.evaluate",
expect.objectContaining({ contextId: 27 }),
);
});
});
@@ -1,116 +0,0 @@
import type { RenderedRef } from "@browser-skill/vom";
import { describe, expect, it } from "vitest";
import type { CapturedNode } from "@/tools/vom/capture";
import { matchTarget } from "../match-target";
function node(backendNodeId: number, overrides: Partial<CapturedNode> = {}): CapturedNode {
return {
backendNodeId,
parentBackendNodeId: null,
tag: "button",
attrs: {},
rect: { x: 10, y: 20, w: 100, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
...overrides,
};
}
describe("matchTarget", () => {
const refs: RenderedRef[] = [
{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 3 },
];
it("matches a unique node by viewport coordinates", () => {
const target = matchTarget({
geometry: {
rect: { x: 10, y: 20, w: 100, h: 30 },
scrollX: 0,
scrollY: 0,
position: "static",
tag: "button",
},
captured: [node(42)],
refs,
fallback: { tag: "button", name: "发布" },
});
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
});
it("matches a non-fixed node after the top frame scrolls", () => {
const target = matchTarget({
geometry: {
rect: { x: 10, y: 20, w: 100, h: 30 },
scrollX: 40,
scrollY: 300,
position: "static",
tag: "button",
},
captured: [
node(42, {
documentRect: { x: 50, y: 320, w: 100, h: 30 },
localRect: { x: 10, y: 20, w: 100, h: 30 },
rect: { x: 10, y: 20, w: 100, h: 30 },
}),
],
refs,
fallback: { tag: "button", name: "发布" },
});
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
});
it("matches a static node by document coordinates when scroll changed after observation", () => {
const target = matchTarget({
geometry: {
rect: { x: 10, y: 50, w: 100, h: 30 },
scrollX: 0,
scrollY: 250,
position: "static",
tag: "button",
},
captured: [
node(42, {
documentRect: { x: 10, y: 300, w: 100, h: 30 },
localRect: { x: 10, y: 200, w: 100, h: 30 },
rect: { x: 10, y: 200, w: 100, h: 30 },
}),
],
refs,
fallback: { tag: "button", name: "发布" },
});
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
});
it("returns unmatched when multiple candidates match", () => {
const target = matchTarget({
geometry: {
rect: { x: 10, y: 20, w: 100, h: 30 },
scrollX: 0,
scrollY: 0,
position: "static",
tag: "button",
},
captured: [node(42), node(43)],
refs,
});
expect(target.unmatched).toBe(true);
});
it("uses viewport coordinates for fixed elements", () => {
const target = matchTarget({
geometry: {
rect: { x: 5, y: 5, w: 80, h: 24 },
scrollX: 100,
scrollY: 200,
position: "fixed",
tag: "button",
},
captured: [node(42, { position: "fixed", rect: { x: 5, y: 5, w: 80, h: 24 } })],
refs,
});
expect(target.ref).toBe("e1");
});
});
@@ -1,43 +0,0 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpRunner } from "@/tools/shared";
import { waitForPageSettled } from "../page-settled";
function quietCdp(): { cdp: CdpRunner; send: ReturnType<typeof vi.fn> } {
const send = vi.fn(async () => ({
result: { value: { idleMs: 1_000, readyState: "complete" } },
}));
return {
cdp: { send: send as unknown as CdpRunner["send"] },
send,
};
}
describe("waitForPageSettled", () => {
afterEach(() => {
vi.useRealTimers();
});
it("waits through the minimum observation floor before accepting a quiet page", async () => {
vi.useFakeTimers();
const { cdp, send } = quietCdp();
const settled = waitForPageSettled(cdp, 7);
await vi.advanceTimersByTimeAsync(180);
await expect(settled).resolves.toBe("quiet");
expect(send).toHaveBeenCalledTimes(3);
});
it("cancels before probing a superseded observation", async () => {
vi.useFakeTimers();
let cancelled = false;
const { cdp, send } = quietCdp();
const settled = waitForPageSettled(cdp, 7, { cancelled: () => cancelled });
cancelled = true;
await vi.advanceTimersByTimeAsync(60);
await expect(settled).resolves.toBe("cancelled");
expect(send).not.toHaveBeenCalled();
});
});
@@ -1,206 +0,0 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import type { CapturedNode } from "@/tools/vom/capture";
import type { DraftTraceStep } from "@/transport/types";
import { resetStateIdCounterForTests } from "../record-constants";
import {
applyTargetMatching,
buildTraceV3,
createObservationState,
flushPendingRedirectLanding,
rememberStepOnPage,
scheduleRedirectLandingFlush,
} from "../record-observation";
import { registerObservation } from "../trace-reducer";
const URL = "https://example.com/login";
function capturedInput(): CapturedNode {
return {
backendNodeId: 42,
parentBackendNodeId: null,
tag: "input",
attrs: {},
rect: { x: 20, y: 40, w: 200, h: 30 },
localRect: { x: 20, y: 40, w: 200, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
};
}
function finalizedFillBody(value: string, redactValues: boolean): string {
const obs = createObservationState({ redactValues });
const rawVomText = '@vom 1\ntextbox "Password" value="••••••" [ref=e1]';
const stateId = registerObservation(obs.stateRegistry, { url: URL, rawVomText });
obs.lastSettled = {
stateId,
captured: [capturedInput()],
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
url: URL,
vomText: rawVomText,
};
const draft: DraftTraceStep = {
op: "fill",
target: { unmatched: true },
value,
geometry: {
rect: { x: 20, y: 40, w: 200, h: 30 },
scrollX: 0,
scrollY: 0,
position: "static",
tag: "input",
},
};
applyTargetMatching(obs, draft, 1);
draft.postStateId = stateId;
rememberStepOnPage(obs, draft, 1);
return (
buildTraceV3({
obs,
steps: [draft],
startedAt: "2026-08-12T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "test",
}).states[0]?.body ?? ""
);
}
describe("record observation annotations", () => {
beforeEach(() => {
resetStateIdCounterForTests();
});
it("omits a fill literal from finalized state bodies when values are redacted", () => {
const secret = "hunter2-private";
const body = finalizedFillBody(secret, true);
expect(body).toContain("step 1: fill");
expect(body).not.toContain(secret);
});
it("keeps fill details in ordinary recording annotations", () => {
const value = "ordinary text";
const body = finalizedFillBody(value, false);
expect(body).toContain(`step 1: fill: ${JSON.stringify(value)}`);
});
});
describe("applyTargetMatching without a settled observation", () => {
it("keeps the capture role/name instead of a bare unmatched target", () => {
const obs = createObservationState();
const draft: DraftTraceStep = {
op: "hover",
target: { unmatched: true },
captureTarget: { tag: "button", role: "button", name: "新建" },
geometry: {
rect: { x: 900, y: 8, w: 60, h: 32 },
scrollX: 0,
scrollY: 0,
position: "static",
tag: "button",
},
};
applyTargetMatching(obs, draft, 1);
expect(draft.preStateId).toBeUndefined();
expect(draft.target).toEqual({
role: "button",
name: "新建",
unmatched: true,
});
});
});
describe("applyTargetMatching while the previous action is still settling", () => {
it("does not bind a new unmatched control to the stale observation", () => {
const obs = createObservationState();
const stateId = registerObservation(obs.stateRegistry, {
url: URL,
rawVomText: '@vom 1\n@e1 textbox "Search"',
});
obs.lastSettled = {
stateId,
captured: [capturedInput()],
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Search", line: 1 }],
url: URL,
vomText: '@vom 1\n@e1 textbox "Search"',
};
obs.settles.set(0, { draftIndex: 0, cancelled: false });
const draft: DraftTraceStep = {
op: "click",
target: { unmatched: true },
captureTarget: { tag: "button", role: "button", name: "Confirm" },
geometry: {
rect: { x: 400, y: 300, w: 80, h: 30 },
scrollX: 0,
scrollY: 0,
position: "fixed",
tag: "button",
},
};
applyTargetMatching(obs, draft, 2);
expect(draft.preStateId).toBeUndefined();
expect(draft.target).toEqual({ role: "button", name: "Confirm", unmatched: true });
});
});
describe("redirect landing coalescing", () => {
it("does not consume a newer redirect hop while reading tab metadata", async () => {
vi.useFakeTimers();
try {
const finalUrl = "https://example.com/final";
const intermediateUrl = "https://example.com/intermediate";
const obs = createObservationState();
obs.lastSettled = {
stateId: "s-final",
captured: [],
refs: [],
url: finalUrl,
vomText: "@vom 1\nFinal",
};
const steps: DraftTraceStep[] = [];
let resolveFirstTab!: (tab: chrome.tabs.Tab) => void;
const firstTab = new Promise<chrome.tabs.Tab>((resolve) => {
resolveFirstTab = resolve;
});
const get = vi
.fn()
.mockImplementationOnce(() => firstTab)
.mockResolvedValue({ id: 7, url: finalUrl });
const tabsApi: ChromeTabsApi = {
get,
query: vi.fn(async () => []),
};
const send = vi.fn(async (_tabId: number, method: string) => {
if (method === "Runtime.evaluate") {
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
}
throw new Error(`unexpected CDP method ${method}`);
});
const cdp: CdpRunner = { send: send as unknown as CdpRunner["send"] };
scheduleRedirectLandingFlush(obs, steps, cdp, 7, tabsApi, intermediateUrl);
await vi.advanceTimersByTimeAsync(180);
expect(get).toHaveBeenCalledTimes(1);
scheduleRedirectLandingFlush(obs, steps, cdp, 7, tabsApi, finalUrl);
resolveFirstTab({ id: 7, url: intermediateUrl } as chrome.tabs.Tab);
await Promise.resolve();
await vi.advanceTimersByTimeAsync(1_000);
await flushPendingRedirectLanding(obs, steps, cdp, 7, tabsApi);
expect(steps).toEqual([]);
expect(obs.pendingRedirect).toBeNull();
} finally {
vi.useRealTimers();
}
});
});
@@ -0,0 +1,150 @@
import { describe, expect, it } from "vitest";
import { ObservationNodeIndex } from "../recording/observation-capture";
import { RecordingObservationSession } from "../recording/observation-session";
import { RecordingStateRegistry } from "../recording/state-registry";
import { buildTraceV3 } from "../recording/trace-builder-v3";
import type { RecordingDraftStep } from "../recording/types";
const URL = "https://example.com/login";
function sessionWithInput(redactValues = false): RecordingObservationSession {
const session = new RecordingObservationSession({ redactValues });
const state = session.registry.register({
url: URL,
rawVomText: '@vom 1\ntextbox "Password" value="••••••" [ref=e1]',
});
session.cursor.lastSettled = {
stateId: state.id,
rootFrameId: "root",
index: new ObservationNodeIndex({
rootFrameId: "root",
frameDocuments: [
{
frameId: "root",
domNodes: [
{
backendNodeId: 42,
parentBackendNodeId: null,
frameId: "root",
tag: "input",
attrs: {},
rect: { x: 20, y: 40, w: 200, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
},
],
},
],
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
}),
url: URL,
};
return session;
}
function finalizedFillBody(value: string, redactValues: boolean): string {
const session = sessionWithInput(redactValues);
const draft: RecordingDraftStep = {
op: "fill",
captureTarget: { tag: "input", role: "textbox", name: "Password" },
value,
targetHint: {
geometry: { rect: { x: 20, y: 40, w: 200, h: 30 }, tag: "input" },
},
};
session.bindDraft(draft, 1);
draft.postStateId = draft.preStateId;
return buildTraceV3({
registry: session.registry,
drafts: [draft],
annotations: session.annotations,
startedAt: "2026-08-12T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "test",
}).states[0]!.body;
}
describe("record observation annotations", () => {
it("omits a fill literal when values are redacted", () => {
const secret = "hunter2-private";
const body = finalizedFillBody(secret, true);
expect(body).toContain("step 1: fill");
expect(body).not.toContain(secret);
});
it("keeps ordinary fill details", () => {
expect(finalizedFillBody("ordinary text", false)).toContain('step 1: fill: "ordinary text"');
});
it("encodes title and URL so line breaks cannot corrupt state metadata", () => {
const registry = new RecordingStateRegistry();
const state = registry.register({
url: "https://example.com/a\nb",
title: "hello\nworld",
rawVomText: "@vom 1",
});
const trace = buildTraceV3({
registry,
drafts: [{ op: "scroll", preStateId: state.id, postStateId: state.id }],
startedAt: "2026-08-12T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "test",
});
expect(trace.states[0]?.body).toContain('url: "https://example.com/a\\nb"');
expect(trace.states[0]?.body).toContain('title: "hello\\nworld"');
});
});
describe("recording state ownership", () => {
it("deduplicates within one recording and isolates ids between recordings", () => {
const first = new RecordingStateRegistry();
const second = new RecordingStateRegistry();
expect(first.register({ url: URL, rawVomText: "same" }).id).toBe("s1");
expect(first.register({ url: URL, rawVomText: "same" }).id).toBe("s1");
expect(second.register({ url: URL, rawVomText: "other" }).id).toBe("s1");
});
it("enriches metadata when a deduplicated observation becomes more complete", () => {
const registry = new RecordingStateRegistry();
registry.register({ url: URL, rawVomText: "same" });
const state = registry.register({
url: URL,
title: "Login",
rawVomText: "same",
truncated: true,
});
expect(state).toMatchObject({ id: "s1", title: "Login", truncated: true });
});
});
describe("draft binding", () => {
it("keeps capture semantics when no observation exists", () => {
const session = new RecordingObservationSession();
const draft: RecordingDraftStep = {
op: "hover",
captureTarget: { tag: "button", role: "button", name: "新建" },
};
session.bindDraft(draft, 1);
expect(draft.matchedTarget).toEqual({ role: "button", name: "新建", unmatched: true });
expect(draft.preStateId).toBeUndefined();
});
it("does not bind an unmatched new action to a stale observation", () => {
const session = sessionWithInput();
const draft: RecordingDraftStep = {
op: "click",
captureTarget: { tag: "button", role: "button", name: "Confirm" },
targetHint: {
geometry: { rect: { x: 400, y: 300, w: 80, h: 30 }, tag: "button" },
},
};
session.bindDraft(draft, 2, true);
expect(draft.preStateId).toBeUndefined();
expect("matchedTarget" in draft ? draft.matchedTarget : undefined).toEqual({
role: "button",
name: "Confirm",
unmatched: true,
});
});
});
@@ -0,0 +1,136 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
import { RecordingObservationSession } from "../recording/observation-session";
import { SettleController } from "../recording/settle-controller";
import type { RecordingDraftStep } from "../recording/types";
const OBSERVATION: RegisteredObservation = {
stateId: "s-next",
rootFrameId: "root",
index: new ObservationNodeIndex({ rootFrameId: "root", frameDocuments: [], refs: [] }),
url: "https://example.com/next",
};
describe("SettleController", () => {
afterEach(() => vi.useRealTimers());
it("aborts superseded capture work and lands the prior action on the next origin", async () => {
vi.useFakeTimers();
const cdp: CdpRunner = {
send: vi.fn(async () => ({
result: { value: { idleMs: 1_000, readyState: "complete" } },
})) as unknown as CdpRunner["send"],
};
const tabsApi: ChromeTabsApi = {
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
query: vi.fn(async () => []),
};
const session = new RecordingObservationSession();
let firstSignal: AbortSignal | undefined;
vi.spyOn(session, "capture")
.mockImplementationOnce(async (_cdp, _tabs, _tabId, signal) => {
firstSignal = signal;
return new Promise<RegisteredObservation>((_resolve, reject) => {
signal?.addEventListener(
"abort",
() => reject(new DOMException("observation aborted", "AbortError")),
{ once: true },
);
});
})
.mockResolvedValueOnce(OBSERVATION);
const drafts: RecordingDraftStep[] = [
{ op: "click", captureTarget: { tag: "button", name: "First" }, preStateId: "s1" },
];
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
controller.schedule(drafts, 0);
await vi.advanceTimersByTimeAsync(180);
drafts.push({
op: "click",
captureTarget: { tag: "button", name: "Second" },
preStateId: "s-next",
});
controller.schedule(drafts, 1);
expect(firstSignal?.aborted).toBe(true);
expect(drafts[0]?.postStateId).toBe("s-next");
await vi.advanceTimersByTimeAsync(500);
await controller.flush();
expect(drafts[1]?.postStateId).toBe("s-next");
});
it("does not consume a newer redirect while reading the prior landing URL", async () => {
vi.useFakeTimers();
const cdp: CdpRunner = {
send: vi.fn(async () => ({
result: { value: { idleMs: 1_000, readyState: "complete" } },
})) as unknown as CdpRunner["send"],
};
let resolveFirstTab!: (tab: chrome.tabs.Tab) => void;
const firstTab = new Promise<chrome.tabs.Tab>((resolve) => {
resolveFirstTab = resolve;
});
const tabsApi: ChromeTabsApi = {
get: vi
.fn()
.mockImplementationOnce(() => firstTab)
.mockResolvedValue({ id: 7, url: OBSERVATION.url }),
query: vi.fn(async () => []),
};
const session = new RecordingObservationSession();
session.cursor.lastSettled = OBSERVATION;
const drafts: RecordingDraftStep[] = [];
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
controller.scheduleRedirect(drafts, "https://example.com/intermediate");
await vi.advanceTimersByTimeAsync(180);
expect(tabsApi.get).toHaveBeenCalledTimes(1);
controller.scheduleRedirect(drafts, OBSERVATION.url);
resolveFirstTab({ id: 7, url: "https://example.com/intermediate" } as chrome.tabs.Tab);
await vi.advanceTimersByTimeAsync(500);
await controller.flushRedirects();
expect(drafts).toEqual([]);
expect(controller.hasPending).toBe(false);
});
it("settles an action in its own OOPIF document scope", async () => {
vi.useFakeTimers();
const send = vi.fn();
const sendToTarget = vi.fn(async (_target, method: string) => {
if (method === "Page.createIsolatedWorld") return { executionContextId: 39 };
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
});
const cdp: CdpRunner = {
send: send as unknown as CdpRunner["send"],
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
};
const tabsApi: ChromeTabsApi = {
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
query: vi.fn(async () => []),
};
const session = new RecordingObservationSession();
vi.spyOn(session, "capture").mockResolvedValue(OBSERVATION);
const drafts: RecordingDraftStep[] = [
{ op: "click", captureTarget: { tag: "button", name: "Inside frame" } },
];
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
const target = { tabId: 7, sessionId: "oopif-session" };
controller.schedule(drafts, 0, { target, frameId: "oopif-frame" });
await vi.advanceTimersByTimeAsync(180);
await controller.flush();
expect(send).not.toHaveBeenCalled();
expect(sendToTarget).toHaveBeenCalledWith(
target,
"Runtime.evaluate",
expect.objectContaining({ contextId: 39 }),
);
expect(drafts[0]?.postStateId).toBe("s-next");
});
});
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import { appendRecordedPayload, observeRecordedNavigation } from "../recording-step-buffer";
import { appendRecordedPayload, observeRecordedNavigation } from "../recording/step-buffer";
describe("recording-step-buffer", () => {
it("stores semantic click without summary", () => {
@@ -12,7 +12,7 @@ describe("recording-step-buffer", () => {
expect(buffer.steps).toEqual([
{
op: "click",
target: { tag: "button", role: "button", name: "发布" },
captureTarget: { tag: "button", role: "button", name: "发布" },
},
]);
expect(buffer.pendingNavigation).toBe(true);
@@ -23,7 +23,7 @@ describe("recording-step-buffer", () => {
steps: [
{
op: "click" as const,
target: { tag: "button", role: "button", name: "发布" },
captureTarget: { tag: "button", role: "button", name: "发布" },
},
],
currentUrl: "https://example.com/a",
@@ -34,8 +34,8 @@ describe("recording-step-buffer", () => {
expect(buffer.steps).toEqual([
{
op: "click",
target: { tag: "button", role: "button", name: "发布" },
navigated_to: "https://example.com/b",
captureTarget: { tag: "button", role: "button", name: "发布" },
navigatedTo: "https://example.com/b",
},
]);
expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
@@ -46,7 +46,7 @@ describe("recording-step-buffer", () => {
steps: [
{
op: "select" as const,
target: { tag: "select", role: "combobox", name: "分类" },
captureTarget: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
},
],
@@ -58,9 +58,9 @@ describe("recording-step-buffer", () => {
expect(buffer.steps).toEqual([
{
op: "select",
target: { tag: "select", role: "combobox", name: "分类" },
captureTarget: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
navigated_to: "https://example.com/list?cat=tech",
navigatedTo: "https://example.com/list?cat=tech",
},
]);
});
@@ -0,0 +1,126 @@
import type { RenderedRef } from "@browser-skill/vom";
import { describe, expect, it } from "vitest";
import type { CapturedNode } from "@/tools/vom/capture";
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
import { matchObservationTarget } from "../recording/target-matcher";
function node(backendNodeId: number, frameId: string, x = 10): CapturedNode {
return {
backendNodeId,
parentBackendNodeId: null,
frameId,
tag: "button",
attrs: {},
rect: { x, y: 20, w: 100, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
};
}
function observation(
documents: Array<{ frameId: string; domNodes: CapturedNode[] }>,
refs: RenderedRef[],
): RegisteredObservation {
return {
stateId: "s1",
rootFrameId: "root",
index: new ObservationNodeIndex({ rootFrameId: "root", frameDocuments: documents, refs }),
url: "https://example.com",
};
}
describe("matchObservationTarget", () => {
it("matches a unique node using canonical top-level viewport geometry", () => {
const target = matchObservationTarget({
observation: observation(
[{ frameId: "root", domNodes: [node(42, "root")] }],
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
});
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
});
it("uses frame id with backend node id so sibling frames cannot collide", () => {
const target = matchObservationTarget({
observation: observation(
[
{ frameId: "left", domNodes: [node(42, "left")] },
{ frameId: "right", domNodes: [node(42, "right")] },
],
[
{ ref: "e1", backendNodeId: 42, frameId: "left", line: 1 },
{ ref: "e2", backendNodeId: 42, frameId: "right", line: 2 },
],
),
hint: {
frameId: "right",
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
});
expect(target.ref).toBe("e2");
});
it("restricts a missing frame hint to the root frame", () => {
const target = matchObservationTarget({
observation: observation(
[
{ frameId: "root", domNodes: [] },
{ frameId: "child", domNodes: [node(42, "child")] },
],
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
fallback: { tag: "button", name: "发布" },
});
expect(target).toEqual({ name: "发布", unmatched: true });
});
it("returns unmatched for ambiguous geometry", () => {
const target = matchObservationTarget({
observation: observation(
[{ frameId: "root", domNodes: [node(42, "root"), node(43, "root")] }],
[
{ ref: "e1", backendNodeId: 42, line: 1 },
{ ref: "e2", backendNodeId: 43, line: 2 },
],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
});
expect(target.unmatched).toBe(true);
});
it("uses semantics when geometry is unavailable without crossing frame boundaries", () => {
const target = matchObservationTarget({
observation: observation(
[
{ frameId: "root", domNodes: [node(41, "root")] },
{ frameId: "child", domNodes: [node(42, "child")] },
],
[
{ ref: "e1", backendNodeId: 41, role: "button", name: "保存", line: 1 },
{ ref: "e2", backendNodeId: 42, frameId: "child", role: "button", name: "保存", line: 2 },
],
),
hint: { frameId: "child" },
fallback: { tag: "button", role: "button", name: "保存" },
});
expect(target.ref).toBe("e2");
});
it("uses semantics to disambiguate equal geometry in one frame", () => {
const target = matchObservationTarget({
observation: observation(
[{ frameId: "root", domNodes: [node(42, "root"), node(43, "root")] }],
[
{ ref: "e1", backendNodeId: 42, role: "button", name: "保存", line: 1 },
{ ref: "e2", backendNodeId: 43, role: "button", name: "取消", line: 2 },
],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
fallback: { tag: "button", role: "button", name: "取消" },
});
expect(target.ref).toBe("e2");
});
});
@@ -1,75 +1,196 @@
import { describe, expect, it } from "vitest";
import { buildTraceV2, shouldRecordPress } from "@/lib/trace-reducer-v2";
import type { DraftTraceStep } from "@/transport/types";
import type { RecordingDraftStep } from "@/lib/recording/types";
import { shouldRecordPress } from "../recording/draft-policy";
import { buildTraceV2 } from "../recording/trace-reducer-v2";
describe("trace-reducer-v2", () => {
it("drops scroll and bare character press steps", () => {
const steps: DraftTraceStep[] = [
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string) {
return buildTraceV2({
steps,
startedAt: "2026-01-01T00:00:00.000Z",
...(startUrl ? { startUrl } : {}),
});
}
describe("shouldRecordPress", () => {
it("keeps Enter and Escape", () => {
expect(shouldRecordPress("Enter")).toBe(true);
expect(shouldRecordPress("Escape")).toBe(true);
});
it("drops modifiers, clipboard shortcuts, and bare typing", () => {
expect(shouldRecordPress("Meta")).toBe(false);
expect(shouldRecordPress("c", ["meta"])).toBe(false);
expect(shouldRecordPress("a", ["ctrl"])).toBe(false);
expect(shouldRecordPress("x")).toBe(false);
expect(shouldRecordPress("中")).toBe(false);
});
});
describe("reduceTraceSteps", () => {
it("builds steps with pages dictionary and page id references", () => {
const drafts: RecordingDraftStep[] = [
{
op: "scroll",
page_url: "https://example.com/",
op: "navigate",
url: "https://example.com/search?q=hello&utm_source=x",
pageUrl: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "fill",
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
value: "browser skill",
pageUrl: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "press",
key: "Enter",
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
navigatedTo: "https://example.com/results/42",
pageUrl: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "click",
captureTarget: { tag: "button", role: "button", name: "发布" },
navigatedTo: "https://example.com/p/99",
pageUrl: "https://example.com/results/42",
},
{
op: "press",
key: "a",
page_url: "https://example.com/",
pageUrl: "https://example.com/p/99",
},
];
const trace = buildTraceV2({
steps,
startedAt: "2026-01-01T00:00:00.000Z",
startUrl: "https://example.com/",
const { pages, steps } = reduceTraceSteps(
drafts,
"https://example.com/search?q=hello&utm_source=x",
);
expect(JSON.stringify(steps)).not.toContain("parameters");
expect(JSON.stringify(steps)).not.toContain("intent");
expect(JSON.stringify(steps)).not.toContain("summary");
expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]);
expect(pages.map((p) => p.url)).toEqual([
"https://example.com/search?q=hello&utm_source=x",
"https://example.com/results/42",
"https://example.com/p/99",
]);
expect(steps[0]).toMatchObject({
id: 1,
op: "navigate",
page: "p1",
to: "https://example.com/search?q=hello&utm_source=x",
});
expect(steps[1]).toMatchObject({
op: "fill",
page: "p1",
value: "browser skill",
});
expect(steps[2]).toMatchObject({
op: "press",
key: "Enter",
page: "p1",
effect: { navigated_to: "p2" },
});
expect(steps[3]).toMatchObject({
op: "click",
page: "p2",
effect: { navigated_to: "p3" },
});
expect(trace.steps).toHaveLength(0);
expect(trace.pages).toHaveLength(1);
});
it("preserves hover steps supported by protocol v2", () => {
const trace = buildTraceV2({
steps: [
it("collapses consecutive navigations", () => {
const { steps } = reduceTraceSteps([
{ op: "navigate", url: "https://a.example/redirect1" },
{ op: "navigate", url: "https://a.example/final" },
]);
expect(steps).toHaveLength(1);
expect(steps[0]).toMatchObject({
op: "navigate",
to: "https://a.example/final",
});
});
it("keeps committed empty fill steps", () => {
const { steps } = reduceTraceSteps(
[
{
op: "hover",
target: { unmatched: true },
captureTarget: { tag: "button", role: "button", name: "结束" },
page_url: "https://example.com/",
op: "fill",
captureTarget: { tag: "input", role: "textbox", name: "Search query" },
value: "",
pageUrl: "https://example.com/search",
},
],
startedAt: "2026-01-01T00:00:00.000Z",
startUrl: "https://example.com/",
});
"https://example.com/search",
);
expect(trace.steps).toEqual([
expect(steps).toEqual([
expect.objectContaining({
op: "hover",
target: expect.objectContaining({ tag: "button", name: "结束" }),
op: "fill",
value: "",
}),
]);
});
it("maps captureTarget to v2 target with required tag", () => {
const steps: DraftTraceStep[] = [
{
op: "click",
target: { unmatched: true },
captureTarget: { tag: "button", role: "button", name: "Submit" },
page_url: "https://example.com/",
},
];
const trace = buildTraceV2({
steps,
startedAt: "2026-01-01T00:00:00.000Z",
startUrl: "https://example.com/",
it("maps select navigatedTo onto effect.navigatedTo (page id)", () => {
const { pages, steps } = reduceTraceSteps(
[
{
op: "select",
captureTarget: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
labels: ["技术"],
navigatedTo: "https://example.com/list?cat=tech",
pageUrl: "https://example.com/list",
},
],
"https://example.com/list",
);
expect(pages.map((p) => p.url)).toEqual([
"https://example.com/list",
"https://example.com/list?cat=tech",
]);
expect(steps[0]).toMatchObject({
op: "select",
page: "p1",
selection: [{ value: "tech", label: "技术" }],
effect: { navigated_to: "p2" },
});
expect(trace.steps).toHaveLength(1);
expect(trace.steps[0]?.op).toBe("click");
if (trace.steps[0]?.op === "click") {
expect(trace.steps[0].target.tag).toBe("button");
expect(trace.steps[0].target.name).toBe("Submit");
}
expect("version" in trace).toBe(false);
});
it("records Enter presses", () => {
expect(shouldRecordPress("Enter")).toBe(true);
it("keeps hover steps before menu clicks", () => {
const { steps } = reduceTraceSteps(
[
{
op: "hover",
captureTarget: { tag: "span", role: "button", name: "Account" },
pageUrl: "https://example.com/app",
},
{
op: "click",
captureTarget: { tag: "a", role: "link", name: "Profile" },
pageUrl: "https://example.com/app",
},
],
"https://example.com/app",
);
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
page: "p1",
target: { name: "Account" },
});
});
it("resolveTraceStartUrl prefers explicit start URL", () => {
expect(
buildTraceV2({
steps: [{ op: "navigate", url: "https://example.com/other" }],
startedAt: "2026-01-01T00:00:00.000Z",
startUrl: "https://example.com/start",
}).entry.start_url,
).toBe("https://example.com/start");
});
});
@@ -0,0 +1,62 @@
import { describe, expect, it } from "vitest";
import { TRACE_VERSION_V3, VOM_FORMAT_VERSION } from "@/transport/types";
import { RecordingStateRegistry } from "../recording/state-registry";
import { buildTraceV3 } from "../recording/trace-builder-v3";
import { reduceTraceStepsV3 } from "../recording/trace-reducer-v3";
import type { RecordingDraftStep } from "../recording/types";
describe("trace reducer v3", () => {
it("collapses redirect hops while retaining draft-to-step identity", () => {
const drafts: RecordingDraftStep[] = [
{ op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" },
{
op: "navigate",
url: "https://example.com/final",
transitionQualifiers: ["server_redirect"],
preStateId: "s2",
postStateId: "s3",
},
];
const before = structuredClone(drafts);
const reduced = reduceTraceStepsV3(drafts);
expect(drafts).toEqual(before);
expect(reduced.steps).toEqual([
expect.objectContaining({
id: 1,
op: "navigate",
state: "s1",
to: "https://example.com/final",
result: { state: "s3" },
}),
]);
expect(reduced.stepIdByDraftId.get(1)).toBe(1);
expect(reduced.stepIdByDraftId.get(2)).toBe(1);
});
it("builds the wire model from protocol constants", () => {
const registry = new RecordingStateRegistry();
const state = registry.register({ url: "https://example.com", rawVomText: "@vom 1" });
const trace = buildTraceV3({
registry,
drafts: [
{
op: "click",
captureTarget: { tag: "button", role: "button", name: "Save" },
preStateId: state.id,
postStateId: state.id,
},
],
startedAt: "2026-08-12T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "test",
});
expect(trace.version).toBe(TRACE_VERSION_V3);
expect(trace.recorder.vom).toBe(VOM_FORMAT_VERSION);
expect(trace.steps[0]).toMatchObject({
op: "click",
target: { role: "button", name: "Save", unmatched: true },
});
});
});
@@ -1,186 +0,0 @@
import { describe, expect, it } from "vitest";
import type { DraftTraceStep } from "@/transport/types";
import { reduceTraceSteps, resolveTraceStartUrl, shouldRecordPress } from "../trace-reducer";
describe("shouldRecordPress", () => {
it("keeps Enter and Escape", () => {
expect(shouldRecordPress("Enter")).toBe(true);
expect(shouldRecordPress("Escape")).toBe(true);
});
it("drops modifiers, clipboard shortcuts, and bare typing", () => {
expect(shouldRecordPress("Meta")).toBe(false);
expect(shouldRecordPress("c", ["meta"])).toBe(false);
expect(shouldRecordPress("a", ["ctrl"])).toBe(false);
expect(shouldRecordPress("x")).toBe(false);
expect(shouldRecordPress("中")).toBe(false);
});
});
describe("reduceTraceSteps", () => {
it("builds steps with pages dictionary and page id references", () => {
const drafts: DraftTraceStep[] = [
{
op: "navigate",
url: "https://example.com/search?q=hello&utm_source=x",
page_url: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "fill",
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
value: "browser skill",
page_url: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "press",
key: "Enter",
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
navigated_to: "https://example.com/results/42",
page_url: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "click",
target: { tag: "button", role: "button", name: "发布" },
navigated_to: "https://example.com/p/99",
page_url: "https://example.com/results/42",
},
{
op: "press",
key: "a",
page_url: "https://example.com/p/99",
},
];
const { pages, steps } = reduceTraceSteps(
drafts,
"https://example.com/search?q=hello&utm_source=x",
);
expect(JSON.stringify(steps)).not.toContain("parameters");
expect(JSON.stringify(steps)).not.toContain("intent");
expect(JSON.stringify(steps)).not.toContain("summary");
expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]);
expect(pages.map((p) => p.url)).toEqual([
"https://example.com/search?q=hello&utm_source=x",
"https://example.com/results/42",
"https://example.com/p/99",
]);
expect(steps[0]).toMatchObject({
id: 1,
op: "navigate",
page: "p1",
to: "https://example.com/search?q=hello&utm_source=x",
});
expect(steps[1]).toMatchObject({
op: "fill",
page: "p1",
value: "browser skill",
});
expect(steps[2]).toMatchObject({
op: "press",
key: "Enter",
page: "p1",
effect: { navigated_to: "p2" },
});
expect(steps[3]).toMatchObject({
op: "click",
page: "p2",
effect: { navigated_to: "p3" },
});
});
it("collapses consecutive navigations", () => {
const { steps } = reduceTraceSteps([
{ op: "navigate", url: "https://a.example/redirect1" },
{ op: "navigate", url: "https://a.example/final" },
]);
expect(steps).toHaveLength(1);
expect(steps[0]).toMatchObject({
op: "navigate",
to: "https://a.example/final",
});
});
it("keeps committed empty fill steps", () => {
const { steps } = reduceTraceSteps(
[
{
op: "fill",
target: { tag: "input", role: "textbox", name: "Search query" },
value: "",
page_url: "https://example.com/search",
},
],
"https://example.com/search",
);
expect(steps).toEqual([
expect.objectContaining({
op: "fill",
value: "",
}),
]);
});
it("maps select navigated_to onto effect.navigated_to (page id)", () => {
const { pages, steps } = reduceTraceSteps(
[
{
op: "select",
target: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
labels: ["技术"],
navigated_to: "https://example.com/list?cat=tech",
page_url: "https://example.com/list",
},
],
"https://example.com/list",
);
expect(pages.map((p) => p.url)).toEqual([
"https://example.com/list",
"https://example.com/list?cat=tech",
]);
expect(steps[0]).toMatchObject({
op: "select",
page: "p1",
selection: [{ value: "tech", label: "技术" }],
effect: { navigated_to: "p2" },
});
});
it("keeps hover steps before menu clicks", () => {
const { steps } = reduceTraceSteps(
[
{
op: "hover",
target: { tag: "span", role: "button", name: "Account" },
page_url: "https://example.com/app",
},
{
op: "click",
target: { tag: "a", role: "link", name: "Profile" },
page_url: "https://example.com/app",
},
],
"https://example.com/app",
);
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
page: "p1",
target: { name: "Account" },
});
});
it("resolveTraceStartUrl prefers explicit start URL", () => {
expect(
resolveTraceStartUrl(
[{ op: "navigate", url: "https://example.com/other" }],
"https://example.com/start",
),
).toBe("https://example.com/start");
});
});