Compare commits

...
Author SHA1 Message Date
Ljy-0827 6300eda0aa fix(record): rebase and dto policy 2026-08-23 21:26:40 +08:00
Ljy-0827 87cdcfe5ca fix(recording): using unified geometry 2026-08-23 21:26:40 +08:00
“haonan”andCursor 21476c3884 feat(record): capture Trace v3 states after pages settle
Wait for DOM quiet after actions, match targets to VOM refs, and reduce
drafts into deduped state-linked Trace v3 while keeping the v2 reducer.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-23 21:26:40 +08:00
“haonan” c2a225377b fix(protocol): align trace schemas with v2/v3 classify rules
RecordedTrace validation now follows classify_value — numeric version
selects v3, otherwise v2 — so mixed pages/states and v2 envelopes with
a version field fail Schema checks before Recall import. Keep standalone
trace_step as a v2|v3 union for tools that validate a single step.
2026-08-23 21:26:39 +08:00
“haonan” e64f4cac9f fix(protocol): keep generic trace schema compatible with v2 and v3
Split standalone v2/v3 schemas and drop TraceV2 deny_unknown_fields so legacy traces with extra keys still parse; mix and version checks stay in RecordedTrace.
2026-08-23 21:26:39 +08:00
Ljy-0827 368507b07d fix(protocol): compatible with dto 2026-08-23 21:26:39 +08:00
Ljy-0827 b750eed5d4 fix(protocol): integrate with vom 2026-08-23 21:26:39 +08:00
“haonan” 3d159c0254 test(protocol): align handshake fixtures with 1.0 floor and clarify v2/v3 naming 2026-08-23 21:26:39 +08:00
“haonan”andCursor 524516cb56 fix(protocol): align Trace v2/v3 classification and hover steps
Require Trace v3 version const 3, reject mixed pages/states payloads, and
align TypeScript selection optionality with the generated schema. Trace v2
on main already records hover steps, so the frozen v2 schema must accept
them instead of dropping them during reduction.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-23 21:26:39 +08:00
“haonan”andCursor f98198949a feat(protocol): negotiate Trace v3 with v2 compatibility
Define Trace v3 wire types and generated schemas while keeping protocol
1.1 peers compatible with 1.0. Default record start still produces Trace
v2, and only an explicit trace_version=3 request enables the new shape.
Refs #90

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-23 21:26:39 +08:00
40 changed files with 7118 additions and 1066 deletions
+12 -9
View File
@@ -1,9 +1,9 @@
import {
type CaptureTargetDescriptor,
describeEventTarget,
describeTarget,
resolveClickableElement,
resolveHoverElement,
type TargetDescriptor,
} from "@/lib/describe-target";
import {
evaluateHoverTrigger,
@@ -27,7 +27,7 @@ import {
type RecordStopAck,
type RecordStopMessage,
} from "@/lib/record-bridge";
import { shouldRecordPress } from "@/lib/trace-reducer";
import { shouldRecordPress } from "@/lib/recording/draft-policy";
import {
closestHoverSurfaceCandidate,
collectHoverSurfaceStates,
@@ -65,14 +65,14 @@ export interface RecordCaptureController {
interface FillSession {
element: FillableElement;
target: TargetDescriptor;
target: CaptureTargetDescriptor;
baselineValue: string;
lastValue: string;
}
interface HoverCandidate {
element: Element;
target: TargetDescriptor;
target: CaptureTargetDescriptor;
recordedAt: number;
score: number;
eligible: boolean;
@@ -232,7 +232,10 @@ function collectHoverTriggerLabelText(root: Element): string {
return normalizeLabelText(text);
}
function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor {
function compactHoverTargetName(
el: Element,
desc: CaptureTargetDescriptor,
): CaptureTargetDescriptor {
if (!desc.name) return desc;
const fullText = normalizeLabelText(el.textContent ?? "");
const compactName = collectHoverTriggerLabelText(el);
@@ -250,7 +253,7 @@ function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDesc
return desc;
}
function isWeakHoverTarget(target: TargetDescriptor): boolean {
function isWeakHoverTarget(target: CaptureTargetDescriptor): boolean {
return !target.role && !target.name && target.tag === "div";
}
@@ -261,9 +264,9 @@ function looksLikeAvatarElement(el: Element): boolean {
function normalizeHoverTarget(
el: Element,
desc: TargetDescriptor,
desc: CaptureTargetDescriptor,
decision: HoverTriggerDecision,
): TargetDescriptor {
): CaptureTargetDescriptor {
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
return { ...desc, name: "image" };
}
@@ -277,7 +280,7 @@ function normalizeHoverTarget(
return desc;
}
function hoverTriggerSignals(el: Element, desc: TargetDescriptor) {
function hoverTriggerSignals(el: Element, desc: CaptureTargetDescriptor) {
if (!(el instanceof HTMLElement)) return null;
const style = hoverTriggerStyle(el);
return {
@@ -0,0 +1,101 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpRunner } from "@/tools/shared";
import { waitForDocumentSettled } from "../recording/document-settle";
function quietCdp(): { cdp: CdpRunner; send: ReturnType<typeof vi.fn> } {
const send = vi.fn(async () => ({
result: { value: { idleMs: 1_000, readyState: "complete" } },
}));
return {
cdp: { send: send as unknown as CdpRunner["send"] },
send,
};
}
describe("waitForDocumentSettled", () => {
afterEach(() => {
vi.useRealTimers();
});
it("waits through the minimum observation floor before accepting a quiet page", async () => {
vi.useFakeTimers();
const { cdp, send } = quietCdp();
const settled = waitForDocumentSettled(cdp, { target: { tabId: 7 } });
await vi.advanceTimersByTimeAsync(180);
await expect(settled).resolves.toBe("quiet");
expect(send).toHaveBeenCalledTimes(3);
});
it("cancels before probing a superseded observation", async () => {
vi.useFakeTimers();
const { cdp, send } = quietCdp();
const controller = new AbortController();
const settled = waitForDocumentSettled(
cdp,
{ target: { tabId: 7 } },
{
signal: controller.signal,
},
);
controller.abort();
await vi.advanceTimersByTimeAsync(60);
await expect(settled).resolves.toBe("cancelled");
expect(send).not.toHaveBeenCalled();
});
it("probes a same-process iframe through its isolated execution context", async () => {
vi.useFakeTimers();
const send = vi.fn(async (_tabId: number, method: string) => {
if (method === "Page.createIsolatedWorld") return { executionContextId: 91 };
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
});
const cdp: CdpRunner = { send: send as unknown as CdpRunner["send"] };
const settled = waitForDocumentSettled(cdp, {
target: { tabId: 7 },
frameId: "child-frame",
});
await vi.advanceTimersByTimeAsync(180);
await expect(settled).resolves.toBe("quiet");
expect(send).toHaveBeenCalledWith(
7,
"Page.createIsolatedWorld",
expect.objectContaining({ frameId: "child-frame" }),
);
expect(send).toHaveBeenCalledWith(
7,
"Runtime.evaluate",
expect.objectContaining({ contextId: 91 }),
);
});
it("routes an OOPIF probe through its CDP target session", async () => {
vi.useFakeTimers();
const send = vi.fn();
const sendToTarget = vi.fn(async (_target, method: string) => {
if (method === "Page.createIsolatedWorld") return { executionContextId: 27 };
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
});
const cdp: CdpRunner = {
send: send as unknown as CdpRunner["send"],
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
};
const target = { tabId: 7, sessionId: "oopif-session" };
const settled = waitForDocumentSettled(cdp, { target, frameId: "oopif-frame" });
await vi.advanceTimersByTimeAsync(180);
await expect(settled).resolves.toBe("quiet");
expect(send).not.toHaveBeenCalled();
expect(sendToTarget).toHaveBeenCalledWith(
target,
"Runtime.evaluate",
expect.objectContaining({ contextId: 27 }),
);
});
});
@@ -0,0 +1,141 @@
import { describe, expect, it } from "vitest";
import { ObservationNodeIndex } from "../recording/observation-capture";
import { RecordingObservationSession } from "../recording/observation-session";
import { RecordingStateRegistry } from "../recording/state-registry";
import { buildTraceV3 } from "../recording/trace-builder-v3";
import type { RecordingDraftStep } from "../recording/types";
const URL = "https://example.com/login";
function sessionWithInput(redactValues = false): RecordingObservationSession {
const session = new RecordingObservationSession({ redactValues });
const state = session.registry.register({
url: URL,
rawVomText: '@vom 1\ntextbox "Password" value="••••••" [ref=e1]',
});
session.cursor.lastSettled = {
stateId: state.id,
rootFrameId: "root",
index: new ObservationNodeIndex({
rootFrameId: "root",
matchNodes: [
{
frameId: "root",
backendNodeId: 42,
tag: "input",
rect: { x: 20, y: 40, w: 200, h: 30 },
localRect: { x: 20, y: 40, w: 200, h: 30 },
},
],
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
}),
url: URL,
};
return session;
}
function finalizedFillBody(value: string, redactValues: boolean): string {
const session = sessionWithInput(redactValues);
const draft: RecordingDraftStep = {
op: "fill",
captureTarget: { tag: "input", role: "textbox", name: "Password" },
value,
targetHint: {
geometry: { rect: { x: 20, y: 40, w: 200, h: 30 }, tag: "input" },
},
};
session.bindDraft(draft, 1);
draft.postStateId = draft.preStateId;
return buildTraceV3({
registry: session.registry,
drafts: [draft],
annotations: session.annotations,
startedAt: "2026-08-12T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "test",
}).states[0]!.body;
}
describe("record observation annotations", () => {
it("omits a fill literal when values are redacted", () => {
const secret = "hunter2-private";
const body = finalizedFillBody(secret, true);
expect(body).toContain("step 1: fill");
expect(body).not.toContain(secret);
});
it("keeps ordinary fill details", () => {
expect(finalizedFillBody("ordinary text", false)).toContain('step 1: fill: "ordinary text"');
});
it("encodes title and URL so line breaks cannot corrupt state metadata", () => {
const registry = new RecordingStateRegistry();
const state = registry.register({
url: "https://example.com/a\nb",
title: "hello\nworld",
rawVomText: "@vom 1",
});
const trace = buildTraceV3({
registry,
drafts: [{ op: "scroll", preStateId: state.id, postStateId: state.id }],
startedAt: "2026-08-12T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "test",
});
expect(trace.states[0]?.body).toContain('url: "https://example.com/a\\nb"');
expect(trace.states[0]?.body).toContain('title: "hello\\nworld"');
});
});
describe("recording state ownership", () => {
it("deduplicates within one recording and isolates ids between recordings", () => {
const first = new RecordingStateRegistry();
const second = new RecordingStateRegistry();
expect(first.register({ url: URL, rawVomText: "same" }).id).toBe("s1");
expect(first.register({ url: URL, rawVomText: "same" }).id).toBe("s1");
expect(second.register({ url: URL, rawVomText: "other" }).id).toBe("s1");
});
it("enriches metadata when a deduplicated observation becomes more complete", () => {
const registry = new RecordingStateRegistry();
registry.register({ url: URL, rawVomText: "same" });
const state = registry.register({
url: URL,
title: "Login",
rawVomText: "same",
truncated: true,
});
expect(state).toMatchObject({ id: "s1", title: "Login", truncated: true });
});
});
describe("draft binding", () => {
it("keeps capture semantics when no observation exists", () => {
const session = new RecordingObservationSession();
const draft: RecordingDraftStep = {
op: "hover",
captureTarget: { tag: "button", role: "button", name: "新建" },
};
session.bindDraft(draft, 1);
expect(draft.matchedTarget).toEqual({ role: "button", name: "新建", unmatched: true });
expect(draft.preStateId).toBeUndefined();
});
it("does not bind an unmatched new action to a stale observation", () => {
const session = sessionWithInput();
const draft: RecordingDraftStep = {
op: "click",
captureTarget: { tag: "button", role: "button", name: "Confirm" },
targetHint: {
geometry: { rect: { x: 400, y: 300, w: 80, h: 30 }, tag: "button" },
},
};
session.bindDraft(draft, 2, true);
expect(draft.preStateId).toBeUndefined();
expect("matchedTarget" in draft ? draft.matchedTarget : undefined).toEqual({
role: "button",
name: "Confirm",
unmatched: true,
});
});
});
@@ -0,0 +1,136 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
import { RecordingObservationSession } from "../recording/observation-session";
import { SettleController } from "../recording/settle-controller";
import type { RecordingDraftStep } from "../recording/types";
const OBSERVATION: RegisteredObservation = {
stateId: "s-next",
rootFrameId: "root",
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
url: "https://example.com/next",
};
describe("SettleController", () => {
afterEach(() => vi.useRealTimers());
it("aborts superseded capture work and lands the prior action on the next origin", async () => {
vi.useFakeTimers();
const cdp: CdpRunner = {
send: vi.fn(async () => ({
result: { value: { idleMs: 1_000, readyState: "complete" } },
})) as unknown as CdpRunner["send"],
};
const tabsApi: ChromeTabsApi = {
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
query: vi.fn(async () => []),
};
const session = new RecordingObservationSession();
let firstSignal: AbortSignal | undefined;
vi.spyOn(session, "capture")
.mockImplementationOnce(async (_cdp, _tabs, _tabId, signal) => {
firstSignal = signal;
return new Promise<RegisteredObservation>((_resolve, reject) => {
signal?.addEventListener(
"abort",
() => reject(new DOMException("observation aborted", "AbortError")),
{ once: true },
);
});
})
.mockResolvedValueOnce(OBSERVATION);
const drafts: RecordingDraftStep[] = [
{ op: "click", captureTarget: { tag: "button", name: "First" }, preStateId: "s1" },
];
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
controller.schedule(drafts, 0);
await vi.advanceTimersByTimeAsync(180);
drafts.push({
op: "click",
captureTarget: { tag: "button", name: "Second" },
preStateId: "s-next",
});
controller.schedule(drafts, 1);
expect(firstSignal?.aborted).toBe(true);
expect(drafts[0]?.postStateId).toBe("s-next");
await vi.advanceTimersByTimeAsync(500);
await controller.flush();
expect(drafts[1]?.postStateId).toBe("s-next");
});
it("does not consume a newer redirect while reading the prior landing URL", async () => {
vi.useFakeTimers();
const cdp: CdpRunner = {
send: vi.fn(async () => ({
result: { value: { idleMs: 1_000, readyState: "complete" } },
})) as unknown as CdpRunner["send"],
};
let resolveFirstTab!: (tab: chrome.tabs.Tab) => void;
const firstTab = new Promise<chrome.tabs.Tab>((resolve) => {
resolveFirstTab = resolve;
});
const tabsApi: ChromeTabsApi = {
get: vi
.fn()
.mockImplementationOnce(() => firstTab)
.mockResolvedValue({ id: 7, url: OBSERVATION.url }),
query: vi.fn(async () => []),
};
const session = new RecordingObservationSession();
session.cursor.lastSettled = OBSERVATION;
const drafts: RecordingDraftStep[] = [];
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
controller.scheduleRedirect(drafts, "https://example.com/intermediate");
await vi.advanceTimersByTimeAsync(180);
expect(tabsApi.get).toHaveBeenCalledTimes(1);
controller.scheduleRedirect(drafts, OBSERVATION.url);
resolveFirstTab({ id: 7, url: "https://example.com/intermediate" } as chrome.tabs.Tab);
await vi.advanceTimersByTimeAsync(500);
await controller.flushRedirects();
expect(drafts).toEqual([]);
expect(controller.hasPending).toBe(false);
});
it("settles an action in its own OOPIF document scope", async () => {
vi.useFakeTimers();
const send = vi.fn();
const sendToTarget = vi.fn(async (_target, method: string) => {
if (method === "Page.createIsolatedWorld") return { executionContextId: 39 };
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
});
const cdp: CdpRunner = {
send: send as unknown as CdpRunner["send"],
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
};
const tabsApi: ChromeTabsApi = {
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
query: vi.fn(async () => []),
};
const session = new RecordingObservationSession();
vi.spyOn(session, "capture").mockResolvedValue(OBSERVATION);
const drafts: RecordingDraftStep[] = [
{ op: "click", captureTarget: { tag: "button", name: "Inside frame" } },
];
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
const target = { tabId: 7, sessionId: "oopif-session" };
controller.schedule(drafts, 0, { target, frameId: "oopif-frame" });
await vi.advanceTimersByTimeAsync(180);
await controller.flush();
expect(send).not.toHaveBeenCalled();
expect(sendToTarget).toHaveBeenCalledWith(
target,
"Runtime.evaluate",
expect.objectContaining({ contextId: 39 }),
);
expect(drafts[0]?.postStateId).toBe("s-next");
});
});
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import { appendRecordedPayload, observeRecordedNavigation } from "../recording-step-buffer";
import { appendRecordedPayload, observeRecordedNavigation } from "../recording/step-buffer";
describe("recording-step-buffer", () => {
it("stores semantic click without summary", () => {
@@ -12,7 +12,7 @@ describe("recording-step-buffer", () => {
expect(buffer.steps).toEqual([
{
op: "click",
target: { tag: "button", role: "button", name: "发布" },
captureTarget: { tag: "button", role: "button", name: "发布" },
},
]);
expect(buffer.pendingNavigation).toBe(true);
@@ -23,7 +23,7 @@ describe("recording-step-buffer", () => {
steps: [
{
op: "click" as const,
target: { tag: "button", role: "button", name: "发布" },
captureTarget: { tag: "button", role: "button", name: "发布" },
},
],
currentUrl: "https://example.com/a",
@@ -34,8 +34,8 @@ describe("recording-step-buffer", () => {
expect(buffer.steps).toEqual([
{
op: "click",
target: { tag: "button", role: "button", name: "发布" },
navigated_to: "https://example.com/b",
captureTarget: { tag: "button", role: "button", name: "发布" },
navigatedTo: "https://example.com/b",
},
]);
expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
@@ -46,7 +46,7 @@ describe("recording-step-buffer", () => {
steps: [
{
op: "select" as const,
target: { tag: "select", role: "combobox", name: "分类" },
captureTarget: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
},
],
@@ -58,9 +58,9 @@ describe("recording-step-buffer", () => {
expect(buffer.steps).toEqual([
{
op: "select",
target: { tag: "select", role: "combobox", name: "分类" },
captureTarget: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
navigated_to: "https://example.com/list?cat=tech",
navigatedTo: "https://example.com/list?cat=tech",
},
]);
});
@@ -0,0 +1,127 @@
import type { RenderedRef } from "@browser-skill/vom";
import { describe, expect, it } from "vitest";
import type { CaptureVomMatchNode } from "@/tools/capture-vom-observation";
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
import { matchObservationTarget } from "../recording/target-matcher";
function node(backendNodeId: number, frameId: string, x = 10): CaptureVomMatchNode {
return {
backendNodeId,
frameId,
tag: "button",
rect: { x, y: 20, w: 100, h: 30 },
localRect: { x, y: 20, w: 100, h: 30 },
};
}
function observation(
matchNodes: CaptureVomMatchNode[],
refs: RenderedRef[],
): RegisteredObservation {
return {
stateId: "s1",
rootFrameId: "root",
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes, refs }),
url: "https://example.com",
};
}
describe("matchObservationTarget", () => {
it("indexes the safe geometry DTO without discarding frame-local geometry", () => {
const geometry = node(42, "child");
geometry.localRect = { x: 5, y: 6, w: 100, h: 30 };
const ref: RenderedRef = {
ref: "e1",
backendNodeId: 42,
frameId: "child",
line: 1,
};
const candidate = observation([geometry], [ref]).index.candidates("child", "button")[0];
expect(candidate).toEqual({ frameId: "child", geometry, ref });
});
it("matches a unique node using canonical top-level viewport geometry", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "root")],
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
});
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
});
it("uses frame id with backend node id so sibling frames cannot collide", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "left"), node(42, "right")],
[
{ ref: "e1", backendNodeId: 42, frameId: "left", line: 1 },
{ ref: "e2", backendNodeId: 42, frameId: "right", line: 2 },
],
),
hint: {
frameId: "right",
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
});
expect(target.ref).toBe("e2");
});
it("restricts a missing frame hint to the root frame", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "child")],
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
fallback: { tag: "button", name: "发布" },
});
expect(target).toEqual({ name: "发布", unmatched: true });
});
it("returns unmatched for ambiguous geometry", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "root"), node(43, "root")],
[
{ ref: "e1", backendNodeId: 42, line: 1 },
{ ref: "e2", backendNodeId: 43, line: 2 },
],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
});
expect(target.unmatched).toBe(true);
});
it("uses semantics when geometry is unavailable without crossing frame boundaries", () => {
const target = matchObservationTarget({
observation: observation(
[node(41, "root"), node(42, "child")],
[
{ ref: "e1", backendNodeId: 41, role: "button", name: "保存", line: 1 },
{ ref: "e2", backendNodeId: 42, frameId: "child", role: "button", name: "保存", line: 2 },
],
),
hint: { frameId: "child" },
fallback: { tag: "button", role: "button", name: "保存" },
});
expect(target.ref).toBe("e2");
});
it("uses semantics to disambiguate equal geometry in one frame", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "root"), node(43, "root")],
[
{ ref: "e1", backendNodeId: 42, role: "button", name: "保存", line: 1 },
{ ref: "e2", backendNodeId: 43, role: "button", name: "取消", line: 2 },
],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
fallback: { tag: "button", role: "button", name: "取消" },
});
expect(target.ref).toBe("e2");
});
});
@@ -1,6 +1,15 @@
import { describe, expect, it } from "vitest";
import type { DraftTraceStep } from "@/transport/types";
import { reduceTraceSteps, resolveTraceStartUrl, shouldRecordPress } from "../trace-reducer";
import type { RecordingDraftStep } from "@/lib/recording/types";
import { shouldRecordPress } from "../recording/draft-policy";
import { buildTraceV2 } from "../recording/trace-reducer-v2";
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string) {
return buildTraceV2({
steps,
startedAt: "2026-01-01T00:00:00.000Z",
...(startUrl ? { startUrl } : {}),
});
}
describe("shouldRecordPress", () => {
it("keeps Enter and Escape", () => {
@@ -19,35 +28,35 @@ describe("shouldRecordPress", () => {
describe("reduceTraceSteps", () => {
it("builds steps with pages dictionary and page id references", () => {
const drafts: DraftTraceStep[] = [
const drafts: RecordingDraftStep[] = [
{
op: "navigate",
url: "https://example.com/search?q=hello&utm_source=x",
page_url: "https://example.com/search?q=hello&utm_source=x",
pageUrl: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "fill",
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
value: "browser skill",
page_url: "https://example.com/search?q=hello&utm_source=x",
pageUrl: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "press",
key: "Enter",
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
navigated_to: "https://example.com/results/42",
page_url: "https://example.com/search?q=hello&utm_source=x",
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
navigatedTo: "https://example.com/results/42",
pageUrl: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "click",
target: { tag: "button", role: "button", name: "发布" },
navigated_to: "https://example.com/p/99",
page_url: "https://example.com/results/42",
captureTarget: { tag: "button", role: "button", name: "发布" },
navigatedTo: "https://example.com/p/99",
pageUrl: "https://example.com/results/42",
},
{
op: "press",
key: "a",
page_url: "https://example.com/p/99",
pageUrl: "https://example.com/p/99",
},
];
@@ -109,9 +118,9 @@ describe("reduceTraceSteps", () => {
[
{
op: "fill",
target: { tag: "input", role: "textbox", name: "Search query" },
captureTarget: { tag: "input", role: "textbox", name: "Search query" },
value: "",
page_url: "https://example.com/search",
pageUrl: "https://example.com/search",
},
],
"https://example.com/search",
@@ -125,16 +134,16 @@ describe("reduceTraceSteps", () => {
]);
});
it("maps select navigated_to onto effect.navigated_to (page id)", () => {
it("maps select navigatedTo onto effect.navigatedTo (page id)", () => {
const { pages, steps } = reduceTraceSteps(
[
{
op: "select",
target: { tag: "select", role: "combobox", name: "分类" },
captureTarget: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
labels: ["技术"],
navigated_to: "https://example.com/list?cat=tech",
page_url: "https://example.com/list",
navigatedTo: "https://example.com/list?cat=tech",
pageUrl: "https://example.com/list",
},
],
"https://example.com/list",
@@ -156,13 +165,13 @@ describe("reduceTraceSteps", () => {
[
{
op: "hover",
target: { tag: "span", role: "button", name: "Account" },
page_url: "https://example.com/app",
captureTarget: { tag: "span", role: "button", name: "Account" },
pageUrl: "https://example.com/app",
},
{
op: "click",
target: { tag: "a", role: "link", name: "Profile" },
page_url: "https://example.com/app",
captureTarget: { tag: "a", role: "link", name: "Profile" },
pageUrl: "https://example.com/app",
},
],
"https://example.com/app",
@@ -177,10 +186,11 @@ describe("reduceTraceSteps", () => {
it("resolveTraceStartUrl prefers explicit start URL", () => {
expect(
resolveTraceStartUrl(
[{ op: "navigate", url: "https://example.com/other" }],
"https://example.com/start",
),
buildTraceV2({
steps: [{ op: "navigate", url: "https://example.com/other" }],
startedAt: "2026-01-01T00:00:00.000Z",
startUrl: "https://example.com/start",
}).entry.start_url,
).toBe("https://example.com/start");
});
});
@@ -0,0 +1,62 @@
import { describe, expect, it } from "vitest";
import { TRACE_VERSION_V3, VOM_FORMAT_VERSION } from "@/transport/types";
import { RecordingStateRegistry } from "../recording/state-registry";
import { buildTraceV3 } from "../recording/trace-builder-v3";
import { reduceTraceStepsV3 } from "../recording/trace-reducer-v3";
import type { RecordingDraftStep } from "../recording/types";
describe("trace reducer v3", () => {
it("collapses redirect hops while retaining draft-to-step identity", () => {
const drafts: RecordingDraftStep[] = [
{ op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" },
{
op: "navigate",
url: "https://example.com/final",
transitionQualifiers: ["server_redirect"],
preStateId: "s2",
postStateId: "s3",
},
];
const before = structuredClone(drafts);
const reduced = reduceTraceStepsV3(drafts);
expect(drafts).toEqual(before);
expect(reduced.steps).toEqual([
expect.objectContaining({
id: 1,
op: "navigate",
state: "s1",
to: "https://example.com/final",
result: { state: "s3" },
}),
]);
expect(reduced.stepIdByDraftId.get(1)).toBe(1);
expect(reduced.stepIdByDraftId.get(2)).toBe(1);
});
it("builds the wire model from protocol constants", () => {
const registry = new RecordingStateRegistry();
const state = registry.register({ url: "https://example.com", rawVomText: "@vom 1" });
const trace = buildTraceV3({
registry,
drafts: [
{
op: "click",
captureTarget: { tag: "button", role: "button", name: "Save" },
preStateId: state.id,
postStateId: state.id,
},
],
startedAt: "2026-08-12T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "test",
});
expect(trace.version).toBe(TRACE_VERSION_V3);
expect(trace.recorder.vom).toBe(VOM_FORMAT_VERSION);
expect(trace.steps[0]).toMatchObject({
op: "click",
target: { role: "button", name: "Save", unmatched: true },
});
});
});
+6 -5
View File
@@ -1,12 +1,13 @@
/**
* Build a semantic TargetDescriptor for an interacted element.
* Build a semantic capture descriptor for an interacted element.
*
* Trace steps are an LLM *textbook*: each click must say what to look for
* on screen (usually a short visible name). Tag-only noise like
* `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
*/
export interface TargetDescriptor {
/** Content-script capture descriptor before VOM geometric matching. */
export interface CaptureTargetDescriptor {
role?: string;
name?: string;
tag: string;
@@ -247,7 +248,7 @@ export function resolveHoverElement(target: Element): Element | null {
* (visible name), or at least a form `name_attr` for checkbox/radio.
* Recording “点击div” with no name fails this bar.
*/
export function isMeaningfulClickTarget(target: TargetDescriptor): boolean {
export function isMeaningfulClickTarget(target: CaptureTargetDescriptor): boolean {
const name = target.name?.trim();
if (name && isActionableLabel(name)) return true;
if (
@@ -323,7 +324,7 @@ function nearbyLabelText(el: Element): string | undefined {
return undefined;
}
export function describeTarget(el: Element): TargetDescriptor {
export function describeTarget(el: Element): CaptureTargetDescriptor {
const tag = el.tagName.toLowerCase();
const role = inferRole(el);
const name = accessibleName(el);
@@ -350,7 +351,7 @@ export function describeTarget(el: Element): TargetDescriptor {
};
}
export function describeEventTarget(target: EventTarget | null): TargetDescriptor | null {
export function describeEventTarget(target: EventTarget | null): CaptureTargetDescriptor | null {
if (!(target instanceof Element)) return null;
const clickable = resolveClickableElement(target);
if (!clickable) return null;
+2 -2
View File
@@ -3,7 +3,7 @@
* service worker and a tab's content script.
*/
import type { TargetDescriptor } from "./describe-target";
import type { CaptureTargetDescriptor } from "./describe-target";
export const RECORD_START = "bsk-record-start";
export const RECORD_STEP = "bsk-record-step";
@@ -42,7 +42,7 @@ export interface RecordStartMessage {
export interface RecordStepPayload {
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
target?: TargetDescriptor;
target?: CaptureTargetDescriptor;
value?: string;
key?: string;
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
@@ -0,0 +1,134 @@
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { type CdpRunner, sendToCdpTarget } from "@/tools/shared";
const SETTLE_MIN_MS = 150;
const SETTLE_QUIET_MS = 250;
const SETTLE_MAX_MS = 2_000;
const SETTLE_POLL_MS = 60;
const SETTLE_WORLD_NAME = "__bsk_record_settle__";
const QUIET_PROBE = `(() => {
const scope = window;
let probe = scope.__bskRecordQuiet;
if (!probe) {
probe = { changedAt: Date.now() };
const observer = new MutationObserver(() => {
probe.changedAt = Date.now();
});
observer.observe(document.documentElement, {
subtree: true,
childList: true,
attributes: true,
characterData: true,
});
scope.__bskRecordQuiet = probe;
}
return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState };
})()`;
export interface DocumentSettleScope {
target: CdpTarget;
/** Omit only for the target's root document. */
frameId?: string;
}
type SettleOutcome = "quiet" | "timeout" | "cancelled";
interface QuietProbe {
idleMs: number;
readyState: string;
}
interface ProbeContext {
executionContextId?: number;
}
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve) => {
if (signal?.aborted) return resolve();
const timer = setTimeout(resolve, ms);
signal?.addEventListener(
"abort",
() => {
clearTimeout(timer);
resolve();
},
{ once: true },
);
});
}
async function executionContextForScope(
cdp: CdpRunner,
scope: DocumentSettleScope,
): Promise<number | undefined> {
if (!scope.frameId) return undefined;
const result = await sendToCdpTarget<{ executionContextId?: number }>(
cdp,
scope.target,
"Page.createIsolatedWorld",
{
frameId: scope.frameId,
worldName: SETTLE_WORLD_NAME,
grantUniveralAccess: false,
},
);
return result.executionContextId;
}
async function readQuietProbe(
cdp: CdpRunner,
scope: DocumentSettleScope,
context: ProbeContext,
): Promise<QuietProbe | null> {
try {
if (scope.frameId && context.executionContextId === undefined) {
context.executionContextId = await executionContextForScope(cdp, scope);
if (context.executionContextId === undefined) return null;
}
const reply = await sendToCdpTarget<{ result?: { value?: unknown } }>(
cdp,
scope.target,
"Runtime.evaluate",
{
expression: QUIET_PROBE,
returnByValue: true,
...(context.executionContextId !== undefined
? { contextId: context.executionContextId }
: {}),
},
);
const value = reply.result?.value;
if (!value || typeof value !== "object") return null;
const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown };
if (typeof idleMs !== "number" || typeof readyState !== "string") return null;
return { idleMs, readyState };
} catch {
context.executionContextId = undefined;
return null;
}
}
export async function waitForDocumentSettled(
cdp: CdpRunner,
scope: DocumentSettleScope,
options: { signal?: AbortSignal } = {},
): Promise<SettleOutcome> {
const startedAt = Date.now();
const floor = startedAt + SETTLE_MIN_MS;
const deadline = startedAt + SETTLE_MAX_MS;
const context: ProbeContext = {};
for (;;) {
if (options.signal?.aborted) return "cancelled";
await sleep(SETTLE_POLL_MS, options.signal);
if (options.signal?.aborted) return "cancelled";
const probe = await readQuietProbe(cdp, scope, context);
const now = Date.now();
if (now < floor) continue;
if (now >= deadline) return "timeout";
if (!probe || probe.readyState === "loading") continue;
if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet";
}
}
@@ -0,0 +1,34 @@
import type { RecordingDraftStep } from "./types";
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
export function shouldRecordPress(
key: string,
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
): boolean {
if (MODIFIER_ONLY_KEYS.has(key)) return false;
const mods = modifiers ?? [];
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
if (key === "Enter" || key === "Escape") return true;
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
return false;
}
export function shouldIncludeDraft(step: RecordingDraftStep): boolean {
return step.op !== "press" || shouldRecordPress(step.key, step.modifiers);
}
export function resolveDraftStartUrl(
drafts: RecordingDraftStep[],
explicitStartUrl?: string,
fallbackUrl?: string,
): string {
if (explicitStartUrl) return explicitStartUrl;
const navigation = drafts.find(
(step): step is Extract<RecordingDraftStep, { op: "navigate" }> => step.op === "navigate",
);
if (navigation) return navigation.url;
return drafts.find((step) => step.pageUrl)?.pageUrl ?? fallbackUrl ?? "about:blank";
}
@@ -0,0 +1,116 @@
import type { RenderedRef } from "@browser-skill/vom";
import {
type CaptureVomMatchNode,
type CaptureVomObservationResult,
captureVomObservation,
} from "@/tools/capture-vom-observation";
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
export interface IndexedObservationNode {
frameId: string;
geometry: CaptureVomMatchNode;
ref?: RenderedRef;
}
export interface CapturedRecordingObservation {
rootFrameId: string;
index: ObservationNodeIndex;
url: string;
title?: string;
vomText: string;
truncated: boolean;
}
export interface RegisteredObservation {
stateId: string;
rootFrameId: string;
index: ObservationNodeIndex;
url: string;
}
function nodeKey(frameId: string, backendNodeId: number): string {
return `${frameId}:${backendNodeId}`;
}
function frameTagKey(frameId: string, tag: string): string {
return `${frameId}:${tag.toLowerCase()}`;
}
export class ObservationNodeIndex {
readonly #nodesByFrameTag = new Map<string, IndexedObservationNode[]>();
readonly #refById = new Map<string, RenderedRef>();
readonly #refsByFrame = new Map<string, RenderedRef[]>();
constructor(input: Pick<CaptureVomObservationResult, "rootFrameId" | "matchNodes" | "refs">) {
const refByNode = new Map<string, RenderedRef>();
for (const ref of input.refs) {
const frameId = ref.frameId ?? input.rootFrameId;
refByNode.set(nodeKey(frameId, ref.backendNodeId), ref);
this.#refById.set(ref.ref, ref);
const frameRefs = this.#refsByFrame.get(frameId) ?? [];
frameRefs.push(ref);
this.#refsByFrame.set(frameId, frameRefs);
}
for (const geometry of input.matchNodes) {
const { frameId } = geometry;
const entry = {
frameId,
geometry,
ref: refByNode.get(nodeKey(frameId, geometry.backendNodeId)),
};
const key = frameTagKey(frameId, geometry.tag);
const bucket = this.#nodesByFrameTag.get(key) ?? [];
bucket.push(entry);
this.#nodesByFrameTag.set(key, bucket);
}
}
candidates(frameId: string, tag: string): readonly IndexedObservationNode[] {
return this.#nodesByFrameTag.get(frameTagKey(frameId, tag)) ?? [];
}
ref(refId: string): RenderedRef | undefined {
return this.#refById.get(refId);
}
refs(frameId: string): readonly RenderedRef[] {
return this.#refsByFrame.get(frameId) ?? [];
}
}
async function readTabMeta(
tabsApi: ChromeTabsApi,
tabId: number,
): Promise<{ url: string; title?: string }> {
try {
const tab = await tabsApi.get(tabId);
return { url: tab.url ?? "about:blank", title: tab.title };
} catch {
return { url: "about:blank" };
}
}
export async function captureRecordingObservation(input: {
cdp: CdpRunner;
tabsApi: ChromeTabsApi;
tabId: number;
maxTokens: number;
redactValues: boolean;
signal?: AbortSignal;
}): Promise<CapturedRecordingObservation> {
const { url, title } = await readTabMeta(input.tabsApi, input.tabId);
const captured = await captureVomObservation(input.cdp, input.tabId, url, {
maxTokens: input.maxTokens,
redactValues: input.redactValues,
conditionalSurfaceProbe: false,
signal: input.signal,
});
return {
rootFrameId: captured.rootFrameId,
index: new ObservationNodeIndex(captured),
url,
title,
vomText: captured.text,
truncated: captured.truncated,
};
}
@@ -0,0 +1,122 @@
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import { captureRecordingObservation, type RegisteredObservation } from "./observation-capture";
import { RecordingStateRegistry } from "./state-registry";
import { matchObservationTarget, unmatchedTarget } from "./target-matcher";
import type { RecordingDraftStep, StepAnnotation, TargetedRecordingDraft } from "./types";
const DEFAULT_MAX_PAGE_TOKENS = 3_000;
const MIN_CAPTURE_INTERVAL_MS = 200;
export interface TabObservationCursor {
lastSettled: RegisteredObservation | null;
lastCaptureAt: number;
}
function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve) => {
if (signal?.aborted) return resolve();
const timer = setTimeout(resolve, ms);
signal?.addEventListener(
"abort",
() => {
clearTimeout(timer);
resolve();
},
{ once: true },
);
});
}
function isTargeted(draft: RecordingDraftStep): draft is TargetedRecordingDraft {
return draft.op !== "navigate" && draft.op !== "scroll";
}
export class RecordingObservationSession {
readonly registry: RecordingStateRegistry;
readonly cursor: TabObservationCursor;
readonly annotations: StepAnnotation[];
readonly #maxTokens: number;
readonly #redactValues: boolean;
constructor(
options: {
registry?: RecordingStateRegistry;
cursor?: TabObservationCursor;
annotations?: StepAnnotation[];
maxTokens?: number;
redactValues?: boolean;
} = {},
) {
this.registry = options.registry ?? new RecordingStateRegistry();
this.cursor = options.cursor ?? { lastSettled: null, lastCaptureAt: 0 };
this.annotations = options.annotations ?? [];
this.#maxTokens = options.maxTokens ?? DEFAULT_MAX_PAGE_TOKENS;
this.#redactValues = options.redactValues ?? false;
}
async capture(
cdp: CdpRunner,
tabsApi: ChromeTabsApi,
tabId: number,
signal?: AbortSignal,
): Promise<RegisteredObservation> {
const waitMs = Math.max(0, MIN_CAPTURE_INTERVAL_MS - (Date.now() - this.cursor.lastCaptureAt));
if (waitMs > 0) await abortableDelay(waitMs, signal);
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
const captured = await captureRecordingObservation({
cdp,
tabsApi,
tabId,
maxTokens: this.#maxTokens,
redactValues: this.#redactValues,
signal,
});
const state = this.registry.register({
url: captured.url,
title: captured.title,
rawVomText: captured.vomText,
truncated: captured.truncated,
});
const observation: RegisteredObservation = {
stateId: state.id,
rootFrameId: captured.rootFrameId,
index: captured.index,
url: captured.url,
};
this.cursor.lastSettled = observation;
this.cursor.lastCaptureAt = Date.now();
return observation;
}
bindDraft(draft: RecordingDraftStep, draftId: number, previousActionPending = false): void {
const observation = this.cursor.lastSettled;
if (isTargeted(draft)) {
draft.matchedTarget = observation
? matchObservationTarget({
observation,
hint: draft.targetHint,
fallback: draft.captureTarget,
})
: unmatchedTarget(draft.captureTarget);
}
if (!observation) return;
const unmatched = isTargeted(draft) && draft.matchedTarget?.unmatched === true;
if (previousActionPending && unmatched) return;
draft.preStateId = observation.stateId;
this.registry.markStep(observation.stateId, draftId);
if (!isTargeted(draft) || !draft.matchedTarget?.ref) return;
const ref = observation.index.ref(draft.matchedTarget.ref);
if (!ref) return;
this.annotations.push({
draftId,
op: draft.op,
line: ref.line,
stateId: observation.stateId,
...(draft.op === "fill" && !this.#redactValues
? { detail: JSON.stringify(draft.value) }
: {}),
});
}
}
@@ -0,0 +1,232 @@
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
import { RecordingObservationSession } from "./observation-session";
import type { RecordingDraftStep } from "./types";
interface PendingSettle {
abort: AbortController;
scope: DocumentSettleScope;
}
interface PendingRedirect {
url: string;
abort: AbortController;
}
const CAPTURE_RETRY_DELAY_MS = 250;
function delay(ms: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve) => {
if (signal?.aborted) return resolve();
const timer = setTimeout(resolve, ms);
signal?.addEventListener(
"abort",
() => {
clearTimeout(timer);
resolve();
},
{ once: true },
);
});
}
function isAbortError(error: unknown): boolean {
return (
typeof error === "object" &&
error !== null &&
(error as { name?: string }).name === "AbortError"
);
}
export function inferMissingPostStates(drafts: RecordingDraftStep[]): void {
for (let index = 0; index < drafts.length - 1; index += 1) {
const draft = drafts[index];
const next = drafts[index + 1];
if (draft && next && !draft.postStateId && next.preStateId) {
draft.postStateId = next.preStateId;
}
}
}
export class SettleController {
readonly #session: RecordingObservationSession;
readonly #cdp: CdpRunner;
readonly #tabsApi: ChromeTabsApi;
readonly #tabId: number;
readonly #rootScope: DocumentSettleScope;
readonly #pending = new Map<number, PendingSettle>();
#queue = Promise.resolve();
#pendingRedirect: PendingRedirect | null = null;
#redirectQueue = Promise.resolve();
constructor(input: {
session: RecordingObservationSession;
cdp: CdpRunner;
tabsApi: ChromeTabsApi;
tabId: number;
}) {
this.#session = input.session;
this.#cdp = input.cdp;
this.#tabsApi = input.tabsApi;
this.#tabId = input.tabId;
this.#rootScope = { target: { tabId: input.tabId } };
}
get hasPending(): boolean {
return this.#pending.size > 0 || this.#pendingRedirect !== null;
}
async #captureWithRetry(signal?: AbortSignal) {
try {
return await this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
} catch (error) {
if (signal?.aborted) throw error;
await delay(CAPTURE_RETRY_DELAY_MS, signal);
return this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
}
}
schedule(
drafts: RecordingDraftStep[],
draftIndex: number,
scope: DocumentSettleScope = this.#rootScope,
): void {
if (scope.target.tabId !== this.#tabId) {
throw new Error(
`settle scope tab ${scope.target.tabId} does not belong to tab ${this.#tabId}`,
);
}
const landing = drafts[draftIndex]?.preStateId;
for (const [index, pending] of this.#pending) {
if (index >= draftIndex) continue;
pending.abort.abort();
this.#pending.delete(index);
if (landing && drafts[index] && !drafts[index].postStateId) {
drafts[index].postStateId = landing;
}
}
this.#pending.get(draftIndex)?.abort.abort();
const pending = { abort: new AbortController(), scope };
this.#pending.set(draftIndex, pending);
const task = async () => {
if (pending.abort.signal.aborted) return;
try {
const outcome = await waitForDocumentSettled(this.#cdp, pending.scope, {
signal: pending.abort.signal,
});
if (outcome === "cancelled") return;
const observation = await this.#captureWithRetry(pending.abort.signal);
if (!pending.abort.signal.aborted && drafts[draftIndex]) {
drafts[draftIndex].postStateId = observation.stateId;
}
} catch (error) {
if (!isAbortError(error)) {
console.warn(
`[bsk record] post-action observation failed for step ${draftIndex + 1}`,
error,
);
}
} finally {
if (this.#pending.get(draftIndex) === pending) this.#pending.delete(draftIndex);
}
};
this.#queue = this.#queue.then(task, task).catch(() => {});
}
cancel(): void {
for (const pending of this.#pending.values()) pending.abort.abort();
this.#pending.clear();
this.clearRedirect();
}
async flush(): Promise<void> {
for (let round = 0; round < 10; round += 1) {
const current = this.#queue;
await current;
if (current === this.#queue) return;
}
console.warn("[bsk record] settle queue kept growing; using completed observations");
}
async settleTrailing(drafts: RecordingDraftStep[]): Promise<void> {
const trailing: RecordingDraftStep[] = [];
for (let index = drafts.length - 1; index >= 0; index -= 1) {
const draft = drafts[index];
if (!draft || draft.postStateId) break;
trailing.push(draft);
}
if (trailing.length === 0) return;
try {
const observation = await this.#captureWithRetry();
for (const draft of trailing) draft.postStateId = observation.stateId;
} catch (error) {
console.warn("[bsk record] final observation at stop failed", error);
}
}
clearRedirect(): void {
this.#pendingRedirect?.abort.abort();
this.#pendingRedirect = null;
}
scheduleRedirect(drafts: RecordingDraftStep[], url: string): void {
this.#pendingRedirect?.abort.abort();
const pending: PendingRedirect = { url, abort: new AbortController() };
this.#pendingRedirect = pending;
const task = () => this.#settleRedirect(drafts, pending);
this.#redirectQueue = this.#redirectQueue.then(task, task).catch(() => {});
}
async #settleRedirect(drafts: RecordingDraftStep[], pending: PendingRedirect): Promise<void> {
const outcome = await waitForDocumentSettled(this.#cdp, this.#rootScope, {
signal: pending.abort.signal,
});
if (outcome === "cancelled" || this.#pendingRedirect !== pending) return;
let finalUrl = pending.url;
try {
finalUrl = (await this.#tabsApi.get(this.#tabId)).url || finalUrl;
} catch {
// The navigation event URL remains the best available destination.
}
if (this.#pendingRedirect !== pending) return;
this.#pendingRedirect = null;
if (
!finalUrl ||
finalUrl === "about:blank" ||
this.#session.cursor.lastSettled?.url === finalUrl
) {
return;
}
const last = drafts[drafts.length - 1];
if (last?.op === "navigate" && last.url === finalUrl) {
if (!last.postStateId) this.schedule(drafts, drafts.length - 1);
await this.flush();
return;
}
const draft: RecordingDraftStep = {
op: "navigate",
url: finalUrl,
pageUrl: finalUrl,
cause: "browser",
preStateId: this.#session.cursor.lastSettled?.stateId,
};
drafts.push(draft);
const draftIndex = drafts.length - 1;
if (draft.preStateId) this.#session.registry.markStep(draft.preStateId, draftIndex + 1);
this.schedule(drafts, draftIndex);
await this.flush();
}
async flushRedirects(): Promise<void> {
for (let round = 0; round < 10; round += 1) {
const current = this.#redirectQueue;
await current;
if (current === this.#redirectQueue) return;
}
console.warn("[bsk record] redirect queue kept growing; using the final completed landing");
}
}
@@ -0,0 +1,56 @@
export interface RecordedStateEntry {
id: string;
url: string;
title?: string;
rawVomText: string;
truncated: boolean;
stepsHere: number[];
}
function stateIdentity(url: string, body: string): string {
return `${url}\0${body}`;
}
export class RecordingStateRegistry {
readonly #entriesById = new Map<string, RecordedStateEntry>();
readonly #idByIdentity = new Map<string, string>();
#nextId = 1;
register(input: {
url: string;
title?: string;
rawVomText: string;
truncated?: boolean;
}): RecordedStateEntry {
const identity = stateIdentity(input.url, input.rawVomText);
const existingId = this.#idByIdentity.get(identity);
if (existingId) {
const existing = this.#entriesById.get(existingId)!;
if (!existing.title && input.title) existing.title = input.title;
if (input.truncated) existing.truncated = true;
return existing;
}
const entry: RecordedStateEntry = {
id: `s${this.#nextId}`,
url: input.url,
...(input.title ? { title: input.title } : {}),
rawVomText: input.rawVomText,
truncated: input.truncated ?? false,
stepsHere: [],
};
this.#nextId += 1;
this.#entriesById.set(entry.id, entry);
this.#idByIdentity.set(identity, entry.id);
return entry;
}
values(): RecordedStateEntry[] {
return [...this.#entriesById.values()];
}
markStep(stateId: string, draftId: number): void {
const entry = this.#entriesById.get(stateId);
if (entry && !entry.stepsHere.includes(draftId)) entry.stepsHere.push(draftId);
}
}
@@ -1,8 +1,8 @@
import type { DraftTraceStep } from "@/transport/types";
import type { RecordStepPayload } from "./record-bridge";
import type { RecordStepPayload } from "../record-bridge";
import type { RecordingDraftStep } from "./types";
export interface RecordingStepBuffer {
steps: DraftTraceStep[];
steps: RecordingDraftStep[];
currentUrl?: string;
pendingNavigation: boolean;
pendingNavigationDeadline?: number;
@@ -10,33 +10,33 @@ export interface RecordingStepBuffer {
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
function toDraftStep(payload: RecordStepPayload): RecordingDraftStep | null {
const pageUrl = payload.page_url;
switch (payload.op) {
case "click":
return payload.target
? {
op: "click",
target: payload.target,
...(pageUrl ? { page_url: pageUrl } : {}),
captureTarget: payload.target,
...(pageUrl ? { pageUrl } : {}),
}
: null;
case "hover":
return payload.target
? {
op: "hover",
target: payload.target,
...(pageUrl ? { page_url: pageUrl } : {}),
captureTarget: payload.target,
...(pageUrl ? { pageUrl } : {}),
}
: null;
case "fill":
return payload.target
? {
op: "fill",
target: payload.target,
captureTarget: payload.target,
value: payload.value ?? "",
...(payload.redacted ? { redacted: true } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
...(pageUrl ? { pageUrl } : {}),
}
: null;
case "press":
@@ -44,19 +44,19 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
? {
op: "press",
key: payload.key,
...(payload.target ? { target: payload.target } : {}),
...(payload.target ? { captureTarget: payload.target } : {}),
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
...(pageUrl ? { pageUrl } : {}),
}
: null;
case "select":
return payload.target && payload.values
? {
op: "select",
target: payload.target,
captureTarget: payload.target,
values: payload.values,
...(payload.labels?.length ? { labels: payload.labels } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
...(pageUrl ? { pageUrl } : {}),
}
: null;
case "navigate":
@@ -69,7 +69,7 @@ function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): b
const step = buffer.steps[i];
if (!step) continue;
if (step.op === "click" || step.op === "press" || step.op === "select") {
buffer.steps[i] = { ...step, navigated_to: url };
buffer.steps[i] = { ...step, navigatedTo: url };
return true;
}
break;
@@ -96,7 +96,7 @@ export function observeRecordedNavigation(
buffer.steps.push({
op: "navigate",
url,
page_url: url,
pageUrl: url,
});
}
return;
@@ -107,7 +107,7 @@ export function observeRecordedNavigation(
buffer.steps.push({
op: "navigate",
url,
page_url: url,
pageUrl: url,
});
}
@@ -0,0 +1,76 @@
import type { RenderedRef } from "@browser-skill/vom";
import type { TargetDescriptorV3 } from "@/transport/types";
import type { CaptureTargetDescriptor } from "../describe-target";
import type { IndexedObservationNode, RegisteredObservation } from "./observation-capture";
import type { TargetGeometry, TargetMatchHint } from "./types";
const MATCH_TOLERANCE_PX = 2;
function close(a: number, b: number): boolean {
return Math.abs(a - b) <= MATCH_TOLERANCE_PX;
}
function rectMatches(a: TargetGeometry["rect"], b: TargetGeometry["rect"]): boolean {
return close(a.x, b.x) && close(a.y, b.y) && close(a.w, b.w) && close(a.h, b.h);
}
function candidateMatches(candidate: IndexedObservationNode, geometry: TargetGeometry): boolean {
return candidate.geometry.rect !== null && rectMatches(geometry.rect, candidate.geometry.rect);
}
export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 {
return {
...(fallback?.role ? { role: fallback.role } : {}),
...(fallback?.name ? { name: fallback.name } : {}),
unmatched: true,
};
}
function normalized(value: string | undefined): string {
return value?.replace(/\s+/g, " ").trim().toLowerCase() ?? "";
}
function matchesSemantics(ref: RenderedRef, fallback?: CaptureTargetDescriptor): boolean {
const role = normalized(fallback?.role);
const name = normalized(fallback?.name);
if (!role && !name) return false;
if (role && normalized(ref.role) !== role) return false;
if (name && normalized(ref.name) !== name) return false;
return true;
}
function descriptor(ref: RenderedRef): TargetDescriptorV3 {
return {
ref: ref.ref,
...(ref.role ? { role: ref.role } : {}),
...(ref.name ? { name: ref.name } : {}),
...(ref.ctx ? { ctx: ref.ctx } : {}),
};
}
export function matchObservationTarget(input: {
observation: RegisteredObservation;
hint?: TargetMatchHint;
fallback?: CaptureTargetDescriptor;
}): TargetDescriptorV3 {
const frameId = input.hint?.frameId ?? input.observation.rootFrameId;
const geometry = input.hint?.geometry;
if (geometry) {
const matches = input.observation.index
.candidates(frameId, geometry.tag)
.filter((candidate) => candidate.ref && candidateMatches(candidate, geometry));
if (matches.length === 1) return descriptor(matches[0]!.ref!);
const semanticMatches = matches.filter(
(candidate) => candidate.ref && matchesSemantics(candidate.ref, input.fallback),
);
if (semanticMatches.length === 1) return descriptor(semanticMatches[0]!.ref!);
return unmatchedTarget(input.fallback);
}
const semanticMatches = input.observation.index
.refs(frameId)
.filter((ref) => matchesSemantics(ref, input.fallback));
return semanticMatches.length === 1
? descriptor(semanticMatches[0]!)
: unmatchedTarget(input.fallback);
}
@@ -0,0 +1,87 @@
import {
type StepV3,
type StopReason,
TRACE_VERSION_V3,
type TraceStateV3,
type TraceV3,
VOM_FORMAT_VERSION,
} from "@/transport/types";
import { resolveDraftStartUrl } from "./draft-policy";
import type { RecordedStateEntry, RecordingStateRegistry } from "./state-registry";
import { reduceTraceStepsV3 } from "./trace-reducer-v3";
import { formatTraceStateBody } from "./trace-state-body";
import type { RecordingDraftStep, StepAnnotation } from "./types";
function publishedEntries(registry: RecordingStateRegistry, steps: StepV3[]): RecordedStateEntry[] {
const entries = registry.values();
if (steps.length === 0) return entries.slice(0, 1);
const referenced = new Set(steps.flatMap((step) => [step.state, step.result.state]));
return entries.filter((entry) => referenced.has(entry.id));
}
function remapDraftIds(draftIds: number[], stepIdByDraftId: Map<number, number>): number[] {
return [
...new Set(
draftIds.flatMap((id) => {
const stepId = stepIdByDraftId.get(id);
return stepId === undefined ? [] : [stepId];
}),
),
].sort((a, b) => a - b);
}
export function buildTraceV3(input: {
registry: RecordingStateRegistry;
drafts: RecordingDraftStep[];
annotations?: StepAnnotation[];
startedAt: string;
purpose?: string;
startUrl?: string;
stoppedBy: StopReason;
bskVersion: string;
}): TraceV3 {
const reduced = reduceTraceStepsV3(input.drafts);
const entries = publishedEntries(input.registry, reduced.steps);
const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`]));
const annotationsByState = new Map<string, StepAnnotation[]>();
for (const annotation of input.annotations ?? []) {
const bucket = annotationsByState.get(annotation.stateId) ?? [];
bucket.push(annotation);
annotationsByState.set(annotation.stateId, bucket);
}
const steps = reduced.steps.map((step) => ({
...step,
state: publishedId.get(step.state) ?? step.state,
result: { state: publishedId.get(step.result.state) ?? step.result.state },
}));
const states: TraceStateV3[] = entries.map((entry) => {
const id = publishedId.get(entry.id) ?? entry.id;
return {
id,
url: entry.url,
...(entry.title ? { title: entry.title } : {}),
body: formatTraceStateBody({
stateId: id,
url: entry.url,
title: entry.title,
stepIds: remapDraftIds(entry.stepsHere, reduced.stepIdByDraftId),
vomText: entry.rawVomText,
annotations: annotationsByState.get(entry.id) ?? [],
stepIdByDraftId: reduced.stepIdByDraftId,
}),
...(entry.truncated ? { truncated: true } : {}),
};
});
return {
version: TRACE_VERSION_V3,
...(input.purpose ? { purpose: input.purpose } : {}),
recorded_at: new Date().toISOString(),
started_at: input.startedAt,
stopped_by: input.stoppedBy,
entry: { start_url: resolveDraftStartUrl(input.drafts, input.startUrl, states[0]?.url) },
recorder: { bsk: input.bskVersion, vom: VOM_FORMAT_VERSION },
states,
steps,
};
}
@@ -1,30 +1,10 @@
import type { DraftTraceStep, PageRef, SelectedOption, Step } from "@/transport/types";
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
export function shouldRecordPress(
key: string,
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
): boolean {
if (MODIFIER_ONLY_KEYS.has(key)) return false;
const mods = modifiers ?? [];
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
if (key === "Enter" || key === "Escape") return true;
// Drop bare character typing — FillSession already records the value.
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
return false;
}
function shouldIncludeDraft(step: DraftTraceStep): boolean {
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
return true;
}
import type { PageRefV2, SelectedOptionV2, StepV2, TraceV2 } from "@/transport/types";
import { resolveDraftStartUrl, shouldIncludeDraft } from "./draft-policy";
import type { RecordingDraftStep } from "./types";
/** Collapse consecutive navigations to the last hop. */
function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
const out: DraftTraceStep[] = [];
function collapseNavigations(steps: RecordingDraftStep[]): RecordingDraftStep[] {
const out: RecordingDraftStep[] = [];
for (const step of steps) {
const prev = out[out.length - 1];
if (step.op === "navigate" && prev?.op === "navigate") {
@@ -36,7 +16,7 @@ function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
return out;
}
function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
function collectUrls(steps: RecordingDraftStep[], startUrl?: string): string[] {
const urls: string[] = [];
if (startUrl) urls.push(startUrl);
for (const step of steps) {
@@ -44,8 +24,8 @@ function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
urls.push(step.url);
continue;
}
if ("page_url" in step && step.page_url) urls.push(step.page_url);
if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to);
if (step.pageUrl) urls.push(step.pageUrl);
if ("navigatedTo" in step && step.navigatedTo) urls.push(step.navigatedTo);
}
const seen = new Set<string>();
const unique: string[] = [];
@@ -58,9 +38,9 @@ function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
}
function buildPageRegistry(
steps: DraftTraceStep[],
steps: RecordingDraftStep[],
startUrl?: string,
): { pages: PageRef[]; urlToId: Map<string, string> } {
): { pages: PageRefV2[]; urlToId: Map<string, string> } {
const urls = collectUrls(steps, startUrl);
const urlToId = new Map<string, string>();
const pages = urls.map((url, index) => {
@@ -81,28 +61,28 @@ function pageIdFor(
return urlToId.values().next().value ?? "p1";
}
function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined {
if (step.op === "navigate") return step.page_url ?? step.url;
if ("page_url" in step && step.page_url) return step.page_url;
function pageUrlForDraft(step: RecordingDraftStep, fallbackUrl?: string): string | undefined {
if (step.op === "navigate") return step.pageUrl ?? step.url;
if (step.pageUrl) return step.pageUrl;
return fallbackUrl;
}
function effectForNavigation(
navigatedTo: string | undefined,
urlToId: Map<string, string>,
): Step["effect"] {
): StepV2["effect"] {
if (!navigatedTo) return undefined;
const pageId = urlToId.get(navigatedTo);
if (!pageId) return undefined;
return { navigated_to: pageId };
}
function withEffect(step: Step, effect: Step["effect"]): Step {
function withEffect(step: StepV2, effect: StepV2["effect"]): StepV2 {
if (!effect) return step;
return { ...step, effect };
}
function toSelection(values: string[], labels?: string[]): SelectedOption[] {
function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] {
return values.map((value, index) => ({
value,
...(labels?.[index] ? { label: labels[index] } : {}),
@@ -110,11 +90,11 @@ function toSelection(values: string[], labels?: string[]): SelectedOption[] {
}
function toV2Step(
step: DraftTraceStep,
step: RecordingDraftStep,
id: number,
urlToId: Map<string, string>,
fallbackUrl?: string,
): Step | null {
): StepV2 | null {
if (!shouldIncludeDraft(step)) return null;
const pageUrl = pageUrlForDraft(step, fallbackUrl);
@@ -129,28 +109,31 @@ function toV2Step(
to: step.url,
};
case "click":
if (!step.captureTarget) return null;
return withEffect(
{
op: "click",
id,
page,
target: step.target,
target: step.captureTarget,
},
effectForNavigation(step.navigated_to, urlToId),
effectForNavigation(step.navigatedTo, urlToId),
);
case "hover":
if (!step.captureTarget) return null;
return {
op: "hover",
id,
page,
target: step.target,
target: step.captureTarget,
};
case "fill":
if (!step.captureTarget) return null;
return {
op: "fill",
id,
page,
target: step.target,
target: step.captureTarget,
value: step.value,
...(step.redacted ? { redacted: true } : {}),
};
@@ -161,44 +144,47 @@ function toV2Step(
id,
page,
key: step.key,
...(step.target ? { target: step.target } : {}),
...(step.captureTarget ? { target: step.captureTarget } : {}),
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
},
effectForNavigation(step.navigated_to, urlToId),
effectForNavigation(step.navigatedTo, urlToId),
);
case "select":
if (!step.captureTarget) return null;
return withEffect(
{
op: "select",
id,
page,
target: step.target,
target: step.captureTarget,
selection: toSelection(step.values, step.labels),
},
effectForNavigation(step.navigated_to, urlToId),
effectForNavigation(step.navigatedTo, urlToId),
);
case "scroll":
return null;
}
}
export interface ReducedTrace {
pages: PageRef[];
steps: Step[];
interface ReducedTrace {
pages: PageRefV2[];
steps: StepV2[];
}
/**
* Compile capture drafts into record-only trace v2 steps.
* Variable inputs are NOT classified here — executing agents infer that at run time.
*/
export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): ReducedTrace {
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string): ReducedTrace {
const collapsed = collapseNavigations(steps);
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
const out: Step[] = [];
const out: StepV2[] = [];
let id = 1;
let lastUrl = startUrl;
for (const draft of collapsed) {
if (draft.op === "navigate") lastUrl = draft.url;
else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to;
else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url;
else if ("navigatedTo" in draft && draft.navigatedTo) lastUrl = draft.navigatedTo;
else if (draft.pageUrl) lastUrl = draft.pageUrl;
const step = toV2Step(draft, id, urlToId, lastUrl);
if (!step) continue;
out.push(step);
@@ -207,18 +193,27 @@ export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): Re
return { pages, steps: out };
}
export function resolveTraceStartUrl(
drafts: DraftTraceStep[],
function resolveTraceStartUrl(
drafts: RecordingDraftStep[],
startUrl?: string,
pages?: PageRef[],
pages?: PageRefV2[],
): string {
if (startUrl) return startUrl;
const navigate = drafts.find((step): step is Extract<DraftTraceStep, { op: "navigate" }> => {
return step.op === "navigate";
});
if (navigate) return navigate.url;
for (const draft of drafts) {
if ("page_url" in draft && draft.page_url) return draft.page_url;
}
return pages?.[0]?.url ?? "about:blank";
return resolveDraftStartUrl(drafts, startUrl, pages?.[0]?.url);
}
export function buildTraceV2(input: {
steps: RecordingDraftStep[];
startedAt: string;
startUrl?: string;
purpose?: string;
}): TraceV2 {
const { pages, steps } = reduceTraceSteps(input.steps, input.startUrl);
return {
recorded_at: new Date().toISOString(),
started_at: input.startedAt,
...(input.purpose ? { purpose: input.purpose } : {}),
entry: { start_url: resolveTraceStartUrl(input.steps, input.startUrl, pages) },
pages,
steps,
};
}
@@ -0,0 +1,129 @@
import type { NavigationCause, StepV3 } from "@/transport/types";
import { shouldIncludeDraft } from "./draft-policy";
import { unmatchedTarget } from "./target-matcher";
import type { RecordingDraftStep } from "./types";
interface CollapsedDraft {
draft: RecordingDraftStep;
draftIds: number[];
}
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
const TRANSITION_CAUSES: Record<string, NavigationCause> = {
typed: "user_typed",
generated: "user_typed",
keyword: "user_typed",
keyword_generated: "user_typed",
link: "link",
form_submit: "form_submit",
reload: "reload",
auto_bookmark: "browser",
start_page: "browser",
};
function isRedirect(step: Extract<RecordingDraftStep, { op: "navigate" }>): boolean {
return (step.transitionQualifiers ?? []).some((qualifier) => REDIRECT_QUALIFIERS.has(qualifier));
}
function collapseRedirects(steps: RecordingDraftStep[]): CollapsedDraft[] {
const output: CollapsedDraft[] = [];
steps.forEach((step, index) => {
const previous = output[output.length - 1];
if (step.op === "navigate" && previous?.draft.op === "navigate" && isRedirect(step)) {
previous.draft = {
...previous.draft,
url: step.url,
postStateId: step.postStateId ?? previous.draft.postStateId,
};
previous.draftIds.push(index + 1);
return;
}
output.push({ draft: { ...step }, draftIds: [index + 1] });
});
return output;
}
function navigationCause(step: Extract<RecordingDraftStep, { op: "navigate" }>): NavigationCause {
if (step.cause) return step.cause;
const qualifiers = step.transitionQualifiers ?? [];
if (qualifiers.includes("forward_back")) return "history";
if (qualifiers.includes("from_address_bar")) return "user_typed";
return TRANSITION_CAUSES[step.transitionType ?? ""] ?? "browser";
}
function selection(values: string[], labels?: string[]): Array<{ value: string; label?: string }> {
return values.map((value, index) => ({
value,
...(labels?.[index] ? { label: labels[index] } : {}),
}));
}
function reduceDraft(draft: RecordingDraftStep, id: number): StepV3 | null {
if (!shouldIncludeDraft(draft)) return null;
const state = draft.preStateId ?? draft.postStateId;
const resultState = draft.postStateId ?? draft.preStateId;
if (!state || !resultState) return null;
const common = { id, state, result: { state: resultState } };
switch (draft.op) {
case "navigate":
return { op: "navigate", ...common, to: draft.url, cause: navigationCause(draft) };
case "click":
return {
op: "click",
...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
};
case "hover":
return {
op: "hover",
...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
};
case "fill":
return {
op: "fill",
...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
value: draft.value,
commit: draft.commit ?? "blur",
...(draft.redacted ? { redacted: true } : {}),
};
case "press":
return {
op: "press",
...common,
key: draft.key,
...(draft.captureTarget || draft.matchedTarget
? { target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget) }
: {}),
...(draft.modifiers?.length ? { modifiers: draft.modifiers } : {}),
};
case "select":
return {
op: "select",
...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
selection: selection(draft.values, draft.labels),
};
case "scroll":
return { op: "scroll", ...common };
}
}
export interface ReducedTraceV3 {
steps: StepV3[];
stepIdByDraftId: Map<number, number>;
}
export function reduceTraceStepsV3(steps: RecordingDraftStep[]): ReducedTraceV3 {
const output: StepV3[] = [];
const stepIdByDraftId = new Map<number, number>();
for (const { draft, draftIds } of collapseRedirects(steps)) {
const step = reduceDraft(draft, output.length + 1);
if (!step) continue;
output.push(step);
for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id);
}
return { steps: output, stepIdByDraftId };
}
@@ -0,0 +1,39 @@
import type { StepAnnotation } from "./types";
function annotationText(annotation: StepAnnotation, stepId: number): string {
return ` ⟵ step ${stepId}: ${annotation.op}${annotation.detail ? `: ${annotation.detail}` : ""}`;
}
export function formatTraceStateBody(input: {
stateId: string;
url: string;
title?: string;
stepIds: number[];
vomText: string;
annotations: StepAnnotation[];
stepIdByDraftId: Map<number, number>;
}): string {
const lines = ["# bsk-observation 1", `state: ${JSON.stringify(input.stateId)}`];
lines.push(`url: ${JSON.stringify(input.url)}`);
if (input.title) lines.push(`title: ${JSON.stringify(input.title)}`);
if (input.stepIds.length > 0) lines.push(`steps_here: [${input.stepIds.join(", ")}]`);
lines.push("---");
const byLine = new Map<number, Array<{ annotation: StepAnnotation; stepId: number }>>();
for (const annotation of input.annotations) {
const stepId = input.stepIdByDraftId.get(annotation.draftId);
if (stepId === undefined) continue;
const bucket = byLine.get(annotation.line) ?? [];
bucket.push({ annotation, stepId });
byLine.set(annotation.line, bucket);
}
input.vomText.split("\n").forEach((bodyLine, lineIndex) => {
let line = bodyLine;
for (const item of byLine.get(lineIndex) ?? []) {
line += annotationText(item.annotation, item.stepId);
}
lines.push(line);
});
return `${lines.join("\n")}\n`;
}
+83
View File
@@ -0,0 +1,83 @@
import type { CaptureTargetDescriptor } from "@/lib/describe-target";
import type {
FillCommit,
KeyModifier,
NavigationCause,
StepV3,
TargetDescriptorV3,
} from "@/transport/types";
export interface TargetGeometry {
/** Top-level viewport-relative CSS pixels, as defined by the geometry module. */
rect: { x: number; y: number; w: number; h: number };
tag: string;
}
export interface TargetMatchHint {
geometry?: TargetGeometry;
/** Missing means the current top frame, never an unrestricted frame search. */
frameId?: string;
}
export interface StepAnnotation {
draftId: number;
op: StepV3["op"];
line: number;
stateId: string;
detail?: string;
}
interface DraftStateLink {
pageUrl?: string;
preStateId?: string;
postStateId?: string;
}
interface DraftTarget {
captureTarget?: CaptureTargetDescriptor;
targetHint?: TargetMatchHint;
matchedTarget?: TargetDescriptorV3;
}
interface DraftNavigationEffect {
navigatedTo?: string;
}
export type RecordingDraftStep =
| ({ op: "click" } & DraftStateLink & DraftTarget & DraftNavigationEffect)
| ({ op: "hover" } & DraftStateLink & DraftTarget)
| ({
op: "fill";
value: string;
commit?: FillCommit;
redacted?: boolean;
} & DraftStateLink &
DraftTarget &
DraftNavigationEffect)
| ({
op: "press";
key: string;
modifiers?: KeyModifier[];
} & DraftStateLink &
DraftTarget &
DraftNavigationEffect)
| ({
op: "select";
values: string[];
labels?: string[];
} & DraftStateLink &
DraftTarget &
DraftNavigationEffect)
| ({ op: "scroll" } & DraftStateLink)
| ({
op: "navigate";
url: string;
cause?: NavigationCause;
transitionType?: string;
transitionQualifiers?: string[];
} & DraftStateLink);
export type TargetedRecordingDraft = Extract<
RecordingDraftStep,
{ op: "click" | "hover" | "fill" | "press" | "select" }
>;
+18 -22
View File
@@ -17,11 +17,11 @@ import {
type RecordStartMessage,
type RecordStopMessage,
} from "@/lib/record-bridge";
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer";
import { reduceTraceSteps, resolveTraceStartUrl } from "@/lib/trace-reducer";
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording/step-buffer";
import { buildTraceV2 } from "@/lib/recording/trace-reducer-v2";
import type { RecordingDraftStep } from "@/lib/recording/types";
import type { SessionManager } from "@/session-manager/manager";
import type {
DraftTraceStep,
RecordAwaitParams,
RecordAwaitResult,
RecordStartParams,
@@ -29,7 +29,7 @@ import type {
RecordStopParams,
RecordStopResult,
RpcError,
Trace,
TraceV2,
} from "@/transport/types";
import { handleNavigate } from "./navigation";
import {
@@ -47,11 +47,11 @@ interface ActiveRecording {
agentWindowId: number;
startUrl?: string;
purpose?: string;
steps: DraftTraceStep[];
steps: RecordingDraftStep[];
startedAt: string;
startedAtMs: number;
finishPromise: Promise<Trace>;
resolveFinish: (trace: Trace) => void;
finishPromise: Promise<TraceV2>;
resolveFinish: (trace: TraceV2) => void;
rejectFinish: (err: Error) => void;
settled: boolean;
finishing: boolean;
@@ -154,17 +154,13 @@ async function sendRecordStartWithAck(
throw lastError ?? new Error("failed to start recording in content script");
}
function buildTrace(recording: ActiveRecording): Trace {
const { pages, steps } = reduceTraceSteps(recording.steps, recording.startUrl);
const startUrl = resolveTraceStartUrl(recording.steps, recording.startUrl, pages);
return {
function buildTrace(recording: ActiveRecording): TraceV2 {
return buildTraceV2({
steps: recording.steps,
startedAt: recording.startedAt,
...(recording.startUrl ? { startUrl: recording.startUrl } : {}),
...(recording.purpose ? { purpose: recording.purpose } : {}),
recorded_at: new Date().toISOString(),
started_at: recording.startedAt,
entry: { start_url: startUrl },
pages,
steps,
};
});
}
export interface RecordDeps {
@@ -532,7 +528,7 @@ async function finishRecordingByRequest(
}
}
async function finishRecording(sessionId: string, deps: RecordDeps): Promise<Trace | null> {
async function finishRecording(sessionId: string, deps: RecordDeps): Promise<TraceV2 | null> {
const recording = recordings.get(sessionId);
if (!recording || recording.settled || recording.finishing) return null;
recording.finishing = true;
@@ -574,9 +570,9 @@ export async function handleRecordStart(
// on the destination page can RECORD_QUERY → rearm → show RecordOverlay
// instead of flashing ControlOverlay ("Agent 正在控制").
const requestId = makeRequestId(target.tabId);
let resolveFinish!: (trace: Trace) => void;
let resolveFinish!: (trace: TraceV2) => void;
let rejectFinish!: (err: Error) => void;
const finishPromise = new Promise<Trace>((resolve, reject) => {
const finishPromise = new Promise<TraceV2>((resolve, reject) => {
resolveFinish = resolve;
rejectFinish = reject;
});
@@ -785,9 +781,9 @@ export async function handleRecordAwait(
return { code: "cancelled", message: "record_await aborted" };
}
const outcome = await new Promise<{ trace: Trace } | { error: RpcError }>((resolve) => {
const outcome = await new Promise<{ trace: TraceV2 } | { error: RpcError }>((resolve) => {
let settled = false;
const finish = (result: { trace: Trace } | { error: RpcError }) => {
const finish = (result: { trace: TraceV2 } | { error: RpcError }) => {
if (settled) return;
settled = true;
if (timer) clearTimeout(timer);
+114 -65
View File
@@ -657,10 +657,60 @@ export interface EmulateResult {
}
// --------------------------------------------------------------------------
// Semantic record payloads — mirror bsk-protocol record.rs (Trace v2)
// Semantic record payloads mirror the versioned Rust protocol models.
// --------------------------------------------------------------------------
export interface TargetDescriptor {
export const TRACE_VERSION_V3 = 3;
/** Logical v2 identifier. Not a wire field — v2 envelopes omit `version`. */
export const TRACE_VERSION_V2 = 2;
export const VOM_FORMAT_VERSION = 1;
export interface TargetDescriptorV3 {
ref?: string;
role?: string;
name?: string;
ctx?: string;
unmatched?: boolean;
}
export interface RecorderInfo {
bsk: string;
vom: number;
}
export type StopReason = "user_finish" | "cli_stop";
export interface TraceStateV3 {
id: string;
url: string;
title?: string;
body: string;
truncated?: boolean;
}
export interface StepResultV3 {
state: string;
}
export interface StepCommonV3 {
id: number;
state: string;
result: StepResultV3;
}
export type NavigationCause =
| "user_typed"
| "link"
| "form_submit"
| "reload"
| "history"
| "script"
| "browser";
export type FillCommit = "enter" | "suggestion" | "blur";
/** Legacy v2 target shape retained for existing record producers. */
export interface TargetDescriptorV2 {
role?: string;
name?: string;
tag: string;
@@ -673,98 +723,97 @@ export interface TraceEntry {
start_url: string;
}
export interface PageRef {
export interface PageRefV2 {
id: string;
url: string;
title?: string;
}
export interface SelectedOption {
export interface SelectedOptionV2 {
value: string;
label?: string;
}
export interface StepEffect {
export interface StepEffectV2 {
navigated_to: string;
}
export interface StepCommon {
export interface StepCommonV2 {
id: number;
page: string;
effect?: StepEffect;
effect?: StepEffectV2;
}
/** Capture/buffer draft before v2 reduction. */
export type DraftTraceStep =
| {
op: "click";
target: TargetDescriptor;
navigated_to?: string;
page_url?: string;
}
| {
op: "hover";
target: TargetDescriptor;
page_url?: string;
}
| {
op: "fill";
target: TargetDescriptor;
value: string;
redacted?: boolean;
page_url?: string;
}
| {
op: "press";
key: string;
target?: TargetDescriptor;
modifiers?: KeyModifier[];
navigated_to?: string;
page_url?: string;
}
| {
op: "select";
target: TargetDescriptor;
values: string[];
labels?: string[];
navigated_to?: string;
page_url?: string;
}
| {
op: "navigate";
url: string;
page_url?: string;
};
/** Exported record-only step (trace v2). */
export type Step =
| ({ op: "navigate" } & StepCommon & { to: string })
| ({ op: "click" } & StepCommon & { target: TargetDescriptor })
| ({ op: "hover" } & StepCommon & { target: TargetDescriptor })
| ({ op: "fill" } & StepCommon & {
target: TargetDescriptor;
export type StepV2 =
| ({ op: "navigate" } & StepCommonV2 & { to: string })
| ({ op: "click" } & StepCommonV2 & { target: TargetDescriptorV2 })
| ({ op: "hover" } & StepCommonV2 & { target: TargetDescriptorV2 })
| ({ op: "fill" } & StepCommonV2 & {
target: TargetDescriptorV2;
value: string;
redacted?: boolean;
})
| ({ op: "select" } & StepCommon & {
target: TargetDescriptor;
selection: SelectedOption[];
| ({ op: "select" } & StepCommonV2 & {
target: TargetDescriptorV2;
selection: SelectedOptionV2[];
})
| ({ op: "press" } & StepCommon & {
| ({ op: "press" } & StepCommonV2 & {
key: string;
modifiers?: KeyModifier[];
target?: TargetDescriptor;
target?: TargetDescriptorV2;
});
export interface Trace {
export interface TraceV2 {
recorded_at: string;
started_at?: string;
purpose?: string;
entry: TraceEntry;
pages: PageRef[];
steps: Step[];
pages: PageRefV2[];
steps: StepV2[];
}
export interface SelectedOptionV3 {
value: string;
label?: string;
}
export type StepV3 =
| ({ op: "navigate" } & StepCommonV3 & { to: string; cause: NavigationCause })
| ({ op: "click" } & StepCommonV3 & { target: TargetDescriptorV3 })
| ({ op: "hover" } & StepCommonV3 & { target: TargetDescriptorV3 })
| ({ op: "fill" } & StepCommonV3 & {
target: TargetDescriptorV3;
value: string;
commit: FillCommit;
redacted?: boolean;
})
| ({ op: "select" } & StepCommonV3 & {
target: TargetDescriptorV3;
selection?: SelectedOptionV3[];
})
| ({ op: "press" } & StepCommonV3 & {
key: string;
modifiers?: KeyModifier[];
target?: TargetDescriptorV3;
})
| ({ op: "scroll" } & StepCommonV3);
export interface TraceV3 {
version: typeof TRACE_VERSION_V3;
recorded_at: string;
started_at?: string;
purpose?: string;
stopped_by: StopReason;
entry: TraceEntry;
recorder: RecorderInfo;
states: TraceStateV3[];
steps: StepV3[];
}
export type RecordedTrace = TraceV2 | TraceV3;
export type RecordedStep = StepV2 | StepV3;
export interface RecordStartParams {
session_id: string;
tab_id?: number;
@@ -782,7 +831,7 @@ export interface RecordStopParams {
}
export interface RecordStopResult {
trace: Trace;
trace: RecordedTrace;
}
export interface RecordAwaitParams {
@@ -791,5 +840,5 @@ export interface RecordAwaitParams {
}
export interface RecordAwaitResult {
trace: Trace;
trace: RecordedTrace;
}
+8 -4
View File
@@ -8,7 +8,7 @@ use anyhow::Context;
use bsk_protocol::Method;
use bsk_protocol::tools::{
RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult, RecordStopParams,
RecordStopResult, Trace,
RecordStopResult, RecordedTrace,
};
use clap::{Args, Subcommand};
@@ -190,7 +190,7 @@ fn record_await_ipc_timeout(timeout_ms: u32) -> Duration {
.unwrap_or(Duration::from_secs(u64::from(timeout_ms / 1_000) + 15))
}
fn write_trace_file(output: &PathBuf, trace: &Trace) -> Result<(), CliError> {
fn write_trace_file(output: &PathBuf, trace: &RecordedTrace) -> Result<(), CliError> {
let json = serde_json::to_string_pretty(trace)
.context("serialize trace JSON")
.map_err(CliError::Local)?;
@@ -207,7 +207,7 @@ fn write_trace_file(output: &PathBuf, trace: &Trace) -> Result<(), CliError> {
Ok(())
}
fn render_finish(trace: &Trace, output: &PathBuf, format: Format) -> Result<(), CliError> {
fn render_finish(trace: &RecordedTrace, output: &PathBuf, format: Format) -> Result<(), CliError> {
match format {
Format::Json => {
println!(
@@ -221,7 +221,11 @@ fn render_finish(trace: &Trace, output: &PathBuf, format: Format) -> Result<(),
);
}
Format::Human => {
println!("saved {} steps to {}", trace.steps.len(), output.display());
let step_count = match trace {
RecordedTrace::V2(trace) => trace.steps.len(),
RecordedTrace::V3(trace) => trace.steps.len(),
};
println!("saved {step_count} steps to {}", output.display());
}
}
Ok(())
@@ -7,10 +7,18 @@
],
"properties": {
"trace": {
"$ref": "#/definitions/Trace"
"$ref": "#/definitions/RecordedTrace"
}
},
"definitions": {
"FillCommit": {
"type": "string",
"enum": [
"enter",
"suggestion",
"blur"
]
},
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
@@ -21,7 +29,19 @@
"shift"
]
},
"PageRef": {
"NavigationCause": {
"type": "string",
"enum": [
"user_typed",
"link",
"form_submit",
"reload",
"history",
"script",
"browser"
]
},
"PageRefV2": {
"description": "Page context dictionary entry — referenced by steps via `page` id.",
"type": "object",
"required": [
@@ -43,7 +63,42 @@
}
}
},
"SelectedOption": {
"RecordedTrace": {
"if": {
"required": [
"version"
],
"properties": {
"version": {
"type": "integer"
}
}
},
"then": {
"$ref": "#/definitions/TraceV3"
},
"else": {
"$ref": "#/definitions/TraceV2"
}
},
"RecorderInfo": {
"type": "object",
"required": [
"bsk",
"vom"
],
"properties": {
"bsk": {
"type": "string"
},
"vom": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
}
}
},
"SelectedOptionV2": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
@@ -61,11 +116,53 @@
}
}
},
"Step": {
"description": "One recorded user action — discriminated union by `op`.",
"SelectedOptionV3": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
"value"
],
"properties": {
"label": {
"type": [
"string",
"null"
]
},
"value": {
"type": "string"
}
}
},
"StepEffectV2": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
}
}
},
"StepResultV3": {
"type": "object",
"required": [
"state"
],
"properties": {
"state": {
"type": "string"
}
}
},
"StepV2": {
"description": "One recorded user action — discriminated union by `op` (v2).",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -77,7 +174,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -105,7 +202,7 @@
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -117,7 +214,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -140,12 +237,12 @@
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -157,7 +254,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -180,12 +277,12 @@
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -198,7 +295,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -227,7 +324,7 @@
]
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
},
"value": {
"type": "string"
@@ -235,7 +332,7 @@
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -248,7 +345,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -273,16 +370,16 @@
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOption"
"$ref": "#/definitions/SelectedOptionV2"
}
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -294,7 +391,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -331,7 +428,7 @@
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
},
{
"type": "null"
@@ -342,21 +439,294 @@
}
]
},
"StepEffect": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
"StepV3": {
"description": "One recorded user action — discriminated union by `op`.",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"cause",
"id",
"op",
"result",
"state",
"to"
],
"properties": {
"cause": {
"$ref": "#/definitions/NavigationCause"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"commit",
"id",
"op",
"result",
"state",
"target",
"value"
],
"properties": {
"commit": {
"$ref": "#/definitions/FillCommit"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"redacted": {
"type": "boolean"
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV3"
}
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV3"
},
{
"type": "null"
}
]
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"scroll"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
}
}
]
},
"TargetDescriptor": {
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
"StopReason": {
"type": "string",
"enum": [
"user_finish",
"cli_stop"
]
},
"TargetDescriptorV2": {
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
"type": "object",
"required": [
"tag"
@@ -397,8 +767,83 @@
}
}
},
"Trace": {
"description": "Persisted user-action trace exported by `tool.record_stop` / `await`.",
"TargetDescriptorV3": {
"description": "Stable semantic handle for an interacted element within a page observation.",
"type": "object",
"properties": {
"ctx": {
"type": [
"string",
"null"
]
},
"name": {
"type": [
"string",
"null"
]
},
"ref": {
"type": [
"string",
"null"
]
},
"role": {
"type": [
"string",
"null"
]
},
"unmatched": {
"type": "boolean"
}
}
},
"TraceEntry": {
"type": "object",
"required": [
"start_url"
],
"properties": {
"start_url": {
"type": "string"
}
}
},
"TraceStateV3": {
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
"type": "object",
"required": [
"body",
"id",
"url"
],
"properties": {
"body": {
"description": "Full page observation (front matter + VOM body + annotations).",
"type": "string"
},
"id": {
"type": "string"
},
"title": {
"type": [
"string",
"null"
]
},
"truncated": {
"type": "boolean"
},
"url": {
"type": "string"
}
}
},
"TraceV2": {
"title": "TraceV2",
"description": "Persisted user-action trace exported by legacy `tool.record_stop` / `await`.\n\nUnknown extension fields are ignored so older traces remain readable. `states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace` classification.",
"type": "object",
"required": [
"entry",
@@ -413,11 +858,10 @@
"pages": {
"type": "array",
"items": {
"$ref": "#/definitions/PageRef"
"$ref": "#/definitions/PageRefV2"
}
},
"purpose": {
"description": "Optional user-provided goal from `--purpose` (metadata only).",
"type": [
"string",
"null"
@@ -433,25 +877,76 @@
"null"
]
},
"states": false,
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/Step"
"$ref": "#/definitions/StepV2"
}
},
"version": {
"description": "Numeric version selects Trace v3. Legacy v2 envelopes omit this field.",
"not": {
"type": "integer"
}
}
}
},
"TraceEntry": {
"description": "Recording entry point — first URL the flow starts from.",
"TraceV3": {
"description": "Wire trace returned by `tool.record_stop` / `await`.",
"type": "object",
"required": [
"start_url"
"entry",
"recorded_at",
"recorder",
"states",
"steps",
"stopped_by",
"version"
],
"properties": {
"start_url": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"purpose": {
"type": [
"string",
"null"
]
},
"recorded_at": {
"type": "string"
},
"recorder": {
"$ref": "#/definitions/RecorderInfo"
},
"started_at": {
"type": [
"string",
"null"
]
},
"states": {
"type": "array",
"items": {
"$ref": "#/definitions/TraceStateV3"
}
},
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/StepV3"
}
},
"stopped_by": {
"$ref": "#/definitions/StopReason"
},
"version": {
"type": "integer",
"const": 3
}
}
},
"additionalProperties": false
}
}
}
@@ -7,10 +7,18 @@
],
"properties": {
"trace": {
"$ref": "#/definitions/Trace"
"$ref": "#/definitions/RecordedTrace"
}
},
"definitions": {
"FillCommit": {
"type": "string",
"enum": [
"enter",
"suggestion",
"blur"
]
},
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
@@ -21,7 +29,19 @@
"shift"
]
},
"PageRef": {
"NavigationCause": {
"type": "string",
"enum": [
"user_typed",
"link",
"form_submit",
"reload",
"history",
"script",
"browser"
]
},
"PageRefV2": {
"description": "Page context dictionary entry — referenced by steps via `page` id.",
"type": "object",
"required": [
@@ -43,7 +63,42 @@
}
}
},
"SelectedOption": {
"RecordedTrace": {
"if": {
"required": [
"version"
],
"properties": {
"version": {
"type": "integer"
}
}
},
"then": {
"$ref": "#/definitions/TraceV3"
},
"else": {
"$ref": "#/definitions/TraceV2"
}
},
"RecorderInfo": {
"type": "object",
"required": [
"bsk",
"vom"
],
"properties": {
"bsk": {
"type": "string"
},
"vom": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
}
}
},
"SelectedOptionV2": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
@@ -61,11 +116,53 @@
}
}
},
"Step": {
"description": "One recorded user action — discriminated union by `op`.",
"SelectedOptionV3": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
"value"
],
"properties": {
"label": {
"type": [
"string",
"null"
]
},
"value": {
"type": "string"
}
}
},
"StepEffectV2": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
}
}
},
"StepResultV3": {
"type": "object",
"required": [
"state"
],
"properties": {
"state": {
"type": "string"
}
}
},
"StepV2": {
"description": "One recorded user action — discriminated union by `op` (v2).",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -77,7 +174,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -105,7 +202,7 @@
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -117,7 +214,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -140,12 +237,12 @@
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -157,7 +254,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -180,12 +277,12 @@
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -198,7 +295,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -227,7 +324,7 @@
]
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
},
"value": {
"type": "string"
@@ -235,7 +332,7 @@
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -248,7 +345,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -273,16 +370,16 @@
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOption"
"$ref": "#/definitions/SelectedOptionV2"
}
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -294,7 +391,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -331,7 +428,7 @@
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
},
{
"type": "null"
@@ -342,21 +439,294 @@
}
]
},
"StepEffect": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
"StepV3": {
"description": "One recorded user action — discriminated union by `op`.",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"cause",
"id",
"op",
"result",
"state",
"to"
],
"properties": {
"cause": {
"$ref": "#/definitions/NavigationCause"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"commit",
"id",
"op",
"result",
"state",
"target",
"value"
],
"properties": {
"commit": {
"$ref": "#/definitions/FillCommit"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"redacted": {
"type": "boolean"
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV3"
}
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV3"
},
{
"type": "null"
}
]
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"scroll"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
}
}
]
},
"TargetDescriptor": {
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
"StopReason": {
"type": "string",
"enum": [
"user_finish",
"cli_stop"
]
},
"TargetDescriptorV2": {
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
"type": "object",
"required": [
"tag"
@@ -397,8 +767,83 @@
}
}
},
"Trace": {
"description": "Persisted user-action trace exported by `tool.record_stop` / `await`.",
"TargetDescriptorV3": {
"description": "Stable semantic handle for an interacted element within a page observation.",
"type": "object",
"properties": {
"ctx": {
"type": [
"string",
"null"
]
},
"name": {
"type": [
"string",
"null"
]
},
"ref": {
"type": [
"string",
"null"
]
},
"role": {
"type": [
"string",
"null"
]
},
"unmatched": {
"type": "boolean"
}
}
},
"TraceEntry": {
"type": "object",
"required": [
"start_url"
],
"properties": {
"start_url": {
"type": "string"
}
}
},
"TraceStateV3": {
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
"type": "object",
"required": [
"body",
"id",
"url"
],
"properties": {
"body": {
"description": "Full page observation (front matter + VOM body + annotations).",
"type": "string"
},
"id": {
"type": "string"
},
"title": {
"type": [
"string",
"null"
]
},
"truncated": {
"type": "boolean"
},
"url": {
"type": "string"
}
}
},
"TraceV2": {
"title": "TraceV2",
"description": "Persisted user-action trace exported by legacy `tool.record_stop` / `await`.\n\nUnknown extension fields are ignored so older traces remain readable. `states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace` classification.",
"type": "object",
"required": [
"entry",
@@ -413,11 +858,10 @@
"pages": {
"type": "array",
"items": {
"$ref": "#/definitions/PageRef"
"$ref": "#/definitions/PageRefV2"
}
},
"purpose": {
"description": "Optional user-provided goal from `--purpose` (metadata only).",
"type": [
"string",
"null"
@@ -433,25 +877,76 @@
"null"
]
},
"states": false,
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/Step"
"$ref": "#/definitions/StepV2"
}
},
"version": {
"description": "Numeric version selects Trace v3. Legacy v2 envelopes omit this field.",
"not": {
"type": "integer"
}
}
}
},
"TraceEntry": {
"description": "Recording entry point — first URL the flow starts from.",
"TraceV3": {
"description": "Wire trace returned by `tool.record_stop` / `await`.",
"type": "object",
"required": [
"start_url"
"entry",
"recorded_at",
"recorder",
"states",
"steps",
"stopped_by",
"version"
],
"properties": {
"start_url": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"purpose": {
"type": [
"string",
"null"
]
},
"recorded_at": {
"type": "string"
},
"recorder": {
"$ref": "#/definitions/RecorderInfo"
},
"started_at": {
"type": [
"string",
"null"
]
},
"states": {
"type": "array",
"items": {
"$ref": "#/definitions/TraceStateV3"
}
},
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/StepV3"
}
},
"stopped_by": {
"$ref": "#/definitions/StopReason"
},
"version": {
"type": "integer",
"const": 3
}
}
},
"additionalProperties": false
}
}
}
+571 -76
View File
@@ -1,49 +1,31 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Trace",
"description": "Persisted user-action trace exported by `tool.record_stop` / `await`.",
"type": "object",
"required": [
"entry",
"pages",
"recorded_at",
"steps"
],
"properties": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"pages": {
"type": "array",
"items": {
"$ref": "#/definitions/PageRef"
}
},
"purpose": {
"description": "Optional user-provided goal from `--purpose` (metadata only).",
"type": [
"string",
"null"
]
},
"recorded_at": {
"description": "RFC 3339 timestamp when recording stopped.",
"type": "string"
},
"started_at": {
"type": [
"string",
"null"
]
},
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/Step"
"title": "RecordedTrace",
"if": {
"required": [
"version"
],
"properties": {
"version": {
"type": "integer"
}
}
},
"then": {
"$ref": "#/definitions/TraceV3"
},
"else": {
"$ref": "#/definitions/TraceV2"
},
"definitions": {
"FillCommit": {
"type": "string",
"enum": [
"enter",
"suggestion",
"blur"
]
},
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
@@ -54,7 +36,19 @@
"shift"
]
},
"PageRef": {
"NavigationCause": {
"type": "string",
"enum": [
"user_typed",
"link",
"form_submit",
"reload",
"history",
"script",
"browser"
]
},
"PageRefV2": {
"description": "Page context dictionary entry — referenced by steps via `page` id.",
"type": "object",
"required": [
@@ -76,7 +70,24 @@
}
}
},
"SelectedOption": {
"RecorderInfo": {
"type": "object",
"required": [
"bsk",
"vom"
],
"properties": {
"bsk": {
"type": "string"
},
"vom": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
}
}
},
"SelectedOptionV2": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
@@ -94,11 +105,53 @@
}
}
},
"Step": {
"description": "One recorded user action — discriminated union by `op`.",
"SelectedOptionV3": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
"value"
],
"properties": {
"label": {
"type": [
"string",
"null"
]
},
"value": {
"type": "string"
}
}
},
"StepEffectV2": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
}
}
},
"StepResultV3": {
"type": "object",
"required": [
"state"
],
"properties": {
"state": {
"type": "string"
}
}
},
"StepV2": {
"description": "One recorded user action — discriminated union by `op` (v2).",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -110,7 +163,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -138,7 +191,7 @@
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -150,7 +203,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -173,12 +226,12 @@
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -190,7 +243,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -213,12 +266,12 @@
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -231,7 +284,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -260,7 +313,7 @@
]
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
},
"value": {
"type": "string"
@@ -268,7 +321,7 @@
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -281,7 +334,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -306,16 +359,16 @@
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOption"
"$ref": "#/definitions/SelectedOptionV2"
}
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -327,7 +380,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -364,7 +417,7 @@
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
},
{
"type": "null"
@@ -375,21 +428,294 @@
}
]
},
"StepEffect": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
"StepV3": {
"description": "One recorded user action — discriminated union by `op`.",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"cause",
"id",
"op",
"result",
"state",
"to"
],
"properties": {
"cause": {
"$ref": "#/definitions/NavigationCause"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"commit",
"id",
"op",
"result",
"state",
"target",
"value"
],
"properties": {
"commit": {
"$ref": "#/definitions/FillCommit"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"redacted": {
"type": "boolean"
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV3"
}
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV3"
},
{
"type": "null"
}
]
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"scroll"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
}
}
]
},
"TargetDescriptor": {
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
"StopReason": {
"type": "string",
"enum": [
"user_finish",
"cli_stop"
]
},
"TargetDescriptorV2": {
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
"type": "object",
"required": [
"tag"
@@ -430,8 +756,40 @@
}
}
},
"TargetDescriptorV3": {
"description": "Stable semantic handle for an interacted element within a page observation.",
"type": "object",
"properties": {
"ctx": {
"type": [
"string",
"null"
]
},
"name": {
"type": [
"string",
"null"
]
},
"ref": {
"type": [
"string",
"null"
]
},
"role": {
"type": [
"string",
"null"
]
},
"unmatched": {
"type": "boolean"
}
}
},
"TraceEntry": {
"description": "Recording entry point — first URL the flow starts from.",
"type": "object",
"required": [
"start_url"
@@ -441,6 +799,143 @@
"type": "string"
}
}
},
"TraceStateV3": {
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
"type": "object",
"required": [
"body",
"id",
"url"
],
"properties": {
"body": {
"description": "Full page observation (front matter + VOM body + annotations).",
"type": "string"
},
"id": {
"type": "string"
},
"title": {
"type": [
"string",
"null"
]
},
"truncated": {
"type": "boolean"
},
"url": {
"type": "string"
}
}
},
"TraceV2": {
"title": "TraceV2",
"description": "Persisted user-action trace exported by legacy `tool.record_stop` / `await`.\n\nUnknown extension fields are ignored so older traces remain readable. `states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace` classification.",
"type": "object",
"required": [
"entry",
"pages",
"recorded_at",
"steps"
],
"properties": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"pages": {
"type": "array",
"items": {
"$ref": "#/definitions/PageRefV2"
}
},
"purpose": {
"type": [
"string",
"null"
]
},
"recorded_at": {
"description": "RFC 3339 timestamp when recording stopped.",
"type": "string"
},
"started_at": {
"type": [
"string",
"null"
]
},
"states": false,
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/StepV2"
}
},
"version": {
"description": "Numeric version selects Trace v3. Legacy v2 envelopes omit this field.",
"not": {
"type": "integer"
}
}
}
},
"TraceV3": {
"description": "Wire trace returned by `tool.record_stop` / `await`.",
"type": "object",
"required": [
"entry",
"recorded_at",
"recorder",
"states",
"steps",
"stopped_by",
"version"
],
"properties": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"purpose": {
"type": [
"string",
"null"
]
},
"recorded_at": {
"type": "string"
},
"recorder": {
"$ref": "#/definitions/RecorderInfo"
},
"started_at": {
"type": [
"string",
"null"
]
},
"states": {
"type": "array",
"items": {
"$ref": "#/definitions/TraceStateV3"
}
},
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/StepV3"
}
},
"stopped_by": {
"$ref": "#/definitions/StopReason"
},
"version": {
"type": "integer",
"const": 3
}
},
"additionalProperties": false
}
}
}
+664 -284
View File
@@ -1,286 +1,32 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Step",
"description": "One recorded user action — discriminated union by `op`.",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"to"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target",
"value"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"redacted": {
"type": [
"boolean",
"null"
]
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"selection",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOption"
}
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"page"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptor"
},
{
"type": "null"
}
]
}
}
}
],
"title": "RecordedStep",
"not": {
"required": [
"page",
"state"
]
},
"if": {
"required": [
"state"
]
},
"then": {
"$ref": "#/definitions/StepV3"
},
"else": {
"$ref": "#/definitions/StepV2"
},
"definitions": {
"FillCommit": {
"type": "string",
"enum": [
"enter",
"suggestion",
"blur"
]
},
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
@@ -291,7 +37,19 @@
"shift"
]
},
"SelectedOption": {
"NavigationCause": {
"type": "string",
"enum": [
"user_typed",
"link",
"form_submit",
"reload",
"history",
"script",
"browser"
]
},
"SelectedOptionV2": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
@@ -309,7 +67,25 @@
}
}
},
"StepEffect": {
"SelectedOptionV3": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
"value"
],
"properties": {
"label": {
"type": [
"string",
"null"
]
},
"value": {
"type": "string"
}
}
},
"StepEffectV2": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
@@ -322,8 +98,579 @@
}
}
},
"TargetDescriptor": {
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
"StepResultV3": {
"type": "object",
"required": [
"state"
],
"properties": {
"state": {
"type": "string"
}
}
},
"StepV2": {
"description": "One recorded user action — discriminated union by `op` (v2).",
"oneOf": [
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"to"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target",
"value"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"redacted": {
"type": [
"boolean",
"null"
]
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"selection",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV2"
}
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"page"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV2"
},
{
"type": "null"
}
]
}
}
}
]
},
"StepV3": {
"description": "One recorded user action — discriminated union by `op`.",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"cause",
"id",
"op",
"result",
"state",
"to"
],
"properties": {
"cause": {
"$ref": "#/definitions/NavigationCause"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"commit",
"id",
"op",
"result",
"state",
"target",
"value"
],
"properties": {
"commit": {
"$ref": "#/definitions/FillCommit"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"redacted": {
"type": "boolean"
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV3"
}
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV3"
},
{
"type": "null"
}
]
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"scroll"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
}
]
},
"TargetDescriptorV2": {
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
"type": "object",
"required": [
"tag"
@@ -363,6 +710,39 @@
"type": "string"
}
}
},
"TargetDescriptorV3": {
"description": "Stable semantic handle for an interacted element within a page observation.",
"type": "object",
"properties": {
"ctx": {
"type": [
"string",
"null"
]
},
"name": {
"type": [
"string",
"null"
]
},
"ref": {
"type": [
"string",
"null"
]
},
"role": {
"type": [
"string",
"null"
]
},
"unmatched": {
"type": "boolean"
}
}
}
}
}
@@ -0,0 +1,368 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "StepV2",
"description": "One recorded user action — discriminated union by `op` (v2).",
"oneOf": [
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"to"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target",
"value"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"redacted": {
"type": [
"boolean",
"null"
]
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"selection",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV2"
}
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"page"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV2"
},
{
"type": "null"
}
]
}
}
}
],
"definitions": {
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
"enum": [
"alt",
"ctrl",
"meta",
"shift"
]
},
"SelectedOptionV2": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
"value"
],
"properties": {
"label": {
"type": [
"string",
"null"
]
},
"value": {
"type": "string"
}
}
},
"StepEffectV2": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
}
}
},
"TargetDescriptorV2": {
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
"type": "object",
"required": [
"tag"
],
"properties": {
"name": {
"type": [
"string",
"null"
]
},
"name_attr": {
"type": [
"string",
"null"
]
},
"nearby_label": {
"type": [
"string",
"null"
]
},
"placeholder": {
"type": [
"string",
"null"
]
},
"role": {
"type": [
"string",
"null"
]
},
"tag": {
"type": "string"
}
}
}
}
}
@@ -0,0 +1,375 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "StepV3",
"description": "One recorded user action — discriminated union by `op`.",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"cause",
"id",
"op",
"result",
"state",
"to"
],
"properties": {
"cause": {
"$ref": "#/definitions/NavigationCause"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"commit",
"id",
"op",
"result",
"state",
"target",
"value"
],
"properties": {
"commit": {
"$ref": "#/definitions/FillCommit"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"redacted": {
"type": "boolean"
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV3"
}
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV3"
},
{
"type": "null"
}
]
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"scroll"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
}
],
"definitions": {
"FillCommit": {
"type": "string",
"enum": [
"enter",
"suggestion",
"blur"
]
},
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
"enum": [
"alt",
"ctrl",
"meta",
"shift"
]
},
"NavigationCause": {
"type": "string",
"enum": [
"user_typed",
"link",
"form_submit",
"reload",
"history",
"script",
"browser"
]
},
"SelectedOptionV3": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
"value"
],
"properties": {
"label": {
"type": [
"string",
"null"
]
},
"value": {
"type": "string"
}
}
},
"StepResultV3": {
"type": "object",
"required": [
"state"
],
"properties": {
"state": {
"type": "string"
}
}
},
"TargetDescriptorV3": {
"description": "Stable semantic handle for an interacted element within a page observation.",
"type": "object",
"properties": {
"ctx": {
"type": [
"string",
"null"
]
},
"name": {
"type": [
"string",
"null"
]
},
"ref": {
"type": [
"string",
"null"
]
},
"role": {
"type": [
"string",
"null"
]
},
"unmatched": {
"type": "boolean"
}
}
}
}
}
+451
View File
@@ -0,0 +1,451 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "TraceV2",
"description": "Persisted user-action trace exported by legacy `tool.record_stop` / `await`.\n\nUnknown extension fields are ignored so older traces remain readable. `states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace` classification.",
"type": "object",
"required": [
"entry",
"pages",
"recorded_at",
"steps"
],
"properties": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"pages": {
"type": "array",
"items": {
"$ref": "#/definitions/PageRefV2"
}
},
"purpose": {
"type": [
"string",
"null"
]
},
"recorded_at": {
"description": "RFC 3339 timestamp when recording stopped.",
"type": "string"
},
"started_at": {
"type": [
"string",
"null"
]
},
"states": false,
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/StepV2"
}
},
"version": {
"description": "Numeric version selects Trace v3. Legacy v2 envelopes omit this field.",
"not": {
"type": "integer"
}
}
},
"definitions": {
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
"enum": [
"alt",
"ctrl",
"meta",
"shift"
]
},
"PageRefV2": {
"description": "Page context dictionary entry — referenced by steps via `page` id.",
"type": "object",
"required": [
"id",
"url"
],
"properties": {
"id": {
"type": "string"
},
"title": {
"type": [
"string",
"null"
]
},
"url": {
"type": "string"
}
}
},
"SelectedOptionV2": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
"value"
],
"properties": {
"label": {
"type": [
"string",
"null"
]
},
"value": {
"type": "string"
}
}
},
"StepEffectV2": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
}
}
},
"StepV2": {
"description": "One recorded user action — discriminated union by `op` (v2).",
"oneOf": [
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"to"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target",
"value"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"redacted": {
"type": [
"boolean",
"null"
]
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"selection",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV2"
}
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"page"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV2"
},
{
"type": "null"
}
]
}
}
}
]
},
"TargetDescriptorV2": {
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
"type": "object",
"required": [
"tag"
],
"properties": {
"name": {
"type": [
"string",
"null"
]
},
"name_attr": {
"type": [
"string",
"null"
]
},
"nearby_label": {
"type": [
"string",
"null"
]
},
"placeholder": {
"type": [
"string",
"null"
]
},
"role": {
"type": [
"string",
"null"
]
},
"tag": {
"type": "string"
}
}
},
"TraceEntry": {
"type": "object",
"required": [
"start_url"
],
"properties": {
"start_url": {
"type": "string"
}
}
}
}
}
+496
View File
@@ -0,0 +1,496 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "TraceV3",
"description": "Wire trace returned by `tool.record_stop` / `await`.",
"type": "object",
"required": [
"entry",
"recorded_at",
"recorder",
"states",
"steps",
"stopped_by",
"version"
],
"properties": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"purpose": {
"type": [
"string",
"null"
]
},
"recorded_at": {
"type": "string"
},
"recorder": {
"$ref": "#/definitions/RecorderInfo"
},
"started_at": {
"type": [
"string",
"null"
]
},
"states": {
"type": "array",
"items": {
"$ref": "#/definitions/TraceStateV3"
}
},
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/StepV3"
}
},
"stopped_by": {
"$ref": "#/definitions/StopReason"
},
"version": {
"type": "integer",
"const": 3
}
},
"additionalProperties": false,
"definitions": {
"FillCommit": {
"type": "string",
"enum": [
"enter",
"suggestion",
"blur"
]
},
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
"enum": [
"alt",
"ctrl",
"meta",
"shift"
]
},
"NavigationCause": {
"type": "string",
"enum": [
"user_typed",
"link",
"form_submit",
"reload",
"history",
"script",
"browser"
]
},
"RecorderInfo": {
"type": "object",
"required": [
"bsk",
"vom"
],
"properties": {
"bsk": {
"type": "string"
},
"vom": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
}
}
},
"SelectedOptionV3": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
"value"
],
"properties": {
"label": {
"type": [
"string",
"null"
]
},
"value": {
"type": "string"
}
}
},
"StepResultV3": {
"type": "object",
"required": [
"state"
],
"properties": {
"state": {
"type": "string"
}
}
},
"StepV3": {
"description": "One recorded user action — discriminated union by `op`.",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"cause",
"id",
"op",
"result",
"state",
"to"
],
"properties": {
"cause": {
"$ref": "#/definitions/NavigationCause"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"commit",
"id",
"op",
"result",
"state",
"target",
"value"
],
"properties": {
"commit": {
"$ref": "#/definitions/FillCommit"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"redacted": {
"type": "boolean"
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV3"
}
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV3"
},
{
"type": "null"
}
]
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"scroll"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
}
]
},
"StopReason": {
"type": "string",
"enum": [
"user_finish",
"cli_stop"
]
},
"TargetDescriptorV3": {
"description": "Stable semantic handle for an interacted element within a page observation.",
"type": "object",
"properties": {
"ctx": {
"type": [
"string",
"null"
]
},
"name": {
"type": [
"string",
"null"
]
},
"ref": {
"type": [
"string",
"null"
]
},
"role": {
"type": [
"string",
"null"
]
},
"unmatched": {
"type": "boolean"
}
}
},
"TraceEntry": {
"type": "object",
"required": [
"start_url"
],
"properties": {
"start_url": {
"type": "string"
}
}
},
"TraceStateV3": {
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
"type": "object",
"required": [
"body",
"id",
"url"
],
"properties": {
"body": {
"description": "Full page observation (front matter + VOM body + annotations).",
"type": "string"
},
"id": {
"type": "string"
},
"title": {
"type": [
"string",
"null"
]
},
"truncated": {
"type": "boolean"
},
"url": {
"type": "string"
}
}
}
}
}
+6 -2
View File
@@ -113,8 +113,12 @@ fn main() {
dump!(RequestHelpParams, "tool_request_help_params");
dump!(RequestHelpResult, "tool_request_help_result");
dump!(Trace, "trace");
dump!(Step, "trace_step");
dump!(TraceV2, "trace_v2");
dump!(TraceV3, "trace_v3");
dump!(RecordedTrace, "trace");
dump!(StepV2, "trace_step_v2");
dump!(StepV3, "trace_step_v3");
dump!(RecordedStep, "trace_step");
dump!(RecordStartParams, "tool_record_start_params");
dump!(RecordStartResult, "tool_record_start_result");
dump!(RecordStopParams, "tool_record_stop_params");
+3
View File
@@ -9,6 +9,9 @@ pub mod navigation;
pub mod network;
pub mod observation;
pub mod record;
mod record_common;
mod record_v2;
mod record_v3;
pub mod script;
pub mod session;
pub mod tabs;
+380 -388
View File
@@ -1,147 +1,23 @@
//! Semantic user-action recording (`tool.record_start` / `stop` / `await`).
//!
//! Trace v2 is a **record-only** log of user actions: what was clicked, filled,
//! selected, and where navigation occurred. No LLM runs during recording, so
//! variable-vs-constant classification is **not** stored — executing agents
//! infer that at run time from raw values and control names.
//! Wire traces are either Trace v2 (`pages[]`) or Trace v3 (`version: 3`,
//! `states[]`). Version-specific models live in `record_v2` / `record_v3`.
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use super::interaction::KeyModifier;
pub use super::record_common::TraceEntry;
pub use super::record_v2::{
PageRefV2, SelectedOptionV2, StepCommonV2, StepEffectV2, StepV2, TargetDescriptorV2, TraceV2,
};
pub use super::record_v3::{
FillCommit, NavigationCause, RecorderInfo, SelectedOptionV3, StepCommonV3, StepResultV3,
StepV3, StopReason, TRACE_VERSION_V3, TargetDescriptorV3, TraceStateV3, TraceV3,
VOM_FORMAT_VERSION,
};
// ---------------------------------------------------------------------------
// Target
// ---------------------------------------------------------------------------
/// Stable semantic handle for an interacted element.
///
/// `name` and `nearby_label` are **untrusted page text**.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct TargetDescriptor {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub role: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
pub tag: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name_attr: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub placeholder: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub nearby_label: Option<String>,
}
// ---------------------------------------------------------------------------
// Trace envelope
// ---------------------------------------------------------------------------
/// Recording entry point — first URL the flow starts from.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct TraceEntry {
pub start_url: String,
}
/// Page context dictionary entry — referenced by steps via `page` id.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct PageRef {
pub id: String,
pub url: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub title: Option<String>,
}
/// One selected option (`select` op).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct SelectedOption {
pub value: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub label: Option<String>,
}
/// Observed navigation after a step (objective fact only).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct StepEffect {
/// Reference into `pages[]` for the destination page.
pub navigated_to: String,
}
/// Fields shared by every step variant (flattened in JSON).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct StepCommon {
pub id: u32,
/// Reference into `pages[]`.
pub page: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub effect: Option<StepEffect>,
}
// ---------------------------------------------------------------------------
// Step op-specific payloads
// ---------------------------------------------------------------------------
/// One recorded user action — discriminated union by `op`.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[serde(tag = "op", rename_all = "snake_case")]
pub enum Step {
Navigate {
#[serde(flatten)]
common: StepCommon,
to: String,
},
Click {
#[serde(flatten)]
common: StepCommon,
target: TargetDescriptor,
},
Hover {
#[serde(flatten)]
common: StepCommon,
target: TargetDescriptor,
},
Fill {
#[serde(flatten)]
common: StepCommon,
target: TargetDescriptor,
value: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
redacted: Option<bool>,
},
Select {
#[serde(flatten)]
common: StepCommon,
target: TargetDescriptor,
selection: Vec<SelectedOption>,
},
Press {
#[serde(flatten)]
common: StepCommon,
key: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
modifiers: Option<Vec<KeyModifier>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
target: Option<TargetDescriptor>,
},
}
// ---------------------------------------------------------------------------
// Trace root
// ---------------------------------------------------------------------------
/// Persisted user-action trace exported by `tool.record_stop` / `await`.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct Trace {
/// RFC 3339 timestamp when recording stopped.
pub recorded_at: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub started_at: Option<String>,
/// Optional user-provided goal from `--purpose` (metadata only).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub purpose: Option<String>,
pub entry: TraceEntry,
pub pages: Vec<PageRef>,
pub steps: Vec<Step>,
}
/// Logical v2 identifier. Not a wire field — v2 envelopes omit `version`.
pub const TRACE_VERSION_V2: u32 = 2;
// ---------------------------------------------------------------------------
// RPC params / results
@@ -169,9 +45,209 @@ pub struct RecordStopParams {
pub session_id: String,
}
/// Wire trace payload — v2 (legacy `pages[]`) or v3 (`version: 3`, `states[]`).
///
/// Classification matches [`RecordedTrace::classify_value`]: a numeric `version`
/// selects v3 (and forbids `pages[]`); otherwise the envelope is v2 and must
/// not include `states[]`.
#[derive(Debug, Clone, PartialEq)]
pub enum RecordedTrace {
V2(TraceV2),
V3(TraceV3),
}
impl RecordedTrace {
pub fn classify_value(v: &serde_json::Value) -> Result<Self, String> {
if let Some(ver) = v.get("version").and_then(|x| x.as_u64()) {
if ver != u64::from(TRACE_VERSION_V3) {
return Err(format!("unsupported trace version {ver}"));
}
if v.get("pages").is_some() {
return Err("trace v3 must not include legacy pages[]".into());
}
return serde_json::from_value(v.clone())
.map(RecordedTrace::V3)
.map_err(|e| e.to_string());
}
if v.get("states").is_some() {
return Err("trace v2 must not include states[]; set version: 3 for Trace v3".into());
}
if v.get("pages").is_some() {
return serde_json::from_value(v.clone())
.map(RecordedTrace::V2)
.map_err(|e| e.to_string());
}
Err("ambiguous or unparseable trace".into())
}
}
impl Serialize for RecordedTrace {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
match self {
RecordedTrace::V2(t) => t.serialize(serializer),
RecordedTrace::V3(t) => t.serialize(serializer),
}
}
}
impl<'de> Deserialize<'de> for RecordedTrace {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let value = serde_json::Value::deserialize(deserializer)?;
Self::classify_value(&value).map_err(serde::de::Error::custom)
}
}
impl JsonSchema for RecordedTrace {
fn schema_name() -> String {
"RecordedTrace".into()
}
fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::schema::Schema {
// Mirror classify_value: a numeric `version` selects v3; otherwise v2.
// TraceV2 stays open for unknown extension fields but forbids `states[]`
// and a numeric `version`. TraceV3 already denies `pages[]`.
let mut version_props = schemars::Map::new();
version_props.insert(
"version".into(),
schemars::schema::SchemaObject {
instance_type: Some(schemars::schema::InstanceType::Integer.into()),
..Default::default()
}
.into(),
);
let mut required = schemars::Set::new();
required.insert("version".into());
schemars::schema::SchemaObject {
subschemas: Some(Box::new(schemars::schema::SubschemaValidation {
if_schema: Some(Box::new(
schemars::schema::SchemaObject {
object: Some(Box::new(schemars::schema::ObjectValidation {
properties: version_props,
required,
..Default::default()
})),
..Default::default()
}
.into(),
)),
then_schema: Some(Box::new(generator.subschema_for::<TraceV3>())),
else_schema: Some(Box::new(generator.subschema_for::<TraceV2>())),
..Default::default()
})),
..Default::default()
}
.into()
}
}
/// One recorded step — v2 (`page`) or v3 (`state` / `result.state`).
///
/// Classification matches [`RecordedStep::classify_value`]: `state` selects v3
/// (and forbids `page`); otherwise the step is v2 and must include `page`.
#[derive(Debug, Clone, PartialEq)]
pub enum RecordedStep {
V2(StepV2),
V3(StepV3),
}
impl RecordedStep {
pub fn classify_value(v: &serde_json::Value) -> Result<Self, String> {
if v.get("state").is_some() {
if v.get("page").is_some() {
return Err("step v3 must not include legacy page".into());
}
return serde_json::from_value(v.clone())
.map(RecordedStep::V3)
.map_err(|e| e.to_string());
}
if v.get("page").is_some() {
return serde_json::from_value(v.clone())
.map(RecordedStep::V2)
.map_err(|e| e.to_string());
}
Err("ambiguous or unparseable step".into())
}
}
impl Serialize for RecordedStep {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
match self {
RecordedStep::V2(s) => s.serialize(serializer),
RecordedStep::V3(s) => s.serialize(serializer),
}
}
}
impl<'de> Deserialize<'de> for RecordedStep {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let value = serde_json::Value::deserialize(deserializer)?;
Self::classify_value(&value).map_err(serde::de::Error::custom)
}
}
impl JsonSchema for RecordedStep {
fn schema_name() -> String {
"RecordedStep".into()
}
fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::schema::Schema {
// Mirror classify_value: `state` selects v3; otherwise v2.
// Mixed page + state is rejected at this layer because StepV2/StepV3
// still allow unknown extension fields.
let mut required = schemars::Set::new();
required.insert("state".into());
let mut mixed_keys = schemars::Set::new();
mixed_keys.insert("page".into());
mixed_keys.insert("state".into());
schemars::schema::SchemaObject {
subschemas: Some(Box::new(schemars::schema::SubschemaValidation {
if_schema: Some(Box::new(
schemars::schema::SchemaObject {
object: Some(Box::new(schemars::schema::ObjectValidation {
required,
..Default::default()
})),
..Default::default()
}
.into(),
)),
then_schema: Some(Box::new(generator.subschema_for::<StepV3>())),
else_schema: Some(Box::new(generator.subschema_for::<StepV2>())),
not: Some(Box::new(
schemars::schema::SchemaObject {
object: Some(Box::new(schemars::schema::ObjectValidation {
required: mixed_keys,
..Default::default()
})),
..Default::default()
}
.into(),
)),
..Default::default()
})),
..Default::default()
}
.into()
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct RecordStopResult {
pub trace: Trace,
pub trace: RecordedTrace,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
@@ -183,281 +259,197 @@ pub struct RecordAwaitParams {
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct RecordAwaitResult {
pub trace: Trace,
pub trace: RecordedTrace,
}
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn sample_common(id: u32) -> StepCommon {
StepCommon {
id,
page: "p1".into(),
effect: None,
#[test]
fn recorded_trace_classifies_v2_and_v3() {
let v2 = json!({
"recorded_at": "2026-07-21T08:00:00Z",
"entry": { "start_url": "https://example.com/" },
"pages": [{ "id": "p1", "url": "https://example.com/" }],
"steps": []
});
match RecordedTrace::classify_value(&v2).unwrap() {
RecordedTrace::V2(_) => {}
other => panic!("expected v2, got {other:?}"),
}
}
fn sample_target() -> TargetDescriptor {
TargetDescriptor {
role: Some("button".into()),
name: Some("发布".into()),
tag: "button".into(),
name_attr: None,
placeholder: None,
nearby_label: None,
}
}
fn sample_trace() -> Trace {
Trace {
recorded_at: "2026-07-17T09:01:10Z".into(),
started_at: Some("2026-07-17T09:00:00Z".into()),
purpose: Some("发布一篇文章".into()),
entry: TraceEntry {
start_url: "https://x.com/editor".into(),
},
pages: vec![
PageRef {
id: "p1".into(),
url: "https://x.com/editor".into(),
title: Some("写文章".into()),
},
PageRef {
id: "p2".into(),
url: "https://x.com/p/99".into(),
title: None,
},
],
steps: vec![
Step::Fill {
common: sample_common(1),
target: TargetDescriptor {
role: Some("textbox".into()),
name: Some("标题".into()),
tag: "input".into(),
name_attr: None,
placeholder: None,
nearby_label: None,
},
value: "我的第一篇文章".into(),
redacted: None,
},
Step::Select {
common: sample_common(3),
target: TargetDescriptor {
role: Some("combobox".into()),
name: Some("分类".into()),
tag: "select".into(),
name_attr: None,
placeholder: None,
nearby_label: None,
},
selection: vec![SelectedOption {
value: "tech".into(),
label: Some("技术分享".into()),
}],
},
Step::Click {
common: StepCommon {
effect: Some(StepEffect {
navigated_to: "p2".into(),
}),
..sample_common(4)
},
target: sample_target(),
},
],
let v3 = json!({
"version": 3,
"recorded_at": "2026-07-21T08:00:00Z",
"stopped_by": "user_finish",
"entry": { "start_url": "https://example.com/" },
"recorder": { "bsk": "0.1.10", "vom": 1 },
"states": [],
"steps": []
});
match RecordedTrace::classify_value(&v3).unwrap() {
RecordedTrace::V3(t) => assert_eq!(t.version, 3),
other => panic!("expected v3, got {other:?}"),
}
}
#[test]
fn step_click_round_trips() {
let step = Step::Click {
common: sample_common(1),
target: sample_target(),
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v.get("op").and_then(|v| v.as_str()), Some("click"));
assert_eq!(v.get("page").and_then(|v| v.as_str()), Some("p1"));
assert!(v.get("key").is_none());
let round: Step = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
fn recorded_trace_rejects_mixed_and_unsupported_versions() {
let v2_with_states = json!({
"recorded_at": "2026-07-21T08:00:00Z",
"entry": { "start_url": "https://example.com/" },
"pages": [{ "id": "p1", "url": "https://example.com/" }],
"states": [],
"steps": []
});
assert!(RecordedTrace::classify_value(&v2_with_states).is_err());
let v3_with_pages = json!({
"version": 3,
"recorded_at": "2026-07-21T08:00:00Z",
"stopped_by": "user_finish",
"entry": { "start_url": "https://example.com/" },
"recorder": { "bsk": "0.1.10", "vom": 1 },
"pages": [{ "id": "p1", "url": "https://example.com/" }],
"states": [],
"steps": []
});
assert!(RecordedTrace::classify_value(&v3_with_pages).is_err());
let unsupported = json!({
"version": 2,
"recorded_at": "2026-07-21T08:00:00Z",
"stopped_by": "user_finish",
"entry": { "start_url": "https://example.com/" },
"recorder": { "bsk": "0.1.10", "vom": 1 },
"states": [],
"steps": []
});
assert!(RecordedTrace::classify_value(&unsupported).is_err());
assert!(serde_json::from_value::<TraceV3>(unsupported).is_err());
}
#[test]
fn step_fill_with_raw_value_round_trips() {
let step = Step::Fill {
common: sample_common(2),
target: TargetDescriptor {
role: Some("textbox".into()),
name: Some("服务名称".into()),
tag: "input".into(),
name_attr: Some("serviceName".into()),
placeholder: None,
nearby_label: None,
},
value: "my-svc".into(),
redacted: None,
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["op"], "fill");
assert_eq!(v["value"], "my-svc");
let round: Step = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn step_fill_password_is_redacted() {
let step = Step::Fill {
common: sample_common(1),
target: TargetDescriptor {
role: Some("textbox".into()),
name: Some("密码".into()),
tag: "input".into(),
name_attr: None,
placeholder: None,
nearby_label: None,
},
value: "***".into(),
redacted: Some(true),
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["value"], "***");
assert_eq!(v["redacted"], true);
}
#[test]
fn step_select_uses_object_array() {
let step = Step::Select {
common: sample_common(3),
target: TargetDescriptor {
role: Some("combobox".into()),
name: Some("分类".into()),
tag: "select".into(),
name_attr: None,
placeholder: None,
nearby_label: None,
},
selection: vec![SelectedOption {
value: "tech".into(),
label: Some("技术分享".into()),
}],
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["selection"][0]["value"], "tech");
assert_eq!(v["selection"][0]["label"], "技术分享");
let round: Step = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn step_navigate_round_trips() {
let step = Step::Navigate {
common: sample_common(1),
to: "https://example.com".into(),
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["op"], "navigate");
assert_eq!(v["to"], "https://example.com");
let round: Step = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn step_press_with_modifiers_round_trips() {
let step = Step::Press {
common: StepCommon {
effect: Some(StepEffect {
navigated_to: "p2".into(),
}),
..sample_common(2)
},
key: "Enter".into(),
modifiers: Some(vec![KeyModifier::Ctrl, KeyModifier::Shift]),
target: Some(sample_target()),
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["key"], "Enter");
assert_eq!(v["modifiers"], json!(["ctrl", "shift"]));
assert_eq!(v["effect"]["navigated_to"], "p2");
let round: Step = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn trace_record_only_round_trips() {
let trace = sample_trace();
let v = serde_json::to_value(&trace).unwrap();
assert!(v.get("version").is_none());
fn assert_classifies_by_integer_version(schema: &serde_json::Value) {
assert_eq!(
v.get("purpose").and_then(|v| v.as_str()),
Some("发布一篇文章")
schema["if"]["required"],
json!(["version"]),
"numeric version is the RecordedTrace classification key"
);
assert!(v.get("parameters").is_none());
assert!(v.get("site").is_none());
assert!(v.get("goal").is_none());
assert_eq!(
v.get("entry")
.and_then(|e| e.get("start_url"))
.and_then(|u| u.as_str()),
Some("https://x.com/editor")
);
let round: Trace = serde_json::from_value(v).unwrap();
assert_eq!(round, trace);
assert_eq!(schema["if"]["properties"]["version"]["type"], "integer");
assert_eq!(schema["then"]["$ref"], "#/definitions/TraceV3");
assert_eq!(schema["else"]["$ref"], "#/definitions/TraceV2");
}
#[test]
fn discriminated_union_click_ignores_extra_key_field() {
let bad = json!({
fn recorded_trace_schema_includes_v2_and_v3() {
let schema = serde_json::to_value(schemars::schema_for!(RecordStopResult)).unwrap();
assert_classifies_by_integer_version(&schema["definitions"]["RecordedTrace"]);
assert_eq!(
schema["definitions"]["TraceV3"]["properties"]["version"]["const"],
3
);
}
#[test]
fn recorded_trace_schema_follows_classify_value() {
let standalone = serde_json::to_value(schemars::schema_for!(RecordedTrace)).unwrap();
assert_classifies_by_integer_version(&standalone);
assert!(standalone["definitions"].get("TraceV2").is_some());
assert!(standalone["definitions"].get("TraceV3").is_some());
let stop_result = serde_json::to_value(schemars::schema_for!(RecordStopResult)).unwrap();
assert_classifies_by_integer_version(&stop_result["definitions"]["RecordedTrace"]);
}
#[test]
fn standalone_trace_schema_is_recorded_trace_union() {
let schema = serde_json::to_value(schemars::schema_for!(RecordedTrace)).unwrap();
assert_eq!(schema["title"], "RecordedTrace");
assert_classifies_by_integer_version(&schema);
assert!(schema["definitions"].get("TraceV2").is_some());
assert!(schema["definitions"].get("TraceV3").is_some());
}
#[test]
fn recorded_trace_accepts_v2_with_unknown_extension_fields() {
let v2 = json!({
"recorded_at": "2026-07-21T08:00:00Z",
"entry": { "start_url": "https://example.com/" },
"pages": [{ "id": "p1", "url": "https://example.com/" }],
"steps": [],
"meta": { "tool": "legacy-exporter" }
});
match RecordedTrace::classify_value(&v2).unwrap() {
RecordedTrace::V2(trace) => assert_eq!(trace.pages.len(), 1),
other => panic!("expected v2, got {other:?}"),
}
}
fn v2_click() -> serde_json::Value {
json!({
"op": "click",
"id": 1,
"page": "p1",
"target": { "tag": "button", "name": "OK" },
"key": "Enter"
});
let step: Step = serde_json::from_value(bad).unwrap();
assert!(matches!(step, Step::Click { .. }));
let v = serde_json::to_value(&step).unwrap();
assert!(v.get("key").is_none());
"target": { "tag": "button", "role": "button", "name": "发布" }
})
}
fn v3_click() -> serde_json::Value {
json!({
"op": "click",
"id": 1,
"state": "s1",
"result": { "state": "s2" },
"target": { "ref": "e1", "role": "button", "name": "发布" }
})
}
fn assert_step_classifies_by_state(schema: &serde_json::Value) {
assert_eq!(
schema["if"]["required"],
json!(["state"]),
"state is the RecordedStep classification key"
);
assert_eq!(schema["then"]["$ref"], "#/definitions/StepV3");
assert_eq!(schema["else"]["$ref"], "#/definitions/StepV2");
let mut forbidden = schema["not"]["required"]
.as_array()
.expect("RecordedStep schema must reject mixed page + state steps")
.iter()
.filter_map(|v| v.as_str())
.collect::<Vec<_>>();
forbidden.sort_unstable();
assert_eq!(forbidden, ["page", "state"]);
}
#[test]
fn extension_trace_deserializes() {
let v = json!({
"recorded_at": "2026-07-21T08:00:00Z",
"started_at": "2026-07-21T07:59:00Z",
"purpose": "demo",
"entry": { "start_url": "https://example.com/editor" },
"pages": [
{ "id": "p1", "url": "https://example.com/editor" },
{ "id": "p2", "url": "https://example.com/p/99" }
],
"steps": [
{
"op": "fill",
"id": 1,
"page": "p1",
"target": { "tag": "input", "role": "textbox", "name": "标题" },
"value": "hello"
},
{
"op": "click",
"id": 2,
"page": "p1",
"target": { "tag": "button", "role": "button", "name": "发布" },
"effect": { "navigated_to": "p2" }
}
]
});
let trace: Trace = serde_json::from_value(v).unwrap();
assert_eq!(trace.entry.start_url, "https://example.com/editor");
assert_eq!(trace.pages.len(), 2);
assert_eq!(trace.steps.len(), 2);
fn recorded_step_classifies_v2_and_v3() {
match RecordedStep::classify_value(&v2_click()).unwrap() {
RecordedStep::V2(StepV2::Click { .. }) => {}
other => panic!("expected v2 click, got {other:?}"),
}
match RecordedStep::classify_value(&v3_click()).unwrap() {
RecordedStep::V3(StepV3::Click { .. }) => {}
other => panic!("expected v3 click, got {other:?}"),
}
}
#[test]
fn recorded_step_rejects_mixed_page_and_state() {
let mut mixed = v2_click();
mixed["state"] = json!("s1");
mixed["result"] = json!({ "state": "s2" });
assert!(RecordedStep::classify_value(&mixed).is_err());
}
#[test]
fn recorded_step_schema_follows_classify_value() {
let schema = serde_json::to_value(schemars::schema_for!(RecordedStep)).unwrap();
assert_eq!(schema["title"], "RecordedStep");
assert_step_classifies_by_state(&schema);
assert!(schema["definitions"].get("StepV2").is_some());
assert!(schema["definitions"].get("StepV3").is_some());
}
}
@@ -0,0 +1,7 @@
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct TraceEntry {
pub start_url: String,
}
+330
View File
@@ -0,0 +1,330 @@
//! Trace v2 — record-only user-action log with `pages[]` and step `page` refs.
//!
//! Legacy wire format with no top-level `version` field.
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use super::interaction::KeyModifier;
use super::record_common::TraceEntry;
/// Stable semantic handle for an interacted element (v2).
///
/// `name` and `nearby_label` are **untrusted page text**.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct TargetDescriptorV2 {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub role: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
pub tag: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name_attr: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub placeholder: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub nearby_label: Option<String>,
}
/// Page context dictionary entry — referenced by steps via `page` id.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct PageRefV2 {
pub id: String,
pub url: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub title: Option<String>,
}
/// One selected option (`select` op).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct SelectedOptionV2 {
pub value: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub label: Option<String>,
}
/// Observed navigation after a step (objective fact only).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct StepEffectV2 {
/// Reference into `pages[]` for the destination page.
pub navigated_to: String,
}
/// Fields shared by every v2 step variant (flattened in JSON).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct StepCommonV2 {
pub id: u32,
/// Reference into `pages[]`.
pub page: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub effect: Option<StepEffectV2>,
}
/// One recorded user action — discriminated union by `op` (v2).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[serde(tag = "op", rename_all = "snake_case")]
pub enum StepV2 {
Navigate {
#[serde(flatten)]
common: StepCommonV2,
to: String,
},
Click {
#[serde(flatten)]
common: StepCommonV2,
target: TargetDescriptorV2,
},
Hover {
#[serde(flatten)]
common: StepCommonV2,
target: TargetDescriptorV2,
},
Fill {
#[serde(flatten)]
common: StepCommonV2,
target: TargetDescriptorV2,
value: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
redacted: Option<bool>,
},
Select {
#[serde(flatten)]
common: StepCommonV2,
target: TargetDescriptorV2,
selection: Vec<SelectedOptionV2>,
},
Press {
#[serde(flatten)]
common: StepCommonV2,
key: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
modifiers: Option<Vec<KeyModifier>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
target: Option<TargetDescriptorV2>,
},
}
/// Persisted user-action trace exported by legacy `tool.record_stop` / `await`.
///
/// Unknown extension fields are ignored so older traces remain readable.
/// `states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace`
/// classification — serde still ignores them, but the JSON Schema rejects them.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct TraceV2 {
/// RFC 3339 timestamp when recording stopped.
pub recorded_at: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub started_at: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub purpose: Option<String>,
pub entry: TraceEntry,
pub pages: Vec<PageRefV2>,
pub steps: Vec<StepV2>,
}
fn constrain_trace_v2_schema(mut schema: schemars::schema::Schema) -> schemars::schema::Schema {
let schemars::schema::Schema::Object(obj) = &mut schema else {
return schema;
};
obj.metadata().title = Some("TraceV2".into());
obj.metadata().description = Some(
"Persisted user-action trace exported by legacy `tool.record_stop` / `await`.\n\n\
Unknown extension fields are ignored so older traces remain readable. \
`states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace` classification."
.into(),
);
let object = obj.object.get_or_insert_with(Default::default);
object
.properties
.insert("states".into(), schemars::schema::Schema::Bool(false));
object.properties.insert(
"version".into(),
schemars::schema::SchemaObject {
metadata: Some(Box::new(schemars::schema::Metadata {
description: Some(
"Numeric version selects Trace v3. Legacy v2 envelopes omit this field.".into(),
),
..Default::default()
})),
subschemas: Some(Box::new(schemars::schema::SubschemaValidation {
not: Some(Box::new(
schemars::schema::SchemaObject {
instance_type: Some(schemars::schema::InstanceType::Integer.into()),
..Default::default()
}
.into(),
)),
..Default::default()
})),
..Default::default()
}
.into(),
);
schema
}
impl JsonSchema for TraceV2 {
fn schema_name() -> String {
"TraceV2".into()
}
fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::schema::Schema {
#[derive(JsonSchema)]
#[allow(dead_code)]
struct TraceV2Shape {
/// RFC 3339 timestamp when recording stopped.
recorded_at: String,
started_at: Option<String>,
purpose: Option<String>,
entry: TraceEntry,
pages: Vec<PageRefV2>,
steps: Vec<StepV2>,
}
constrain_trace_v2_schema(TraceV2Shape::json_schema(generator))
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn sample_common(id: u32) -> StepCommonV2 {
StepCommonV2 {
id,
page: "p1".into(),
effect: None,
}
}
fn sample_target() -> TargetDescriptorV2 {
TargetDescriptorV2 {
role: Some("button".into()),
name: Some("发布".into()),
tag: "button".into(),
name_attr: None,
placeholder: None,
nearby_label: None,
}
}
#[test]
fn trace_v2_has_no_version_field() {
let trace = TraceV2 {
recorded_at: "2026-07-17T09:01:10Z".into(),
started_at: None,
purpose: None,
entry: TraceEntry {
start_url: "https://example.com/".into(),
},
pages: vec![PageRefV2 {
id: "p1".into(),
url: "https://example.com/".into(),
title: None,
}],
steps: vec![StepV2::Click {
common: sample_common(1),
target: sample_target(),
}],
};
let v = serde_json::to_value(&trace).unwrap();
assert!(v.get("version").is_none());
assert!(v.get("pages").is_some());
assert!(v.get("states").is_none());
}
#[test]
fn extension_v2_trace_deserializes() {
let v = json!({
"recorded_at": "2026-07-21T08:00:00Z",
"started_at": "2026-07-21T07:59:00Z",
"purpose": "demo",
"entry": { "start_url": "https://example.com/editor" },
"pages": [
{ "id": "p1", "url": "https://example.com/editor" },
{ "id": "p2", "url": "https://example.com/p/99" }
],
"steps": [
{
"op": "fill",
"id": 1,
"page": "p1",
"target": { "tag": "input", "role": "textbox", "name": "标题" },
"value": "hello"
},
{
"op": "click",
"id": 2,
"page": "p1",
"target": { "tag": "button", "role": "button", "name": "发布" },
"effect": { "navigated_to": "p2" }
}
]
});
let trace: TraceV2 = serde_json::from_value(v).unwrap();
assert_eq!(trace.pages.len(), 2);
assert_eq!(trace.steps.len(), 2);
}
#[test]
fn v2_hover_steps_round_trip() {
let value = json!({
"recorded_at": "2026-07-21T08:00:00Z",
"entry": { "start_url": "https://example.com/" },
"pages": [{ "id": "p1", "url": "https://example.com/" }],
"steps": [{
"op": "hover",
"id": 1,
"page": "p1",
"target": { "tag": "span", "role": "button", "name": "Account" }
}]
});
let trace: TraceV2 = serde_json::from_value(value).unwrap();
assert!(matches!(trace.steps.as_slice(), [StepV2::Hover { .. }]));
assert_eq!(
serde_json::to_value(&trace).unwrap()["steps"][0]["op"],
"hover"
);
}
#[test]
fn trace_v2_ignores_unknown_extension_fields() {
let value = json!({
"recorded_at": "2026-07-21T08:00:00Z",
"entry": { "start_url": "https://example.com/" },
"pages": [{ "id": "p1", "url": "https://example.com/" }],
"steps": [],
"meta": { "tool": "legacy-exporter" }
});
let trace: TraceV2 = serde_json::from_value(value).unwrap();
assert_eq!(trace.pages.len(), 1);
assert!(trace.steps.is_empty());
}
#[test]
fn trace_v2_schema_allows_additional_properties() {
let schema = serde_json::to_value(schemars::schema_for!(TraceV2)).unwrap();
assert_ne!(
schema.get("additionalProperties"),
Some(&serde_json::Value::Bool(false)),
"Trace v2 must keep accepting traces with unknown extension fields"
);
}
#[test]
fn trace_v2_schema_forbids_classification_keys() {
let schema = serde_json::to_value(schemars::schema_for!(TraceV2)).unwrap();
assert_eq!(
schema["properties"]["states"],
json!(false),
"Trace v2 schema must reject states[] (reserved for v3)"
);
assert_eq!(
schema["properties"]["version"]["not"]["type"], "integer",
"Trace v2 schema must reject a numeric version (that selects the v3 path)"
);
}
}
+428
View File
@@ -0,0 +1,428 @@
//! Trace v3 — state-action-state log with `states[]` and step `state` refs.
//!
//! Wire format with top-level `version: 3`.
use schemars::JsonSchema;
use serde::{Deserialize, Deserializer, Serialize};
use super::interaction::KeyModifier;
use super::record_common::TraceEntry;
pub const TRACE_VERSION_V3: u32 = 3;
pub const VOM_FORMAT_VERSION: u32 = 1;
fn deserialize_trace_v3_version<'de, D>(deserializer: D) -> Result<u32, D::Error>
where
D: Deserializer<'de>,
{
let version = u32::deserialize(deserializer)?;
if version != TRACE_VERSION_V3 {
return Err(serde::de::Error::custom(format!(
"unsupported trace version {version} (expected {TRACE_VERSION_V3})"
)));
}
Ok(version)
}
fn trace_v3_version_schema(_: &mut schemars::r#gen::SchemaGenerator) -> schemars::schema::Schema {
schemars::schema::SchemaObject {
instance_type: Some(schemars::schema::InstanceType::Integer.into()),
const_value: Some(serde_json::json!(TRACE_VERSION_V3)),
..Default::default()
}
.into()
}
/// Stable semantic handle for an interacted element within a page observation.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct TargetDescriptorV3 {
#[serde(default, skip_serializing_if = "Option::is_none", rename = "ref")]
pub element_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub role: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ctx: Option<String>,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub unmatched: bool,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct RecorderInfo {
pub bsk: String,
pub vom: u32,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum StopReason {
UserFinish,
CliStop,
}
/// Page observation dictionary entry — referenced by steps via `state` / `result.state`.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct TraceStateV3 {
pub id: String,
pub url: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub title: Option<String>,
/// Full page observation (front matter + VOM body + annotations).
pub body: String,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub truncated: bool,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct StepResultV3 {
pub state: String,
}
/// Fields shared by every step variant (flattened in JSON).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct StepCommonV3 {
pub id: u32,
/// Observation id immediately before this action.
pub state: String,
pub result: StepResultV3,
}
/// One selected option (`select` op).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct SelectedOptionV3 {
pub value: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub label: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum NavigationCause {
UserTyped,
Link,
FormSubmit,
Reload,
History,
Script,
Browser,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum FillCommit {
Enter,
Suggestion,
Blur,
}
/// One recorded user action — discriminated union by `op`.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[serde(tag = "op", rename_all = "snake_case")]
pub enum StepV3 {
Navigate {
#[serde(flatten)]
common: StepCommonV3,
to: String,
cause: NavigationCause,
},
Click {
#[serde(flatten)]
common: StepCommonV3,
target: TargetDescriptorV3,
},
Hover {
#[serde(flatten)]
common: StepCommonV3,
target: TargetDescriptorV3,
},
Fill {
#[serde(flatten)]
common: StepCommonV3,
target: TargetDescriptorV3,
value: String,
commit: FillCommit,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
redacted: bool,
},
Select {
#[serde(flatten)]
common: StepCommonV3,
target: TargetDescriptorV3,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
selection: Vec<SelectedOptionV3>,
},
Press {
#[serde(flatten)]
common: StepCommonV3,
key: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
modifiers: Option<Vec<KeyModifier>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
target: Option<TargetDescriptorV3>,
},
Scroll {
#[serde(flatten)]
common: StepCommonV3,
},
}
/// Wire trace returned by `tool.record_stop` / `await`.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct TraceV3 {
#[serde(deserialize_with = "deserialize_trace_v3_version")]
#[schemars(schema_with = "trace_v3_version_schema")]
pub version: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub purpose: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub started_at: Option<String>,
pub recorded_at: String,
pub stopped_by: StopReason,
pub entry: TraceEntry,
pub recorder: RecorderInfo,
pub states: Vec<TraceStateV3>,
pub steps: Vec<StepV3>,
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn sample_common(id: u32, state: &str, result_state: &str) -> StepCommonV3 {
StepCommonV3 {
id,
state: state.into(),
result: StepResultV3 {
state: result_state.into(),
},
}
}
fn sample_target() -> TargetDescriptorV3 {
TargetDescriptorV3 {
element_ref: Some("e21".into()),
role: Some("button".into()),
name: Some("发布".into()),
ctx: Some("金桔柠檬 6 号".into()),
unmatched: false,
}
}
fn sample_trace() -> TraceV3 {
TraceV3 {
version: TRACE_VERSION_V3,
purpose: Some("把草稿商品发布上架".into()),
started_at: Some("2026-08-10T02:10:41.080Z".into()),
recorded_at: "2026-08-10T02:12:55.360Z".into(),
stopped_by: StopReason::UserFinish,
entry: TraceEntry {
start_url: "https://example.com/".into(),
},
recorder: RecorderInfo {
bsk: "0.1.10".into(),
vom: VOM_FORMAT_VERSION,
},
states: vec![
TraceStateV3 {
id: "s1".into(),
url: "https://example.com/".into(),
title: Some("Example Domain".into()),
body: "@vom 1\nRootWebArea \"Example Domain\"".into(),
truncated: false,
},
TraceStateV3 {
id: "s2".into(),
url: "https://shop.example.com/products?status=draft".into(),
title: Some("商品管理".into()),
body: "@vom 1\nRootWebArea \"商品管理\"".into(),
truncated: false,
},
],
steps: vec![
StepV3::Navigate {
common: sample_common(1, "s1", "s2"),
to: "https://shop.example.com/products?status=draft".into(),
cause: NavigationCause::UserTyped,
},
StepV3::Fill {
common: sample_common(2, "s2", "s2"),
target: TargetDescriptorV3 {
element_ref: Some("e12".into()),
role: Some("textbox".into()),
name: Some("搜索商品".into()),
ctx: None,
unmatched: false,
},
value: "金桔柠檬".into(),
commit: FillCommit::Enter,
redacted: false,
},
StepV3::Click {
common: sample_common(3, "s2", "s2"),
target: sample_target(),
},
],
}
}
#[test]
fn step_click_round_trips() {
let step = StepV3::Click {
common: sample_common(1, "s1", "s2"),
target: sample_target(),
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v.get("op").and_then(|v| v.as_str()), Some("click"));
assert_eq!(v.get("state").and_then(|v| v.as_str()), Some("s1"));
assert_eq!(v["result"]["state"], "s2");
assert_eq!(v["target"]["ref"], "e21");
let round: StepV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn step_fill_with_commit_round_trips() {
let step = StepV3::Fill {
common: sample_common(2, "s2", "s3"),
target: TargetDescriptorV3 {
element_ref: Some("e12".into()),
role: Some("textbox".into()),
name: Some("搜索商品".into()),
ctx: None,
unmatched: false,
},
value: "browser skill".into(),
commit: FillCommit::Enter,
redacted: false,
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["op"], "fill");
assert_eq!(v["commit"], "enter");
assert!(v.get("redacted").is_none());
let round: StepV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn step_fill_password_is_redacted() {
let step = StepV3::Fill {
common: sample_common(1, "s1", "s1"),
target: TargetDescriptorV3 {
element_ref: Some("e3".into()),
role: Some("textbox".into()),
name: Some("密码".into()),
ctx: None,
unmatched: false,
},
value: "***".into(),
commit: FillCommit::Blur,
redacted: true,
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["value"], "***");
assert_eq!(v["redacted"], true);
}
#[test]
fn step_navigate_with_cause_round_trips() {
let step = StepV3::Navigate {
common: sample_common(1, "s1", "s2"),
to: "https://example.com".into(),
cause: NavigationCause::UserTyped,
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["op"], "navigate");
assert_eq!(v["cause"], "user_typed");
let round: StepV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn trace_v3_round_trips() {
let trace = sample_trace();
let v = serde_json::to_value(&trace).unwrap();
assert_eq!(v.get("version").and_then(|v| v.as_u64()), Some(3));
assert!(v.get("pages").is_none());
assert_eq!(v["recorder"]["vom"], 1);
assert_eq!(v["states"].as_array().unwrap().len(), 2);
let round: TraceV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, trace);
}
#[test]
fn default_fields_are_omitted() {
let step = StepV3::Click {
common: sample_common(1, "s1", "s1"),
target: TargetDescriptorV3 {
element_ref: Some("e1".into()),
role: Some("button".into()),
name: Some("OK".into()),
ctx: None,
unmatched: false,
},
};
let v = serde_json::to_value(&step).unwrap();
assert!(v.get("unmatched").is_none());
assert!(v["target"].get("ctx").is_none());
assert!(v["target"].get("unmatched").is_none());
}
#[test]
fn unmatched_target_serializes_flag() {
let step = StepV3::Click {
common: sample_common(1, "s1", "s2"),
target: TargetDescriptorV3 {
element_ref: None,
role: Some("button".into()),
name: Some("发布".into()),
ctx: None,
unmatched: true,
},
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["target"]["unmatched"], true);
assert!(v["target"].get("ref").is_none());
}
#[test]
fn extension_trace_deserializes() {
let v = json!({
"version": 3,
"recorded_at": "2026-07-21T08:00:00Z",
"started_at": "2026-07-21T07:59:00Z",
"purpose": "demo",
"stopped_by": "user_finish",
"entry": { "start_url": "https://example.com/editor" },
"recorder": { "bsk": "0.1.10", "vom": 1 },
"states": [
{ "id": "s1", "url": "https://example.com/editor", "body": "@vom 1" },
{ "id": "s2", "url": "https://example.com/p/99", "body": "@vom 1" }
],
"steps": [
{
"op": "fill",
"id": 1,
"state": "s1",
"result": { "state": "s1" },
"target": { "ref": "e1", "role": "textbox", "name": "标题" },
"value": "hello",
"commit": "blur"
},
{
"op": "click",
"id": 2,
"state": "s1",
"result": { "state": "s2" },
"target": { "ref": "e2", "role": "button", "name": "发布" }
}
]
});
let trace: TraceV3 = serde_json::from_value(v).unwrap();
assert_eq!(trace.version, 3);
assert_eq!(trace.states.len(), 2);
assert_eq!(trace.steps.len(), 2);
}
}