mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
22
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6300eda0aa | ||
|
|
87cdcfe5ca | ||
|
|
21476c3884 | ||
|
|
c2a225377b | ||
|
|
e64f4cac9f | ||
|
|
368507b07d | ||
|
|
b750eed5d4 | ||
|
|
3d159c0254 | ||
|
|
524516cb56 | ||
|
|
f98198949a | ||
|
|
d05279f991 | ||
|
|
2719e76438 | ||
|
|
dd9fc9edba | ||
|
|
005a6fd534 | ||
|
|
20554508db | ||
|
|
8352c3c67a | ||
|
|
4eedd98c1d | ||
|
|
6aa931b60a | ||
|
|
3df2738280 | ||
|
|
f86e506d47 | ||
|
|
45a246c41b | ||
|
|
7035e46024 |
@@ -1,9 +1,9 @@
|
||||
import {
|
||||
type CaptureTargetDescriptor,
|
||||
describeEventTarget,
|
||||
describeTarget,
|
||||
resolveClickableElement,
|
||||
resolveHoverElement,
|
||||
type TargetDescriptor,
|
||||
} from "@/lib/describe-target";
|
||||
import {
|
||||
evaluateHoverTrigger,
|
||||
@@ -27,7 +27,7 @@ import {
|
||||
type RecordStopAck,
|
||||
type RecordStopMessage,
|
||||
} from "@/lib/record-bridge";
|
||||
import { shouldRecordPress } from "@/lib/trace-reducer";
|
||||
import { shouldRecordPress } from "@/lib/recording/draft-policy";
|
||||
import {
|
||||
closestHoverSurfaceCandidate,
|
||||
collectHoverSurfaceStates,
|
||||
@@ -65,14 +65,14 @@ export interface RecordCaptureController {
|
||||
|
||||
interface FillSession {
|
||||
element: FillableElement;
|
||||
target: TargetDescriptor;
|
||||
target: CaptureTargetDescriptor;
|
||||
baselineValue: string;
|
||||
lastValue: string;
|
||||
}
|
||||
|
||||
interface HoverCandidate {
|
||||
element: Element;
|
||||
target: TargetDescriptor;
|
||||
target: CaptureTargetDescriptor;
|
||||
recordedAt: number;
|
||||
score: number;
|
||||
eligible: boolean;
|
||||
@@ -232,7 +232,10 @@ function collectHoverTriggerLabelText(root: Element): string {
|
||||
return normalizeLabelText(text);
|
||||
}
|
||||
|
||||
function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor {
|
||||
function compactHoverTargetName(
|
||||
el: Element,
|
||||
desc: CaptureTargetDescriptor,
|
||||
): CaptureTargetDescriptor {
|
||||
if (!desc.name) return desc;
|
||||
const fullText = normalizeLabelText(el.textContent ?? "");
|
||||
const compactName = collectHoverTriggerLabelText(el);
|
||||
@@ -250,7 +253,7 @@ function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDesc
|
||||
return desc;
|
||||
}
|
||||
|
||||
function isWeakHoverTarget(target: TargetDescriptor): boolean {
|
||||
function isWeakHoverTarget(target: CaptureTargetDescriptor): boolean {
|
||||
return !target.role && !target.name && target.tag === "div";
|
||||
}
|
||||
|
||||
@@ -261,9 +264,9 @@ function looksLikeAvatarElement(el: Element): boolean {
|
||||
|
||||
function normalizeHoverTarget(
|
||||
el: Element,
|
||||
desc: TargetDescriptor,
|
||||
desc: CaptureTargetDescriptor,
|
||||
decision: HoverTriggerDecision,
|
||||
): TargetDescriptor {
|
||||
): CaptureTargetDescriptor {
|
||||
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
|
||||
return { ...desc, name: "image" };
|
||||
}
|
||||
@@ -277,7 +280,7 @@ function normalizeHoverTarget(
|
||||
return desc;
|
||||
}
|
||||
|
||||
function hoverTriggerSignals(el: Element, desc: TargetDescriptor) {
|
||||
function hoverTriggerSignals(el: Element, desc: CaptureTargetDescriptor) {
|
||||
if (!(el instanceof HTMLElement)) return null;
|
||||
const style = hoverTriggerStyle(el);
|
||||
return {
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { CdpRunner } from "@/tools/shared";
|
||||
import { waitForDocumentSettled } from "../recording/document-settle";
|
||||
|
||||
function quietCdp(): { cdp: CdpRunner; send: ReturnType<typeof vi.fn> } {
|
||||
const send = vi.fn(async () => ({
|
||||
result: { value: { idleMs: 1_000, readyState: "complete" } },
|
||||
}));
|
||||
return {
|
||||
cdp: { send: send as unknown as CdpRunner["send"] },
|
||||
send,
|
||||
};
|
||||
}
|
||||
|
||||
describe("waitForDocumentSettled", () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("waits through the minimum observation floor before accepting a quiet page", async () => {
|
||||
vi.useFakeTimers();
|
||||
const { cdp, send } = quietCdp();
|
||||
|
||||
const settled = waitForDocumentSettled(cdp, { target: { tabId: 7 } });
|
||||
await vi.advanceTimersByTimeAsync(180);
|
||||
|
||||
await expect(settled).resolves.toBe("quiet");
|
||||
expect(send).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it("cancels before probing a superseded observation", async () => {
|
||||
vi.useFakeTimers();
|
||||
const { cdp, send } = quietCdp();
|
||||
|
||||
const controller = new AbortController();
|
||||
const settled = waitForDocumentSettled(
|
||||
cdp,
|
||||
{ target: { tabId: 7 } },
|
||||
{
|
||||
signal: controller.signal,
|
||||
},
|
||||
);
|
||||
controller.abort();
|
||||
await vi.advanceTimersByTimeAsync(60);
|
||||
|
||||
await expect(settled).resolves.toBe("cancelled");
|
||||
expect(send).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("probes a same-process iframe through its isolated execution context", async () => {
|
||||
vi.useFakeTimers();
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Page.createIsolatedWorld") return { executionContextId: 91 };
|
||||
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
|
||||
});
|
||||
const cdp: CdpRunner = { send: send as unknown as CdpRunner["send"] };
|
||||
|
||||
const settled = waitForDocumentSettled(cdp, {
|
||||
target: { tabId: 7 },
|
||||
frameId: "child-frame",
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(180);
|
||||
|
||||
await expect(settled).resolves.toBe("quiet");
|
||||
expect(send).toHaveBeenCalledWith(
|
||||
7,
|
||||
"Page.createIsolatedWorld",
|
||||
expect.objectContaining({ frameId: "child-frame" }),
|
||||
);
|
||||
expect(send).toHaveBeenCalledWith(
|
||||
7,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({ contextId: 91 }),
|
||||
);
|
||||
});
|
||||
|
||||
it("routes an OOPIF probe through its CDP target session", async () => {
|
||||
vi.useFakeTimers();
|
||||
const send = vi.fn();
|
||||
const sendToTarget = vi.fn(async (_target, method: string) => {
|
||||
if (method === "Page.createIsolatedWorld") return { executionContextId: 27 };
|
||||
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as unknown as CdpRunner["send"],
|
||||
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
};
|
||||
|
||||
const target = { tabId: 7, sessionId: "oopif-session" };
|
||||
const settled = waitForDocumentSettled(cdp, { target, frameId: "oopif-frame" });
|
||||
await vi.advanceTimersByTimeAsync(180);
|
||||
|
||||
await expect(settled).resolves.toBe("quiet");
|
||||
expect(send).not.toHaveBeenCalled();
|
||||
expect(sendToTarget).toHaveBeenCalledWith(
|
||||
target,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({ contextId: 27 }),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,141 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { ObservationNodeIndex } from "../recording/observation-capture";
|
||||
import { RecordingObservationSession } from "../recording/observation-session";
|
||||
import { RecordingStateRegistry } from "../recording/state-registry";
|
||||
import { buildTraceV3 } from "../recording/trace-builder-v3";
|
||||
import type { RecordingDraftStep } from "../recording/types";
|
||||
|
||||
const URL = "https://example.com/login";
|
||||
|
||||
function sessionWithInput(redactValues = false): RecordingObservationSession {
|
||||
const session = new RecordingObservationSession({ redactValues });
|
||||
const state = session.registry.register({
|
||||
url: URL,
|
||||
rawVomText: '@vom 1\ntextbox "Password" value="••••••" [ref=e1]',
|
||||
});
|
||||
session.cursor.lastSettled = {
|
||||
stateId: state.id,
|
||||
rootFrameId: "root",
|
||||
index: new ObservationNodeIndex({
|
||||
rootFrameId: "root",
|
||||
matchNodes: [
|
||||
{
|
||||
frameId: "root",
|
||||
backendNodeId: 42,
|
||||
tag: "input",
|
||||
rect: { x: 20, y: 40, w: 200, h: 30 },
|
||||
localRect: { x: 20, y: 40, w: 200, h: 30 },
|
||||
},
|
||||
],
|
||||
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
|
||||
}),
|
||||
url: URL,
|
||||
};
|
||||
return session;
|
||||
}
|
||||
|
||||
function finalizedFillBody(value: string, redactValues: boolean): string {
|
||||
const session = sessionWithInput(redactValues);
|
||||
const draft: RecordingDraftStep = {
|
||||
op: "fill",
|
||||
captureTarget: { tag: "input", role: "textbox", name: "Password" },
|
||||
value,
|
||||
targetHint: {
|
||||
geometry: { rect: { x: 20, y: 40, w: 200, h: 30 }, tag: "input" },
|
||||
},
|
||||
};
|
||||
session.bindDraft(draft, 1);
|
||||
draft.postStateId = draft.preStateId;
|
||||
return buildTraceV3({
|
||||
registry: session.registry,
|
||||
drafts: [draft],
|
||||
annotations: session.annotations,
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
stoppedBy: "user_finish",
|
||||
bskVersion: "test",
|
||||
}).states[0]!.body;
|
||||
}
|
||||
|
||||
describe("record observation annotations", () => {
|
||||
it("omits a fill literal when values are redacted", () => {
|
||||
const secret = "hunter2-private";
|
||||
const body = finalizedFillBody(secret, true);
|
||||
expect(body).toContain("step 1: fill");
|
||||
expect(body).not.toContain(secret);
|
||||
});
|
||||
|
||||
it("keeps ordinary fill details", () => {
|
||||
expect(finalizedFillBody("ordinary text", false)).toContain('step 1: fill: "ordinary text"');
|
||||
});
|
||||
|
||||
it("encodes title and URL so line breaks cannot corrupt state metadata", () => {
|
||||
const registry = new RecordingStateRegistry();
|
||||
const state = registry.register({
|
||||
url: "https://example.com/a\nb",
|
||||
title: "hello\nworld",
|
||||
rawVomText: "@vom 1",
|
||||
});
|
||||
const trace = buildTraceV3({
|
||||
registry,
|
||||
drafts: [{ op: "scroll", preStateId: state.id, postStateId: state.id }],
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
stoppedBy: "user_finish",
|
||||
bskVersion: "test",
|
||||
});
|
||||
expect(trace.states[0]?.body).toContain('url: "https://example.com/a\\nb"');
|
||||
expect(trace.states[0]?.body).toContain('title: "hello\\nworld"');
|
||||
});
|
||||
});
|
||||
|
||||
describe("recording state ownership", () => {
|
||||
it("deduplicates within one recording and isolates ids between recordings", () => {
|
||||
const first = new RecordingStateRegistry();
|
||||
const second = new RecordingStateRegistry();
|
||||
expect(first.register({ url: URL, rawVomText: "same" }).id).toBe("s1");
|
||||
expect(first.register({ url: URL, rawVomText: "same" }).id).toBe("s1");
|
||||
expect(second.register({ url: URL, rawVomText: "other" }).id).toBe("s1");
|
||||
});
|
||||
|
||||
it("enriches metadata when a deduplicated observation becomes more complete", () => {
|
||||
const registry = new RecordingStateRegistry();
|
||||
registry.register({ url: URL, rawVomText: "same" });
|
||||
const state = registry.register({
|
||||
url: URL,
|
||||
title: "Login",
|
||||
rawVomText: "same",
|
||||
truncated: true,
|
||||
});
|
||||
expect(state).toMatchObject({ id: "s1", title: "Login", truncated: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe("draft binding", () => {
|
||||
it("keeps capture semantics when no observation exists", () => {
|
||||
const session = new RecordingObservationSession();
|
||||
const draft: RecordingDraftStep = {
|
||||
op: "hover",
|
||||
captureTarget: { tag: "button", role: "button", name: "新建" },
|
||||
};
|
||||
session.bindDraft(draft, 1);
|
||||
expect(draft.matchedTarget).toEqual({ role: "button", name: "新建", unmatched: true });
|
||||
expect(draft.preStateId).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not bind an unmatched new action to a stale observation", () => {
|
||||
const session = sessionWithInput();
|
||||
const draft: RecordingDraftStep = {
|
||||
op: "click",
|
||||
captureTarget: { tag: "button", role: "button", name: "Confirm" },
|
||||
targetHint: {
|
||||
geometry: { rect: { x: 400, y: 300, w: 80, h: 30 }, tag: "button" },
|
||||
},
|
||||
};
|
||||
session.bindDraft(draft, 2, true);
|
||||
expect(draft.preStateId).toBeUndefined();
|
||||
expect("matchedTarget" in draft ? draft.matchedTarget : undefined).toEqual({
|
||||
role: "button",
|
||||
name: "Confirm",
|
||||
unmatched: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,136 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
|
||||
import { RecordingObservationSession } from "../recording/observation-session";
|
||||
import { SettleController } from "../recording/settle-controller";
|
||||
import type { RecordingDraftStep } from "../recording/types";
|
||||
|
||||
const OBSERVATION: RegisteredObservation = {
|
||||
stateId: "s-next",
|
||||
rootFrameId: "root",
|
||||
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
|
||||
url: "https://example.com/next",
|
||||
};
|
||||
|
||||
describe("SettleController", () => {
|
||||
afterEach(() => vi.useRealTimers());
|
||||
|
||||
it("aborts superseded capture work and lands the prior action on the next origin", async () => {
|
||||
vi.useFakeTimers();
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({
|
||||
result: { value: { idleMs: 1_000, readyState: "complete" } },
|
||||
})) as unknown as CdpRunner["send"],
|
||||
};
|
||||
const tabsApi: ChromeTabsApi = {
|
||||
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
|
||||
query: vi.fn(async () => []),
|
||||
};
|
||||
const session = new RecordingObservationSession();
|
||||
let firstSignal: AbortSignal | undefined;
|
||||
vi.spyOn(session, "capture")
|
||||
.mockImplementationOnce(async (_cdp, _tabs, _tabId, signal) => {
|
||||
firstSignal = signal;
|
||||
return new Promise<RegisteredObservation>((_resolve, reject) => {
|
||||
signal?.addEventListener(
|
||||
"abort",
|
||||
() => reject(new DOMException("observation aborted", "AbortError")),
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
})
|
||||
.mockResolvedValueOnce(OBSERVATION);
|
||||
|
||||
const drafts: RecordingDraftStep[] = [
|
||||
{ op: "click", captureTarget: { tag: "button", name: "First" }, preStateId: "s1" },
|
||||
];
|
||||
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
|
||||
controller.schedule(drafts, 0);
|
||||
await vi.advanceTimersByTimeAsync(180);
|
||||
|
||||
drafts.push({
|
||||
op: "click",
|
||||
captureTarget: { tag: "button", name: "Second" },
|
||||
preStateId: "s-next",
|
||||
});
|
||||
controller.schedule(drafts, 1);
|
||||
expect(firstSignal?.aborted).toBe(true);
|
||||
expect(drafts[0]?.postStateId).toBe("s-next");
|
||||
|
||||
await vi.advanceTimersByTimeAsync(500);
|
||||
await controller.flush();
|
||||
expect(drafts[1]?.postStateId).toBe("s-next");
|
||||
});
|
||||
|
||||
it("does not consume a newer redirect while reading the prior landing URL", async () => {
|
||||
vi.useFakeTimers();
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({
|
||||
result: { value: { idleMs: 1_000, readyState: "complete" } },
|
||||
})) as unknown as CdpRunner["send"],
|
||||
};
|
||||
let resolveFirstTab!: (tab: chrome.tabs.Tab) => void;
|
||||
const firstTab = new Promise<chrome.tabs.Tab>((resolve) => {
|
||||
resolveFirstTab = resolve;
|
||||
});
|
||||
const tabsApi: ChromeTabsApi = {
|
||||
get: vi
|
||||
.fn()
|
||||
.mockImplementationOnce(() => firstTab)
|
||||
.mockResolvedValue({ id: 7, url: OBSERVATION.url }),
|
||||
query: vi.fn(async () => []),
|
||||
};
|
||||
const session = new RecordingObservationSession();
|
||||
session.cursor.lastSettled = OBSERVATION;
|
||||
const drafts: RecordingDraftStep[] = [];
|
||||
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
|
||||
|
||||
controller.scheduleRedirect(drafts, "https://example.com/intermediate");
|
||||
await vi.advanceTimersByTimeAsync(180);
|
||||
expect(tabsApi.get).toHaveBeenCalledTimes(1);
|
||||
|
||||
controller.scheduleRedirect(drafts, OBSERVATION.url);
|
||||
resolveFirstTab({ id: 7, url: "https://example.com/intermediate" } as chrome.tabs.Tab);
|
||||
await vi.advanceTimersByTimeAsync(500);
|
||||
await controller.flushRedirects();
|
||||
|
||||
expect(drafts).toEqual([]);
|
||||
expect(controller.hasPending).toBe(false);
|
||||
});
|
||||
|
||||
it("settles an action in its own OOPIF document scope", async () => {
|
||||
vi.useFakeTimers();
|
||||
const send = vi.fn();
|
||||
const sendToTarget = vi.fn(async (_target, method: string) => {
|
||||
if (method === "Page.createIsolatedWorld") return { executionContextId: 39 };
|
||||
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as unknown as CdpRunner["send"],
|
||||
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
};
|
||||
const tabsApi: ChromeTabsApi = {
|
||||
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
|
||||
query: vi.fn(async () => []),
|
||||
};
|
||||
const session = new RecordingObservationSession();
|
||||
vi.spyOn(session, "capture").mockResolvedValue(OBSERVATION);
|
||||
const drafts: RecordingDraftStep[] = [
|
||||
{ op: "click", captureTarget: { tag: "button", name: "Inside frame" } },
|
||||
];
|
||||
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
|
||||
const target = { tabId: 7, sessionId: "oopif-session" };
|
||||
|
||||
controller.schedule(drafts, 0, { target, frameId: "oopif-frame" });
|
||||
await vi.advanceTimersByTimeAsync(180);
|
||||
await controller.flush();
|
||||
|
||||
expect(send).not.toHaveBeenCalled();
|
||||
expect(sendToTarget).toHaveBeenCalledWith(
|
||||
target,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({ contextId: 39 }),
|
||||
);
|
||||
expect(drafts[0]?.postStateId).toBe("s-next");
|
||||
});
|
||||
});
|
||||
+8
-8
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { appendRecordedPayload, observeRecordedNavigation } from "../recording-step-buffer";
|
||||
import { appendRecordedPayload, observeRecordedNavigation } from "../recording/step-buffer";
|
||||
|
||||
describe("recording-step-buffer", () => {
|
||||
it("stores semantic click without summary", () => {
|
||||
@@ -12,7 +12,7 @@ describe("recording-step-buffer", () => {
|
||||
expect(buffer.steps).toEqual([
|
||||
{
|
||||
op: "click",
|
||||
target: { tag: "button", role: "button", name: "发布" },
|
||||
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||
},
|
||||
]);
|
||||
expect(buffer.pendingNavigation).toBe(true);
|
||||
@@ -23,7 +23,7 @@ describe("recording-step-buffer", () => {
|
||||
steps: [
|
||||
{
|
||||
op: "click" as const,
|
||||
target: { tag: "button", role: "button", name: "发布" },
|
||||
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||
},
|
||||
],
|
||||
currentUrl: "https://example.com/a",
|
||||
@@ -34,8 +34,8 @@ describe("recording-step-buffer", () => {
|
||||
expect(buffer.steps).toEqual([
|
||||
{
|
||||
op: "click",
|
||||
target: { tag: "button", role: "button", name: "发布" },
|
||||
navigated_to: "https://example.com/b",
|
||||
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||
navigatedTo: "https://example.com/b",
|
||||
},
|
||||
]);
|
||||
expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
|
||||
@@ -46,7 +46,7 @@ describe("recording-step-buffer", () => {
|
||||
steps: [
|
||||
{
|
||||
op: "select" as const,
|
||||
target: { tag: "select", role: "combobox", name: "分类" },
|
||||
captureTarget: { tag: "select", role: "combobox", name: "分类" },
|
||||
values: ["tech"],
|
||||
},
|
||||
],
|
||||
@@ -58,9 +58,9 @@ describe("recording-step-buffer", () => {
|
||||
expect(buffer.steps).toEqual([
|
||||
{
|
||||
op: "select",
|
||||
target: { tag: "select", role: "combobox", name: "分类" },
|
||||
captureTarget: { tag: "select", role: "combobox", name: "分类" },
|
||||
values: ["tech"],
|
||||
navigated_to: "https://example.com/list?cat=tech",
|
||||
navigatedTo: "https://example.com/list?cat=tech",
|
||||
},
|
||||
]);
|
||||
});
|
||||
@@ -0,0 +1,127 @@
|
||||
import type { RenderedRef } from "@browser-skill/vom";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { CaptureVomMatchNode } from "@/tools/capture-vom-observation";
|
||||
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
|
||||
import { matchObservationTarget } from "../recording/target-matcher";
|
||||
|
||||
function node(backendNodeId: number, frameId: string, x = 10): CaptureVomMatchNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
frameId,
|
||||
tag: "button",
|
||||
rect: { x, y: 20, w: 100, h: 30 },
|
||||
localRect: { x, y: 20, w: 100, h: 30 },
|
||||
};
|
||||
}
|
||||
|
||||
function observation(
|
||||
matchNodes: CaptureVomMatchNode[],
|
||||
refs: RenderedRef[],
|
||||
): RegisteredObservation {
|
||||
return {
|
||||
stateId: "s1",
|
||||
rootFrameId: "root",
|
||||
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes, refs }),
|
||||
url: "https://example.com",
|
||||
};
|
||||
}
|
||||
|
||||
describe("matchObservationTarget", () => {
|
||||
it("indexes the safe geometry DTO without discarding frame-local geometry", () => {
|
||||
const geometry = node(42, "child");
|
||||
geometry.localRect = { x: 5, y: 6, w: 100, h: 30 };
|
||||
const ref: RenderedRef = {
|
||||
ref: "e1",
|
||||
backendNodeId: 42,
|
||||
frameId: "child",
|
||||
line: 1,
|
||||
};
|
||||
const candidate = observation([geometry], [ref]).index.candidates("child", "button")[0];
|
||||
|
||||
expect(candidate).toEqual({ frameId: "child", geometry, ref });
|
||||
});
|
||||
|
||||
it("matches a unique node using canonical top-level viewport geometry", () => {
|
||||
const target = matchObservationTarget({
|
||||
observation: observation(
|
||||
[node(42, "root")],
|
||||
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
|
||||
),
|
||||
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||
});
|
||||
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||
});
|
||||
|
||||
it("uses frame id with backend node id so sibling frames cannot collide", () => {
|
||||
const target = matchObservationTarget({
|
||||
observation: observation(
|
||||
[node(42, "left"), node(42, "right")],
|
||||
[
|
||||
{ ref: "e1", backendNodeId: 42, frameId: "left", line: 1 },
|
||||
{ ref: "e2", backendNodeId: 42, frameId: "right", line: 2 },
|
||||
],
|
||||
),
|
||||
hint: {
|
||||
frameId: "right",
|
||||
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
|
||||
},
|
||||
});
|
||||
expect(target.ref).toBe("e2");
|
||||
});
|
||||
|
||||
it("restricts a missing frame hint to the root frame", () => {
|
||||
const target = matchObservationTarget({
|
||||
observation: observation(
|
||||
[node(42, "child")],
|
||||
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
|
||||
),
|
||||
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||
fallback: { tag: "button", name: "发布" },
|
||||
});
|
||||
expect(target).toEqual({ name: "发布", unmatched: true });
|
||||
});
|
||||
|
||||
it("returns unmatched for ambiguous geometry", () => {
|
||||
const target = matchObservationTarget({
|
||||
observation: observation(
|
||||
[node(42, "root"), node(43, "root")],
|
||||
[
|
||||
{ ref: "e1", backendNodeId: 42, line: 1 },
|
||||
{ ref: "e2", backendNodeId: 43, line: 2 },
|
||||
],
|
||||
),
|
||||
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||
});
|
||||
expect(target.unmatched).toBe(true);
|
||||
});
|
||||
|
||||
it("uses semantics when geometry is unavailable without crossing frame boundaries", () => {
|
||||
const target = matchObservationTarget({
|
||||
observation: observation(
|
||||
[node(41, "root"), node(42, "child")],
|
||||
[
|
||||
{ ref: "e1", backendNodeId: 41, role: "button", name: "保存", line: 1 },
|
||||
{ ref: "e2", backendNodeId: 42, frameId: "child", role: "button", name: "保存", line: 2 },
|
||||
],
|
||||
),
|
||||
hint: { frameId: "child" },
|
||||
fallback: { tag: "button", role: "button", name: "保存" },
|
||||
});
|
||||
expect(target.ref).toBe("e2");
|
||||
});
|
||||
|
||||
it("uses semantics to disambiguate equal geometry in one frame", () => {
|
||||
const target = matchObservationTarget({
|
||||
observation: observation(
|
||||
[node(42, "root"), node(43, "root")],
|
||||
[
|
||||
{ ref: "e1", backendNodeId: 42, role: "button", name: "保存", line: 1 },
|
||||
{ ref: "e2", backendNodeId: 43, role: "button", name: "取消", line: 2 },
|
||||
],
|
||||
),
|
||||
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
|
||||
fallback: { tag: "button", role: "button", name: "取消" },
|
||||
});
|
||||
expect(target.ref).toBe("e2");
|
||||
});
|
||||
});
|
||||
+37
-27
@@ -1,6 +1,15 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { DraftTraceStep } from "@/transport/types";
|
||||
import { reduceTraceSteps, resolveTraceStartUrl, shouldRecordPress } from "../trace-reducer";
|
||||
import type { RecordingDraftStep } from "@/lib/recording/types";
|
||||
import { shouldRecordPress } from "../recording/draft-policy";
|
||||
import { buildTraceV2 } from "../recording/trace-reducer-v2";
|
||||
|
||||
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string) {
|
||||
return buildTraceV2({
|
||||
steps,
|
||||
startedAt: "2026-01-01T00:00:00.000Z",
|
||||
...(startUrl ? { startUrl } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
describe("shouldRecordPress", () => {
|
||||
it("keeps Enter and Escape", () => {
|
||||
@@ -19,35 +28,35 @@ describe("shouldRecordPress", () => {
|
||||
|
||||
describe("reduceTraceSteps", () => {
|
||||
it("builds steps with pages dictionary and page id references", () => {
|
||||
const drafts: DraftTraceStep[] = [
|
||||
const drafts: RecordingDraftStep[] = [
|
||||
{
|
||||
op: "navigate",
|
||||
url: "https://example.com/search?q=hello&utm_source=x",
|
||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
||||
pageUrl: "https://example.com/search?q=hello&utm_source=x",
|
||||
},
|
||||
{
|
||||
op: "fill",
|
||||
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
||||
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
||||
value: "browser skill",
|
||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
||||
pageUrl: "https://example.com/search?q=hello&utm_source=x",
|
||||
},
|
||||
{
|
||||
op: "press",
|
||||
key: "Enter",
|
||||
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
||||
navigated_to: "https://example.com/results/42",
|
||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
||||
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
||||
navigatedTo: "https://example.com/results/42",
|
||||
pageUrl: "https://example.com/search?q=hello&utm_source=x",
|
||||
},
|
||||
{
|
||||
op: "click",
|
||||
target: { tag: "button", role: "button", name: "发布" },
|
||||
navigated_to: "https://example.com/p/99",
|
||||
page_url: "https://example.com/results/42",
|
||||
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||
navigatedTo: "https://example.com/p/99",
|
||||
pageUrl: "https://example.com/results/42",
|
||||
},
|
||||
{
|
||||
op: "press",
|
||||
key: "a",
|
||||
page_url: "https://example.com/p/99",
|
||||
pageUrl: "https://example.com/p/99",
|
||||
},
|
||||
];
|
||||
|
||||
@@ -109,9 +118,9 @@ describe("reduceTraceSteps", () => {
|
||||
[
|
||||
{
|
||||
op: "fill",
|
||||
target: { tag: "input", role: "textbox", name: "Search query" },
|
||||
captureTarget: { tag: "input", role: "textbox", name: "Search query" },
|
||||
value: "",
|
||||
page_url: "https://example.com/search",
|
||||
pageUrl: "https://example.com/search",
|
||||
},
|
||||
],
|
||||
"https://example.com/search",
|
||||
@@ -125,16 +134,16 @@ describe("reduceTraceSteps", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("maps select navigated_to onto effect.navigated_to (page id)", () => {
|
||||
it("maps select navigatedTo onto effect.navigatedTo (page id)", () => {
|
||||
const { pages, steps } = reduceTraceSteps(
|
||||
[
|
||||
{
|
||||
op: "select",
|
||||
target: { tag: "select", role: "combobox", name: "分类" },
|
||||
captureTarget: { tag: "select", role: "combobox", name: "分类" },
|
||||
values: ["tech"],
|
||||
labels: ["技术"],
|
||||
navigated_to: "https://example.com/list?cat=tech",
|
||||
page_url: "https://example.com/list",
|
||||
navigatedTo: "https://example.com/list?cat=tech",
|
||||
pageUrl: "https://example.com/list",
|
||||
},
|
||||
],
|
||||
"https://example.com/list",
|
||||
@@ -156,13 +165,13 @@ describe("reduceTraceSteps", () => {
|
||||
[
|
||||
{
|
||||
op: "hover",
|
||||
target: { tag: "span", role: "button", name: "Account" },
|
||||
page_url: "https://example.com/app",
|
||||
captureTarget: { tag: "span", role: "button", name: "Account" },
|
||||
pageUrl: "https://example.com/app",
|
||||
},
|
||||
{
|
||||
op: "click",
|
||||
target: { tag: "a", role: "link", name: "Profile" },
|
||||
page_url: "https://example.com/app",
|
||||
captureTarget: { tag: "a", role: "link", name: "Profile" },
|
||||
pageUrl: "https://example.com/app",
|
||||
},
|
||||
],
|
||||
"https://example.com/app",
|
||||
@@ -177,10 +186,11 @@ describe("reduceTraceSteps", () => {
|
||||
|
||||
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
||||
expect(
|
||||
resolveTraceStartUrl(
|
||||
[{ op: "navigate", url: "https://example.com/other" }],
|
||||
"https://example.com/start",
|
||||
),
|
||||
buildTraceV2({
|
||||
steps: [{ op: "navigate", url: "https://example.com/other" }],
|
||||
startedAt: "2026-01-01T00:00:00.000Z",
|
||||
startUrl: "https://example.com/start",
|
||||
}).entry.start_url,
|
||||
).toBe("https://example.com/start");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,62 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { TRACE_VERSION_V3, VOM_FORMAT_VERSION } from "@/transport/types";
|
||||
import { RecordingStateRegistry } from "../recording/state-registry";
|
||||
import { buildTraceV3 } from "../recording/trace-builder-v3";
|
||||
import { reduceTraceStepsV3 } from "../recording/trace-reducer-v3";
|
||||
import type { RecordingDraftStep } from "../recording/types";
|
||||
|
||||
describe("trace reducer v3", () => {
|
||||
it("collapses redirect hops while retaining draft-to-step identity", () => {
|
||||
const drafts: RecordingDraftStep[] = [
|
||||
{ op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" },
|
||||
{
|
||||
op: "navigate",
|
||||
url: "https://example.com/final",
|
||||
transitionQualifiers: ["server_redirect"],
|
||||
preStateId: "s2",
|
||||
postStateId: "s3",
|
||||
},
|
||||
];
|
||||
const before = structuredClone(drafts);
|
||||
const reduced = reduceTraceStepsV3(drafts);
|
||||
|
||||
expect(drafts).toEqual(before);
|
||||
expect(reduced.steps).toEqual([
|
||||
expect.objectContaining({
|
||||
id: 1,
|
||||
op: "navigate",
|
||||
state: "s1",
|
||||
to: "https://example.com/final",
|
||||
result: { state: "s3" },
|
||||
}),
|
||||
]);
|
||||
expect(reduced.stepIdByDraftId.get(1)).toBe(1);
|
||||
expect(reduced.stepIdByDraftId.get(2)).toBe(1);
|
||||
});
|
||||
|
||||
it("builds the wire model from protocol constants", () => {
|
||||
const registry = new RecordingStateRegistry();
|
||||
const state = registry.register({ url: "https://example.com", rawVomText: "@vom 1" });
|
||||
const trace = buildTraceV3({
|
||||
registry,
|
||||
drafts: [
|
||||
{
|
||||
op: "click",
|
||||
captureTarget: { tag: "button", role: "button", name: "Save" },
|
||||
preStateId: state.id,
|
||||
postStateId: state.id,
|
||||
},
|
||||
],
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
stoppedBy: "user_finish",
|
||||
bskVersion: "test",
|
||||
});
|
||||
|
||||
expect(trace.version).toBe(TRACE_VERSION_V3);
|
||||
expect(trace.recorder.vom).toBe(VOM_FORMAT_VERSION);
|
||||
expect(trace.steps[0]).toMatchObject({
|
||||
op: "click",
|
||||
target: { role: "button", name: "Save", unmatched: true },
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,12 +1,13 @@
|
||||
/**
|
||||
* Build a semantic TargetDescriptor for an interacted element.
|
||||
* Build a semantic capture descriptor for an interacted element.
|
||||
*
|
||||
* Trace steps are an LLM *textbook*: each click must say what to look for
|
||||
* on screen (usually a short visible name). Tag-only noise like
|
||||
* `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
|
||||
*/
|
||||
|
||||
export interface TargetDescriptor {
|
||||
/** Content-script capture descriptor before VOM geometric matching. */
|
||||
export interface CaptureTargetDescriptor {
|
||||
role?: string;
|
||||
name?: string;
|
||||
tag: string;
|
||||
@@ -247,7 +248,7 @@ export function resolveHoverElement(target: Element): Element | null {
|
||||
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
||||
* Recording “点击div” with no name fails this bar.
|
||||
*/
|
||||
export function isMeaningfulClickTarget(target: TargetDescriptor): boolean {
|
||||
export function isMeaningfulClickTarget(target: CaptureTargetDescriptor): boolean {
|
||||
const name = target.name?.trim();
|
||||
if (name && isActionableLabel(name)) return true;
|
||||
if (
|
||||
@@ -323,7 +324,7 @@ function nearbyLabelText(el: Element): string | undefined {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function describeTarget(el: Element): TargetDescriptor {
|
||||
export function describeTarget(el: Element): CaptureTargetDescriptor {
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const role = inferRole(el);
|
||||
const name = accessibleName(el);
|
||||
@@ -350,7 +351,7 @@ export function describeTarget(el: Element): TargetDescriptor {
|
||||
};
|
||||
}
|
||||
|
||||
export function describeEventTarget(target: EventTarget | null): TargetDescriptor | null {
|
||||
export function describeEventTarget(target: EventTarget | null): CaptureTargetDescriptor | null {
|
||||
if (!(target instanceof Element)) return null;
|
||||
const clickable = resolveClickableElement(target);
|
||||
if (!clickable) return null;
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
* service worker and a tab's content script.
|
||||
*/
|
||||
|
||||
import type { TargetDescriptor } from "./describe-target";
|
||||
import type { CaptureTargetDescriptor } from "./describe-target";
|
||||
|
||||
export const RECORD_START = "bsk-record-start";
|
||||
export const RECORD_STEP = "bsk-record-step";
|
||||
@@ -42,7 +42,7 @@ export interface RecordStartMessage {
|
||||
|
||||
export interface RecordStepPayload {
|
||||
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
|
||||
target?: TargetDescriptor;
|
||||
target?: CaptureTargetDescriptor;
|
||||
value?: string;
|
||||
key?: string;
|
||||
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import { type CdpRunner, sendToCdpTarget } from "@/tools/shared";
|
||||
|
||||
const SETTLE_MIN_MS = 150;
|
||||
const SETTLE_QUIET_MS = 250;
|
||||
const SETTLE_MAX_MS = 2_000;
|
||||
const SETTLE_POLL_MS = 60;
|
||||
const SETTLE_WORLD_NAME = "__bsk_record_settle__";
|
||||
|
||||
const QUIET_PROBE = `(() => {
|
||||
const scope = window;
|
||||
let probe = scope.__bskRecordQuiet;
|
||||
if (!probe) {
|
||||
probe = { changedAt: Date.now() };
|
||||
const observer = new MutationObserver(() => {
|
||||
probe.changedAt = Date.now();
|
||||
});
|
||||
observer.observe(document.documentElement, {
|
||||
subtree: true,
|
||||
childList: true,
|
||||
attributes: true,
|
||||
characterData: true,
|
||||
});
|
||||
scope.__bskRecordQuiet = probe;
|
||||
}
|
||||
return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState };
|
||||
})()`;
|
||||
|
||||
export interface DocumentSettleScope {
|
||||
target: CdpTarget;
|
||||
/** Omit only for the target's root document. */
|
||||
frameId?: string;
|
||||
}
|
||||
|
||||
type SettleOutcome = "quiet" | "timeout" | "cancelled";
|
||||
|
||||
interface QuietProbe {
|
||||
idleMs: number;
|
||||
readyState: string;
|
||||
}
|
||||
|
||||
interface ProbeContext {
|
||||
executionContextId?: number;
|
||||
}
|
||||
|
||||
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
if (signal?.aborted) return resolve();
|
||||
const timer = setTimeout(resolve, ms);
|
||||
signal?.addEventListener(
|
||||
"abort",
|
||||
() => {
|
||||
clearTimeout(timer);
|
||||
resolve();
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async function executionContextForScope(
|
||||
cdp: CdpRunner,
|
||||
scope: DocumentSettleScope,
|
||||
): Promise<number | undefined> {
|
||||
if (!scope.frameId) return undefined;
|
||||
const result = await sendToCdpTarget<{ executionContextId?: number }>(
|
||||
cdp,
|
||||
scope.target,
|
||||
"Page.createIsolatedWorld",
|
||||
{
|
||||
frameId: scope.frameId,
|
||||
worldName: SETTLE_WORLD_NAME,
|
||||
grantUniveralAccess: false,
|
||||
},
|
||||
);
|
||||
return result.executionContextId;
|
||||
}
|
||||
|
||||
async function readQuietProbe(
|
||||
cdp: CdpRunner,
|
||||
scope: DocumentSettleScope,
|
||||
context: ProbeContext,
|
||||
): Promise<QuietProbe | null> {
|
||||
try {
|
||||
if (scope.frameId && context.executionContextId === undefined) {
|
||||
context.executionContextId = await executionContextForScope(cdp, scope);
|
||||
if (context.executionContextId === undefined) return null;
|
||||
}
|
||||
const reply = await sendToCdpTarget<{ result?: { value?: unknown } }>(
|
||||
cdp,
|
||||
scope.target,
|
||||
"Runtime.evaluate",
|
||||
{
|
||||
expression: QUIET_PROBE,
|
||||
returnByValue: true,
|
||||
...(context.executionContextId !== undefined
|
||||
? { contextId: context.executionContextId }
|
||||
: {}),
|
||||
},
|
||||
);
|
||||
const value = reply.result?.value;
|
||||
if (!value || typeof value !== "object") return null;
|
||||
const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown };
|
||||
if (typeof idleMs !== "number" || typeof readyState !== "string") return null;
|
||||
return { idleMs, readyState };
|
||||
} catch {
|
||||
context.executionContextId = undefined;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function waitForDocumentSettled(
|
||||
cdp: CdpRunner,
|
||||
scope: DocumentSettleScope,
|
||||
options: { signal?: AbortSignal } = {},
|
||||
): Promise<SettleOutcome> {
|
||||
const startedAt = Date.now();
|
||||
const floor = startedAt + SETTLE_MIN_MS;
|
||||
const deadline = startedAt + SETTLE_MAX_MS;
|
||||
const context: ProbeContext = {};
|
||||
|
||||
for (;;) {
|
||||
if (options.signal?.aborted) return "cancelled";
|
||||
await sleep(SETTLE_POLL_MS, options.signal);
|
||||
if (options.signal?.aborted) return "cancelled";
|
||||
|
||||
const probe = await readQuietProbe(cdp, scope, context);
|
||||
const now = Date.now();
|
||||
if (now < floor) continue;
|
||||
if (now >= deadline) return "timeout";
|
||||
if (!probe || probe.readyState === "loading") continue;
|
||||
if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import type { RecordingDraftStep } from "./types";
|
||||
|
||||
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
||||
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
||||
|
||||
export function shouldRecordPress(
|
||||
key: string,
|
||||
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
|
||||
): boolean {
|
||||
if (MODIFIER_ONLY_KEYS.has(key)) return false;
|
||||
const mods = modifiers ?? [];
|
||||
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
||||
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
||||
if (key === "Enter" || key === "Escape") return true;
|
||||
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
||||
return false;
|
||||
}
|
||||
|
||||
export function shouldIncludeDraft(step: RecordingDraftStep): boolean {
|
||||
return step.op !== "press" || shouldRecordPress(step.key, step.modifiers);
|
||||
}
|
||||
|
||||
export function resolveDraftStartUrl(
|
||||
drafts: RecordingDraftStep[],
|
||||
explicitStartUrl?: string,
|
||||
fallbackUrl?: string,
|
||||
): string {
|
||||
if (explicitStartUrl) return explicitStartUrl;
|
||||
const navigation = drafts.find(
|
||||
(step): step is Extract<RecordingDraftStep, { op: "navigate" }> => step.op === "navigate",
|
||||
);
|
||||
if (navigation) return navigation.url;
|
||||
return drafts.find((step) => step.pageUrl)?.pageUrl ?? fallbackUrl ?? "about:blank";
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
import type { RenderedRef } from "@browser-skill/vom";
|
||||
import {
|
||||
type CaptureVomMatchNode,
|
||||
type CaptureVomObservationResult,
|
||||
captureVomObservation,
|
||||
} from "@/tools/capture-vom-observation";
|
||||
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||
|
||||
export interface IndexedObservationNode {
|
||||
frameId: string;
|
||||
geometry: CaptureVomMatchNode;
|
||||
ref?: RenderedRef;
|
||||
}
|
||||
|
||||
export interface CapturedRecordingObservation {
|
||||
rootFrameId: string;
|
||||
index: ObservationNodeIndex;
|
||||
url: string;
|
||||
title?: string;
|
||||
vomText: string;
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
export interface RegisteredObservation {
|
||||
stateId: string;
|
||||
rootFrameId: string;
|
||||
index: ObservationNodeIndex;
|
||||
url: string;
|
||||
}
|
||||
|
||||
function nodeKey(frameId: string, backendNodeId: number): string {
|
||||
return `${frameId}:${backendNodeId}`;
|
||||
}
|
||||
|
||||
function frameTagKey(frameId: string, tag: string): string {
|
||||
return `${frameId}:${tag.toLowerCase()}`;
|
||||
}
|
||||
|
||||
export class ObservationNodeIndex {
|
||||
readonly #nodesByFrameTag = new Map<string, IndexedObservationNode[]>();
|
||||
readonly #refById = new Map<string, RenderedRef>();
|
||||
readonly #refsByFrame = new Map<string, RenderedRef[]>();
|
||||
|
||||
constructor(input: Pick<CaptureVomObservationResult, "rootFrameId" | "matchNodes" | "refs">) {
|
||||
const refByNode = new Map<string, RenderedRef>();
|
||||
for (const ref of input.refs) {
|
||||
const frameId = ref.frameId ?? input.rootFrameId;
|
||||
refByNode.set(nodeKey(frameId, ref.backendNodeId), ref);
|
||||
this.#refById.set(ref.ref, ref);
|
||||
const frameRefs = this.#refsByFrame.get(frameId) ?? [];
|
||||
frameRefs.push(ref);
|
||||
this.#refsByFrame.set(frameId, frameRefs);
|
||||
}
|
||||
for (const geometry of input.matchNodes) {
|
||||
const { frameId } = geometry;
|
||||
const entry = {
|
||||
frameId,
|
||||
geometry,
|
||||
ref: refByNode.get(nodeKey(frameId, geometry.backendNodeId)),
|
||||
};
|
||||
const key = frameTagKey(frameId, geometry.tag);
|
||||
const bucket = this.#nodesByFrameTag.get(key) ?? [];
|
||||
bucket.push(entry);
|
||||
this.#nodesByFrameTag.set(key, bucket);
|
||||
}
|
||||
}
|
||||
|
||||
candidates(frameId: string, tag: string): readonly IndexedObservationNode[] {
|
||||
return this.#nodesByFrameTag.get(frameTagKey(frameId, tag)) ?? [];
|
||||
}
|
||||
|
||||
ref(refId: string): RenderedRef | undefined {
|
||||
return this.#refById.get(refId);
|
||||
}
|
||||
|
||||
refs(frameId: string): readonly RenderedRef[] {
|
||||
return this.#refsByFrame.get(frameId) ?? [];
|
||||
}
|
||||
}
|
||||
|
||||
async function readTabMeta(
|
||||
tabsApi: ChromeTabsApi,
|
||||
tabId: number,
|
||||
): Promise<{ url: string; title?: string }> {
|
||||
try {
|
||||
const tab = await tabsApi.get(tabId);
|
||||
return { url: tab.url ?? "about:blank", title: tab.title };
|
||||
} catch {
|
||||
return { url: "about:blank" };
|
||||
}
|
||||
}
|
||||
|
||||
export async function captureRecordingObservation(input: {
|
||||
cdp: CdpRunner;
|
||||
tabsApi: ChromeTabsApi;
|
||||
tabId: number;
|
||||
maxTokens: number;
|
||||
redactValues: boolean;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<CapturedRecordingObservation> {
|
||||
const { url, title } = await readTabMeta(input.tabsApi, input.tabId);
|
||||
const captured = await captureVomObservation(input.cdp, input.tabId, url, {
|
||||
maxTokens: input.maxTokens,
|
||||
redactValues: input.redactValues,
|
||||
conditionalSurfaceProbe: false,
|
||||
signal: input.signal,
|
||||
});
|
||||
return {
|
||||
rootFrameId: captured.rootFrameId,
|
||||
index: new ObservationNodeIndex(captured),
|
||||
url,
|
||||
title,
|
||||
vomText: captured.text,
|
||||
truncated: captured.truncated,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||
import { captureRecordingObservation, type RegisteredObservation } from "./observation-capture";
|
||||
import { RecordingStateRegistry } from "./state-registry";
|
||||
import { matchObservationTarget, unmatchedTarget } from "./target-matcher";
|
||||
import type { RecordingDraftStep, StepAnnotation, TargetedRecordingDraft } from "./types";
|
||||
|
||||
const DEFAULT_MAX_PAGE_TOKENS = 3_000;
|
||||
const MIN_CAPTURE_INTERVAL_MS = 200;
|
||||
|
||||
export interface TabObservationCursor {
|
||||
lastSettled: RegisteredObservation | null;
|
||||
lastCaptureAt: number;
|
||||
}
|
||||
|
||||
function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
if (signal?.aborted) return resolve();
|
||||
const timer = setTimeout(resolve, ms);
|
||||
signal?.addEventListener(
|
||||
"abort",
|
||||
() => {
|
||||
clearTimeout(timer);
|
||||
resolve();
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function isTargeted(draft: RecordingDraftStep): draft is TargetedRecordingDraft {
|
||||
return draft.op !== "navigate" && draft.op !== "scroll";
|
||||
}
|
||||
|
||||
export class RecordingObservationSession {
|
||||
readonly registry: RecordingStateRegistry;
|
||||
readonly cursor: TabObservationCursor;
|
||||
readonly annotations: StepAnnotation[];
|
||||
readonly #maxTokens: number;
|
||||
readonly #redactValues: boolean;
|
||||
|
||||
constructor(
|
||||
options: {
|
||||
registry?: RecordingStateRegistry;
|
||||
cursor?: TabObservationCursor;
|
||||
annotations?: StepAnnotation[];
|
||||
maxTokens?: number;
|
||||
redactValues?: boolean;
|
||||
} = {},
|
||||
) {
|
||||
this.registry = options.registry ?? new RecordingStateRegistry();
|
||||
this.cursor = options.cursor ?? { lastSettled: null, lastCaptureAt: 0 };
|
||||
this.annotations = options.annotations ?? [];
|
||||
this.#maxTokens = options.maxTokens ?? DEFAULT_MAX_PAGE_TOKENS;
|
||||
this.#redactValues = options.redactValues ?? false;
|
||||
}
|
||||
|
||||
async capture(
|
||||
cdp: CdpRunner,
|
||||
tabsApi: ChromeTabsApi,
|
||||
tabId: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<RegisteredObservation> {
|
||||
const waitMs = Math.max(0, MIN_CAPTURE_INTERVAL_MS - (Date.now() - this.cursor.lastCaptureAt));
|
||||
if (waitMs > 0) await abortableDelay(waitMs, signal);
|
||||
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
||||
const captured = await captureRecordingObservation({
|
||||
cdp,
|
||||
tabsApi,
|
||||
tabId,
|
||||
maxTokens: this.#maxTokens,
|
||||
redactValues: this.#redactValues,
|
||||
signal,
|
||||
});
|
||||
const state = this.registry.register({
|
||||
url: captured.url,
|
||||
title: captured.title,
|
||||
rawVomText: captured.vomText,
|
||||
truncated: captured.truncated,
|
||||
});
|
||||
const observation: RegisteredObservation = {
|
||||
stateId: state.id,
|
||||
rootFrameId: captured.rootFrameId,
|
||||
index: captured.index,
|
||||
url: captured.url,
|
||||
};
|
||||
this.cursor.lastSettled = observation;
|
||||
this.cursor.lastCaptureAt = Date.now();
|
||||
return observation;
|
||||
}
|
||||
|
||||
bindDraft(draft: RecordingDraftStep, draftId: number, previousActionPending = false): void {
|
||||
const observation = this.cursor.lastSettled;
|
||||
if (isTargeted(draft)) {
|
||||
draft.matchedTarget = observation
|
||||
? matchObservationTarget({
|
||||
observation,
|
||||
hint: draft.targetHint,
|
||||
fallback: draft.captureTarget,
|
||||
})
|
||||
: unmatchedTarget(draft.captureTarget);
|
||||
}
|
||||
if (!observation) return;
|
||||
|
||||
const unmatched = isTargeted(draft) && draft.matchedTarget?.unmatched === true;
|
||||
if (previousActionPending && unmatched) return;
|
||||
draft.preStateId = observation.stateId;
|
||||
this.registry.markStep(observation.stateId, draftId);
|
||||
|
||||
if (!isTargeted(draft) || !draft.matchedTarget?.ref) return;
|
||||
const ref = observation.index.ref(draft.matchedTarget.ref);
|
||||
if (!ref) return;
|
||||
this.annotations.push({
|
||||
draftId,
|
||||
op: draft.op,
|
||||
line: ref.line,
|
||||
stateId: observation.stateId,
|
||||
...(draft.op === "fill" && !this.#redactValues
|
||||
? { detail: JSON.stringify(draft.value) }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||
import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
|
||||
import { RecordingObservationSession } from "./observation-session";
|
||||
import type { RecordingDraftStep } from "./types";
|
||||
|
||||
interface PendingSettle {
|
||||
abort: AbortController;
|
||||
scope: DocumentSettleScope;
|
||||
}
|
||||
|
||||
interface PendingRedirect {
|
||||
url: string;
|
||||
abort: AbortController;
|
||||
}
|
||||
|
||||
const CAPTURE_RETRY_DELAY_MS = 250;
|
||||
|
||||
function delay(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
if (signal?.aborted) return resolve();
|
||||
const timer = setTimeout(resolve, ms);
|
||||
signal?.addEventListener(
|
||||
"abort",
|
||||
() => {
|
||||
clearTimeout(timer);
|
||||
resolve();
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function isAbortError(error: unknown): boolean {
|
||||
return (
|
||||
typeof error === "object" &&
|
||||
error !== null &&
|
||||
(error as { name?: string }).name === "AbortError"
|
||||
);
|
||||
}
|
||||
|
||||
export function inferMissingPostStates(drafts: RecordingDraftStep[]): void {
|
||||
for (let index = 0; index < drafts.length - 1; index += 1) {
|
||||
const draft = drafts[index];
|
||||
const next = drafts[index + 1];
|
||||
if (draft && next && !draft.postStateId && next.preStateId) {
|
||||
draft.postStateId = next.preStateId;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class SettleController {
|
||||
readonly #session: RecordingObservationSession;
|
||||
readonly #cdp: CdpRunner;
|
||||
readonly #tabsApi: ChromeTabsApi;
|
||||
readonly #tabId: number;
|
||||
readonly #rootScope: DocumentSettleScope;
|
||||
readonly #pending = new Map<number, PendingSettle>();
|
||||
#queue = Promise.resolve();
|
||||
#pendingRedirect: PendingRedirect | null = null;
|
||||
#redirectQueue = Promise.resolve();
|
||||
|
||||
constructor(input: {
|
||||
session: RecordingObservationSession;
|
||||
cdp: CdpRunner;
|
||||
tabsApi: ChromeTabsApi;
|
||||
tabId: number;
|
||||
}) {
|
||||
this.#session = input.session;
|
||||
this.#cdp = input.cdp;
|
||||
this.#tabsApi = input.tabsApi;
|
||||
this.#tabId = input.tabId;
|
||||
this.#rootScope = { target: { tabId: input.tabId } };
|
||||
}
|
||||
|
||||
get hasPending(): boolean {
|
||||
return this.#pending.size > 0 || this.#pendingRedirect !== null;
|
||||
}
|
||||
|
||||
async #captureWithRetry(signal?: AbortSignal) {
|
||||
try {
|
||||
return await this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
|
||||
} catch (error) {
|
||||
if (signal?.aborted) throw error;
|
||||
await delay(CAPTURE_RETRY_DELAY_MS, signal);
|
||||
return this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
|
||||
}
|
||||
}
|
||||
|
||||
schedule(
|
||||
drafts: RecordingDraftStep[],
|
||||
draftIndex: number,
|
||||
scope: DocumentSettleScope = this.#rootScope,
|
||||
): void {
|
||||
if (scope.target.tabId !== this.#tabId) {
|
||||
throw new Error(
|
||||
`settle scope tab ${scope.target.tabId} does not belong to tab ${this.#tabId}`,
|
||||
);
|
||||
}
|
||||
const landing = drafts[draftIndex]?.preStateId;
|
||||
for (const [index, pending] of this.#pending) {
|
||||
if (index >= draftIndex) continue;
|
||||
pending.abort.abort();
|
||||
this.#pending.delete(index);
|
||||
if (landing && drafts[index] && !drafts[index].postStateId) {
|
||||
drafts[index].postStateId = landing;
|
||||
}
|
||||
}
|
||||
|
||||
this.#pending.get(draftIndex)?.abort.abort();
|
||||
const pending = { abort: new AbortController(), scope };
|
||||
this.#pending.set(draftIndex, pending);
|
||||
const task = async () => {
|
||||
if (pending.abort.signal.aborted) return;
|
||||
try {
|
||||
const outcome = await waitForDocumentSettled(this.#cdp, pending.scope, {
|
||||
signal: pending.abort.signal,
|
||||
});
|
||||
if (outcome === "cancelled") return;
|
||||
const observation = await this.#captureWithRetry(pending.abort.signal);
|
||||
if (!pending.abort.signal.aborted && drafts[draftIndex]) {
|
||||
drafts[draftIndex].postStateId = observation.stateId;
|
||||
}
|
||||
} catch (error) {
|
||||
if (!isAbortError(error)) {
|
||||
console.warn(
|
||||
`[bsk record] post-action observation failed for step ${draftIndex + 1}`,
|
||||
error,
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
if (this.#pending.get(draftIndex) === pending) this.#pending.delete(draftIndex);
|
||||
}
|
||||
};
|
||||
this.#queue = this.#queue.then(task, task).catch(() => {});
|
||||
}
|
||||
|
||||
cancel(): void {
|
||||
for (const pending of this.#pending.values()) pending.abort.abort();
|
||||
this.#pending.clear();
|
||||
this.clearRedirect();
|
||||
}
|
||||
|
||||
async flush(): Promise<void> {
|
||||
for (let round = 0; round < 10; round += 1) {
|
||||
const current = this.#queue;
|
||||
await current;
|
||||
if (current === this.#queue) return;
|
||||
}
|
||||
console.warn("[bsk record] settle queue kept growing; using completed observations");
|
||||
}
|
||||
|
||||
async settleTrailing(drafts: RecordingDraftStep[]): Promise<void> {
|
||||
const trailing: RecordingDraftStep[] = [];
|
||||
for (let index = drafts.length - 1; index >= 0; index -= 1) {
|
||||
const draft = drafts[index];
|
||||
if (!draft || draft.postStateId) break;
|
||||
trailing.push(draft);
|
||||
}
|
||||
if (trailing.length === 0) return;
|
||||
try {
|
||||
const observation = await this.#captureWithRetry();
|
||||
for (const draft of trailing) draft.postStateId = observation.stateId;
|
||||
} catch (error) {
|
||||
console.warn("[bsk record] final observation at stop failed", error);
|
||||
}
|
||||
}
|
||||
|
||||
clearRedirect(): void {
|
||||
this.#pendingRedirect?.abort.abort();
|
||||
this.#pendingRedirect = null;
|
||||
}
|
||||
|
||||
scheduleRedirect(drafts: RecordingDraftStep[], url: string): void {
|
||||
this.#pendingRedirect?.abort.abort();
|
||||
const pending: PendingRedirect = { url, abort: new AbortController() };
|
||||
this.#pendingRedirect = pending;
|
||||
const task = () => this.#settleRedirect(drafts, pending);
|
||||
this.#redirectQueue = this.#redirectQueue.then(task, task).catch(() => {});
|
||||
}
|
||||
|
||||
async #settleRedirect(drafts: RecordingDraftStep[], pending: PendingRedirect): Promise<void> {
|
||||
const outcome = await waitForDocumentSettled(this.#cdp, this.#rootScope, {
|
||||
signal: pending.abort.signal,
|
||||
});
|
||||
if (outcome === "cancelled" || this.#pendingRedirect !== pending) return;
|
||||
|
||||
let finalUrl = pending.url;
|
||||
try {
|
||||
finalUrl = (await this.#tabsApi.get(this.#tabId)).url || finalUrl;
|
||||
} catch {
|
||||
// The navigation event URL remains the best available destination.
|
||||
}
|
||||
if (this.#pendingRedirect !== pending) return;
|
||||
this.#pendingRedirect = null;
|
||||
if (
|
||||
!finalUrl ||
|
||||
finalUrl === "about:blank" ||
|
||||
this.#session.cursor.lastSettled?.url === finalUrl
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const last = drafts[drafts.length - 1];
|
||||
if (last?.op === "navigate" && last.url === finalUrl) {
|
||||
if (!last.postStateId) this.schedule(drafts, drafts.length - 1);
|
||||
await this.flush();
|
||||
return;
|
||||
}
|
||||
|
||||
const draft: RecordingDraftStep = {
|
||||
op: "navigate",
|
||||
url: finalUrl,
|
||||
pageUrl: finalUrl,
|
||||
cause: "browser",
|
||||
preStateId: this.#session.cursor.lastSettled?.stateId,
|
||||
};
|
||||
drafts.push(draft);
|
||||
const draftIndex = drafts.length - 1;
|
||||
if (draft.preStateId) this.#session.registry.markStep(draft.preStateId, draftIndex + 1);
|
||||
this.schedule(drafts, draftIndex);
|
||||
await this.flush();
|
||||
}
|
||||
|
||||
async flushRedirects(): Promise<void> {
|
||||
for (let round = 0; round < 10; round += 1) {
|
||||
const current = this.#redirectQueue;
|
||||
await current;
|
||||
if (current === this.#redirectQueue) return;
|
||||
}
|
||||
console.warn("[bsk record] redirect queue kept growing; using the final completed landing");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
export interface RecordedStateEntry {
|
||||
id: string;
|
||||
url: string;
|
||||
title?: string;
|
||||
rawVomText: string;
|
||||
truncated: boolean;
|
||||
stepsHere: number[];
|
||||
}
|
||||
|
||||
function stateIdentity(url: string, body: string): string {
|
||||
return `${url}\0${body}`;
|
||||
}
|
||||
|
||||
export class RecordingStateRegistry {
|
||||
readonly #entriesById = new Map<string, RecordedStateEntry>();
|
||||
readonly #idByIdentity = new Map<string, string>();
|
||||
#nextId = 1;
|
||||
|
||||
register(input: {
|
||||
url: string;
|
||||
title?: string;
|
||||
rawVomText: string;
|
||||
truncated?: boolean;
|
||||
}): RecordedStateEntry {
|
||||
const identity = stateIdentity(input.url, input.rawVomText);
|
||||
const existingId = this.#idByIdentity.get(identity);
|
||||
if (existingId) {
|
||||
const existing = this.#entriesById.get(existingId)!;
|
||||
if (!existing.title && input.title) existing.title = input.title;
|
||||
if (input.truncated) existing.truncated = true;
|
||||
return existing;
|
||||
}
|
||||
|
||||
const entry: RecordedStateEntry = {
|
||||
id: `s${this.#nextId}`,
|
||||
url: input.url,
|
||||
...(input.title ? { title: input.title } : {}),
|
||||
rawVomText: input.rawVomText,
|
||||
truncated: input.truncated ?? false,
|
||||
stepsHere: [],
|
||||
};
|
||||
this.#nextId += 1;
|
||||
this.#entriesById.set(entry.id, entry);
|
||||
this.#idByIdentity.set(identity, entry.id);
|
||||
return entry;
|
||||
}
|
||||
|
||||
values(): RecordedStateEntry[] {
|
||||
return [...this.#entriesById.values()];
|
||||
}
|
||||
|
||||
markStep(stateId: string, draftId: number): void {
|
||||
const entry = this.#entriesById.get(stateId);
|
||||
if (entry && !entry.stepsHere.includes(draftId)) entry.stepsHere.push(draftId);
|
||||
}
|
||||
}
|
||||
+17
-17
@@ -1,8 +1,8 @@
|
||||
import type { DraftTraceStep } from "@/transport/types";
|
||||
import type { RecordStepPayload } from "./record-bridge";
|
||||
import type { RecordStepPayload } from "../record-bridge";
|
||||
import type { RecordingDraftStep } from "./types";
|
||||
|
||||
export interface RecordingStepBuffer {
|
||||
steps: DraftTraceStep[];
|
||||
steps: RecordingDraftStep[];
|
||||
currentUrl?: string;
|
||||
pendingNavigation: boolean;
|
||||
pendingNavigationDeadline?: number;
|
||||
@@ -10,33 +10,33 @@ export interface RecordingStepBuffer {
|
||||
|
||||
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
|
||||
|
||||
function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
||||
function toDraftStep(payload: RecordStepPayload): RecordingDraftStep | null {
|
||||
const pageUrl = payload.page_url;
|
||||
switch (payload.op) {
|
||||
case "click":
|
||||
return payload.target
|
||||
? {
|
||||
op: "click",
|
||||
target: payload.target,
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
captureTarget: payload.target,
|
||||
...(pageUrl ? { pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
case "hover":
|
||||
return payload.target
|
||||
? {
|
||||
op: "hover",
|
||||
target: payload.target,
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
captureTarget: payload.target,
|
||||
...(pageUrl ? { pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
case "fill":
|
||||
return payload.target
|
||||
? {
|
||||
op: "fill",
|
||||
target: payload.target,
|
||||
captureTarget: payload.target,
|
||||
value: payload.value ?? "",
|
||||
...(payload.redacted ? { redacted: true } : {}),
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
...(pageUrl ? { pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
case "press":
|
||||
@@ -44,19 +44,19 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
||||
? {
|
||||
op: "press",
|
||||
key: payload.key,
|
||||
...(payload.target ? { target: payload.target } : {}),
|
||||
...(payload.target ? { captureTarget: payload.target } : {}),
|
||||
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
...(pageUrl ? { pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
case "select":
|
||||
return payload.target && payload.values
|
||||
? {
|
||||
op: "select",
|
||||
target: payload.target,
|
||||
captureTarget: payload.target,
|
||||
values: payload.values,
|
||||
...(payload.labels?.length ? { labels: payload.labels } : {}),
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
...(pageUrl ? { pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
case "navigate":
|
||||
@@ -69,7 +69,7 @@ function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): b
|
||||
const step = buffer.steps[i];
|
||||
if (!step) continue;
|
||||
if (step.op === "click" || step.op === "press" || step.op === "select") {
|
||||
buffer.steps[i] = { ...step, navigated_to: url };
|
||||
buffer.steps[i] = { ...step, navigatedTo: url };
|
||||
return true;
|
||||
}
|
||||
break;
|
||||
@@ -96,7 +96,7 @@ export function observeRecordedNavigation(
|
||||
buffer.steps.push({
|
||||
op: "navigate",
|
||||
url,
|
||||
page_url: url,
|
||||
pageUrl: url,
|
||||
});
|
||||
}
|
||||
return;
|
||||
@@ -107,7 +107,7 @@ export function observeRecordedNavigation(
|
||||
buffer.steps.push({
|
||||
op: "navigate",
|
||||
url,
|
||||
page_url: url,
|
||||
pageUrl: url,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
import type { RenderedRef } from "@browser-skill/vom";
|
||||
import type { TargetDescriptorV3 } from "@/transport/types";
|
||||
import type { CaptureTargetDescriptor } from "../describe-target";
|
||||
import type { IndexedObservationNode, RegisteredObservation } from "./observation-capture";
|
||||
import type { TargetGeometry, TargetMatchHint } from "./types";
|
||||
|
||||
const MATCH_TOLERANCE_PX = 2;
|
||||
|
||||
function close(a: number, b: number): boolean {
|
||||
return Math.abs(a - b) <= MATCH_TOLERANCE_PX;
|
||||
}
|
||||
|
||||
function rectMatches(a: TargetGeometry["rect"], b: TargetGeometry["rect"]): boolean {
|
||||
return close(a.x, b.x) && close(a.y, b.y) && close(a.w, b.w) && close(a.h, b.h);
|
||||
}
|
||||
|
||||
function candidateMatches(candidate: IndexedObservationNode, geometry: TargetGeometry): boolean {
|
||||
return candidate.geometry.rect !== null && rectMatches(geometry.rect, candidate.geometry.rect);
|
||||
}
|
||||
|
||||
export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 {
|
||||
return {
|
||||
...(fallback?.role ? { role: fallback.role } : {}),
|
||||
...(fallback?.name ? { name: fallback.name } : {}),
|
||||
unmatched: true,
|
||||
};
|
||||
}
|
||||
|
||||
function normalized(value: string | undefined): string {
|
||||
return value?.replace(/\s+/g, " ").trim().toLowerCase() ?? "";
|
||||
}
|
||||
|
||||
function matchesSemantics(ref: RenderedRef, fallback?: CaptureTargetDescriptor): boolean {
|
||||
const role = normalized(fallback?.role);
|
||||
const name = normalized(fallback?.name);
|
||||
if (!role && !name) return false;
|
||||
if (role && normalized(ref.role) !== role) return false;
|
||||
if (name && normalized(ref.name) !== name) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function descriptor(ref: RenderedRef): TargetDescriptorV3 {
|
||||
return {
|
||||
ref: ref.ref,
|
||||
...(ref.role ? { role: ref.role } : {}),
|
||||
...(ref.name ? { name: ref.name } : {}),
|
||||
...(ref.ctx ? { ctx: ref.ctx } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function matchObservationTarget(input: {
|
||||
observation: RegisteredObservation;
|
||||
hint?: TargetMatchHint;
|
||||
fallback?: CaptureTargetDescriptor;
|
||||
}): TargetDescriptorV3 {
|
||||
const frameId = input.hint?.frameId ?? input.observation.rootFrameId;
|
||||
const geometry = input.hint?.geometry;
|
||||
if (geometry) {
|
||||
const matches = input.observation.index
|
||||
.candidates(frameId, geometry.tag)
|
||||
.filter((candidate) => candidate.ref && candidateMatches(candidate, geometry));
|
||||
if (matches.length === 1) return descriptor(matches[0]!.ref!);
|
||||
const semanticMatches = matches.filter(
|
||||
(candidate) => candidate.ref && matchesSemantics(candidate.ref, input.fallback),
|
||||
);
|
||||
if (semanticMatches.length === 1) return descriptor(semanticMatches[0]!.ref!);
|
||||
return unmatchedTarget(input.fallback);
|
||||
}
|
||||
|
||||
const semanticMatches = input.observation.index
|
||||
.refs(frameId)
|
||||
.filter((ref) => matchesSemantics(ref, input.fallback));
|
||||
return semanticMatches.length === 1
|
||||
? descriptor(semanticMatches[0]!)
|
||||
: unmatchedTarget(input.fallback);
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
import {
|
||||
type StepV3,
|
||||
type StopReason,
|
||||
TRACE_VERSION_V3,
|
||||
type TraceStateV3,
|
||||
type TraceV3,
|
||||
VOM_FORMAT_VERSION,
|
||||
} from "@/transport/types";
|
||||
import { resolveDraftStartUrl } from "./draft-policy";
|
||||
import type { RecordedStateEntry, RecordingStateRegistry } from "./state-registry";
|
||||
import { reduceTraceStepsV3 } from "./trace-reducer-v3";
|
||||
import { formatTraceStateBody } from "./trace-state-body";
|
||||
import type { RecordingDraftStep, StepAnnotation } from "./types";
|
||||
|
||||
function publishedEntries(registry: RecordingStateRegistry, steps: StepV3[]): RecordedStateEntry[] {
|
||||
const entries = registry.values();
|
||||
if (steps.length === 0) return entries.slice(0, 1);
|
||||
const referenced = new Set(steps.flatMap((step) => [step.state, step.result.state]));
|
||||
return entries.filter((entry) => referenced.has(entry.id));
|
||||
}
|
||||
|
||||
function remapDraftIds(draftIds: number[], stepIdByDraftId: Map<number, number>): number[] {
|
||||
return [
|
||||
...new Set(
|
||||
draftIds.flatMap((id) => {
|
||||
const stepId = stepIdByDraftId.get(id);
|
||||
return stepId === undefined ? [] : [stepId];
|
||||
}),
|
||||
),
|
||||
].sort((a, b) => a - b);
|
||||
}
|
||||
|
||||
export function buildTraceV3(input: {
|
||||
registry: RecordingStateRegistry;
|
||||
drafts: RecordingDraftStep[];
|
||||
annotations?: StepAnnotation[];
|
||||
startedAt: string;
|
||||
purpose?: string;
|
||||
startUrl?: string;
|
||||
stoppedBy: StopReason;
|
||||
bskVersion: string;
|
||||
}): TraceV3 {
|
||||
const reduced = reduceTraceStepsV3(input.drafts);
|
||||
const entries = publishedEntries(input.registry, reduced.steps);
|
||||
const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`]));
|
||||
const annotationsByState = new Map<string, StepAnnotation[]>();
|
||||
for (const annotation of input.annotations ?? []) {
|
||||
const bucket = annotationsByState.get(annotation.stateId) ?? [];
|
||||
bucket.push(annotation);
|
||||
annotationsByState.set(annotation.stateId, bucket);
|
||||
}
|
||||
const steps = reduced.steps.map((step) => ({
|
||||
...step,
|
||||
state: publishedId.get(step.state) ?? step.state,
|
||||
result: { state: publishedId.get(step.result.state) ?? step.result.state },
|
||||
}));
|
||||
const states: TraceStateV3[] = entries.map((entry) => {
|
||||
const id = publishedId.get(entry.id) ?? entry.id;
|
||||
return {
|
||||
id,
|
||||
url: entry.url,
|
||||
...(entry.title ? { title: entry.title } : {}),
|
||||
body: formatTraceStateBody({
|
||||
stateId: id,
|
||||
url: entry.url,
|
||||
title: entry.title,
|
||||
stepIds: remapDraftIds(entry.stepsHere, reduced.stepIdByDraftId),
|
||||
vomText: entry.rawVomText,
|
||||
annotations: annotationsByState.get(entry.id) ?? [],
|
||||
stepIdByDraftId: reduced.stepIdByDraftId,
|
||||
}),
|
||||
...(entry.truncated ? { truncated: true } : {}),
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
version: TRACE_VERSION_V3,
|
||||
...(input.purpose ? { purpose: input.purpose } : {}),
|
||||
recorded_at: new Date().toISOString(),
|
||||
started_at: input.startedAt,
|
||||
stopped_by: input.stoppedBy,
|
||||
entry: { start_url: resolveDraftStartUrl(input.drafts, input.startUrl, states[0]?.url) },
|
||||
recorder: { bsk: input.bskVersion, vom: VOM_FORMAT_VERSION },
|
||||
states,
|
||||
steps,
|
||||
};
|
||||
}
|
||||
+60
-65
@@ -1,30 +1,10 @@
|
||||
import type { DraftTraceStep, PageRef, SelectedOption, Step } from "@/transport/types";
|
||||
|
||||
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
||||
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
||||
|
||||
export function shouldRecordPress(
|
||||
key: string,
|
||||
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
|
||||
): boolean {
|
||||
if (MODIFIER_ONLY_KEYS.has(key)) return false;
|
||||
const mods = modifiers ?? [];
|
||||
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
||||
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
||||
if (key === "Enter" || key === "Escape") return true;
|
||||
// Drop bare character typing — FillSession already records the value.
|
||||
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
||||
return false;
|
||||
}
|
||||
|
||||
function shouldIncludeDraft(step: DraftTraceStep): boolean {
|
||||
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
|
||||
return true;
|
||||
}
|
||||
import type { PageRefV2, SelectedOptionV2, StepV2, TraceV2 } from "@/transport/types";
|
||||
import { resolveDraftStartUrl, shouldIncludeDraft } from "./draft-policy";
|
||||
import type { RecordingDraftStep } from "./types";
|
||||
|
||||
/** Collapse consecutive navigations to the last hop. */
|
||||
function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
|
||||
const out: DraftTraceStep[] = [];
|
||||
function collapseNavigations(steps: RecordingDraftStep[]): RecordingDraftStep[] {
|
||||
const out: RecordingDraftStep[] = [];
|
||||
for (const step of steps) {
|
||||
const prev = out[out.length - 1];
|
||||
if (step.op === "navigate" && prev?.op === "navigate") {
|
||||
@@ -36,7 +16,7 @@ function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
|
||||
return out;
|
||||
}
|
||||
|
||||
function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
||||
function collectUrls(steps: RecordingDraftStep[], startUrl?: string): string[] {
|
||||
const urls: string[] = [];
|
||||
if (startUrl) urls.push(startUrl);
|
||||
for (const step of steps) {
|
||||
@@ -44,8 +24,8 @@ function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
||||
urls.push(step.url);
|
||||
continue;
|
||||
}
|
||||
if ("page_url" in step && step.page_url) urls.push(step.page_url);
|
||||
if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to);
|
||||
if (step.pageUrl) urls.push(step.pageUrl);
|
||||
if ("navigatedTo" in step && step.navigatedTo) urls.push(step.navigatedTo);
|
||||
}
|
||||
const seen = new Set<string>();
|
||||
const unique: string[] = [];
|
||||
@@ -58,9 +38,9 @@ function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
||||
}
|
||||
|
||||
function buildPageRegistry(
|
||||
steps: DraftTraceStep[],
|
||||
steps: RecordingDraftStep[],
|
||||
startUrl?: string,
|
||||
): { pages: PageRef[]; urlToId: Map<string, string> } {
|
||||
): { pages: PageRefV2[]; urlToId: Map<string, string> } {
|
||||
const urls = collectUrls(steps, startUrl);
|
||||
const urlToId = new Map<string, string>();
|
||||
const pages = urls.map((url, index) => {
|
||||
@@ -81,28 +61,28 @@ function pageIdFor(
|
||||
return urlToId.values().next().value ?? "p1";
|
||||
}
|
||||
|
||||
function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined {
|
||||
if (step.op === "navigate") return step.page_url ?? step.url;
|
||||
if ("page_url" in step && step.page_url) return step.page_url;
|
||||
function pageUrlForDraft(step: RecordingDraftStep, fallbackUrl?: string): string | undefined {
|
||||
if (step.op === "navigate") return step.pageUrl ?? step.url;
|
||||
if (step.pageUrl) return step.pageUrl;
|
||||
return fallbackUrl;
|
||||
}
|
||||
|
||||
function effectForNavigation(
|
||||
navigatedTo: string | undefined,
|
||||
urlToId: Map<string, string>,
|
||||
): Step["effect"] {
|
||||
): StepV2["effect"] {
|
||||
if (!navigatedTo) return undefined;
|
||||
const pageId = urlToId.get(navigatedTo);
|
||||
if (!pageId) return undefined;
|
||||
return { navigated_to: pageId };
|
||||
}
|
||||
|
||||
function withEffect(step: Step, effect: Step["effect"]): Step {
|
||||
function withEffect(step: StepV2, effect: StepV2["effect"]): StepV2 {
|
||||
if (!effect) return step;
|
||||
return { ...step, effect };
|
||||
}
|
||||
|
||||
function toSelection(values: string[], labels?: string[]): SelectedOption[] {
|
||||
function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] {
|
||||
return values.map((value, index) => ({
|
||||
value,
|
||||
...(labels?.[index] ? { label: labels[index] } : {}),
|
||||
@@ -110,11 +90,11 @@ function toSelection(values: string[], labels?: string[]): SelectedOption[] {
|
||||
}
|
||||
|
||||
function toV2Step(
|
||||
step: DraftTraceStep,
|
||||
step: RecordingDraftStep,
|
||||
id: number,
|
||||
urlToId: Map<string, string>,
|
||||
fallbackUrl?: string,
|
||||
): Step | null {
|
||||
): StepV2 | null {
|
||||
if (!shouldIncludeDraft(step)) return null;
|
||||
|
||||
const pageUrl = pageUrlForDraft(step, fallbackUrl);
|
||||
@@ -129,28 +109,31 @@ function toV2Step(
|
||||
to: step.url,
|
||||
};
|
||||
case "click":
|
||||
if (!step.captureTarget) return null;
|
||||
return withEffect(
|
||||
{
|
||||
op: "click",
|
||||
id,
|
||||
page,
|
||||
target: step.target,
|
||||
target: step.captureTarget,
|
||||
},
|
||||
effectForNavigation(step.navigated_to, urlToId),
|
||||
effectForNavigation(step.navigatedTo, urlToId),
|
||||
);
|
||||
case "hover":
|
||||
if (!step.captureTarget) return null;
|
||||
return {
|
||||
op: "hover",
|
||||
id,
|
||||
page,
|
||||
target: step.target,
|
||||
target: step.captureTarget,
|
||||
};
|
||||
case "fill":
|
||||
if (!step.captureTarget) return null;
|
||||
return {
|
||||
op: "fill",
|
||||
id,
|
||||
page,
|
||||
target: step.target,
|
||||
target: step.captureTarget,
|
||||
value: step.value,
|
||||
...(step.redacted ? { redacted: true } : {}),
|
||||
};
|
||||
@@ -161,44 +144,47 @@ function toV2Step(
|
||||
id,
|
||||
page,
|
||||
key: step.key,
|
||||
...(step.target ? { target: step.target } : {}),
|
||||
...(step.captureTarget ? { target: step.captureTarget } : {}),
|
||||
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
|
||||
},
|
||||
effectForNavigation(step.navigated_to, urlToId),
|
||||
effectForNavigation(step.navigatedTo, urlToId),
|
||||
);
|
||||
case "select":
|
||||
if (!step.captureTarget) return null;
|
||||
return withEffect(
|
||||
{
|
||||
op: "select",
|
||||
id,
|
||||
page,
|
||||
target: step.target,
|
||||
target: step.captureTarget,
|
||||
selection: toSelection(step.values, step.labels),
|
||||
},
|
||||
effectForNavigation(step.navigated_to, urlToId),
|
||||
effectForNavigation(step.navigatedTo, urlToId),
|
||||
);
|
||||
case "scroll":
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export interface ReducedTrace {
|
||||
pages: PageRef[];
|
||||
steps: Step[];
|
||||
interface ReducedTrace {
|
||||
pages: PageRefV2[];
|
||||
steps: StepV2[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Compile capture drafts into record-only trace v2 steps.
|
||||
* Variable inputs are NOT classified here — executing agents infer that at run time.
|
||||
*/
|
||||
export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): ReducedTrace {
|
||||
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string): ReducedTrace {
|
||||
const collapsed = collapseNavigations(steps);
|
||||
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
|
||||
const out: Step[] = [];
|
||||
const out: StepV2[] = [];
|
||||
let id = 1;
|
||||
let lastUrl = startUrl;
|
||||
for (const draft of collapsed) {
|
||||
if (draft.op === "navigate") lastUrl = draft.url;
|
||||
else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to;
|
||||
else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url;
|
||||
else if ("navigatedTo" in draft && draft.navigatedTo) lastUrl = draft.navigatedTo;
|
||||
else if (draft.pageUrl) lastUrl = draft.pageUrl;
|
||||
const step = toV2Step(draft, id, urlToId, lastUrl);
|
||||
if (!step) continue;
|
||||
out.push(step);
|
||||
@@ -207,18 +193,27 @@ export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): Re
|
||||
return { pages, steps: out };
|
||||
}
|
||||
|
||||
export function resolveTraceStartUrl(
|
||||
drafts: DraftTraceStep[],
|
||||
function resolveTraceStartUrl(
|
||||
drafts: RecordingDraftStep[],
|
||||
startUrl?: string,
|
||||
pages?: PageRef[],
|
||||
pages?: PageRefV2[],
|
||||
): string {
|
||||
if (startUrl) return startUrl;
|
||||
const navigate = drafts.find((step): step is Extract<DraftTraceStep, { op: "navigate" }> => {
|
||||
return step.op === "navigate";
|
||||
});
|
||||
if (navigate) return navigate.url;
|
||||
for (const draft of drafts) {
|
||||
if ("page_url" in draft && draft.page_url) return draft.page_url;
|
||||
}
|
||||
return pages?.[0]?.url ?? "about:blank";
|
||||
return resolveDraftStartUrl(drafts, startUrl, pages?.[0]?.url);
|
||||
}
|
||||
|
||||
export function buildTraceV2(input: {
|
||||
steps: RecordingDraftStep[];
|
||||
startedAt: string;
|
||||
startUrl?: string;
|
||||
purpose?: string;
|
||||
}): TraceV2 {
|
||||
const { pages, steps } = reduceTraceSteps(input.steps, input.startUrl);
|
||||
return {
|
||||
recorded_at: new Date().toISOString(),
|
||||
started_at: input.startedAt,
|
||||
...(input.purpose ? { purpose: input.purpose } : {}),
|
||||
entry: { start_url: resolveTraceStartUrl(input.steps, input.startUrl, pages) },
|
||||
pages,
|
||||
steps,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
import type { NavigationCause, StepV3 } from "@/transport/types";
|
||||
import { shouldIncludeDraft } from "./draft-policy";
|
||||
import { unmatchedTarget } from "./target-matcher";
|
||||
import type { RecordingDraftStep } from "./types";
|
||||
|
||||
interface CollapsedDraft {
|
||||
draft: RecordingDraftStep;
|
||||
draftIds: number[];
|
||||
}
|
||||
|
||||
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
|
||||
const TRANSITION_CAUSES: Record<string, NavigationCause> = {
|
||||
typed: "user_typed",
|
||||
generated: "user_typed",
|
||||
keyword: "user_typed",
|
||||
keyword_generated: "user_typed",
|
||||
link: "link",
|
||||
form_submit: "form_submit",
|
||||
reload: "reload",
|
||||
auto_bookmark: "browser",
|
||||
start_page: "browser",
|
||||
};
|
||||
|
||||
function isRedirect(step: Extract<RecordingDraftStep, { op: "navigate" }>): boolean {
|
||||
return (step.transitionQualifiers ?? []).some((qualifier) => REDIRECT_QUALIFIERS.has(qualifier));
|
||||
}
|
||||
|
||||
function collapseRedirects(steps: RecordingDraftStep[]): CollapsedDraft[] {
|
||||
const output: CollapsedDraft[] = [];
|
||||
steps.forEach((step, index) => {
|
||||
const previous = output[output.length - 1];
|
||||
if (step.op === "navigate" && previous?.draft.op === "navigate" && isRedirect(step)) {
|
||||
previous.draft = {
|
||||
...previous.draft,
|
||||
url: step.url,
|
||||
postStateId: step.postStateId ?? previous.draft.postStateId,
|
||||
};
|
||||
previous.draftIds.push(index + 1);
|
||||
return;
|
||||
}
|
||||
output.push({ draft: { ...step }, draftIds: [index + 1] });
|
||||
});
|
||||
return output;
|
||||
}
|
||||
|
||||
function navigationCause(step: Extract<RecordingDraftStep, { op: "navigate" }>): NavigationCause {
|
||||
if (step.cause) return step.cause;
|
||||
const qualifiers = step.transitionQualifiers ?? [];
|
||||
if (qualifiers.includes("forward_back")) return "history";
|
||||
if (qualifiers.includes("from_address_bar")) return "user_typed";
|
||||
return TRANSITION_CAUSES[step.transitionType ?? ""] ?? "browser";
|
||||
}
|
||||
|
||||
function selection(values: string[], labels?: string[]): Array<{ value: string; label?: string }> {
|
||||
return values.map((value, index) => ({
|
||||
value,
|
||||
...(labels?.[index] ? { label: labels[index] } : {}),
|
||||
}));
|
||||
}
|
||||
|
||||
function reduceDraft(draft: RecordingDraftStep, id: number): StepV3 | null {
|
||||
if (!shouldIncludeDraft(draft)) return null;
|
||||
const state = draft.preStateId ?? draft.postStateId;
|
||||
const resultState = draft.postStateId ?? draft.preStateId;
|
||||
if (!state || !resultState) return null;
|
||||
const common = { id, state, result: { state: resultState } };
|
||||
|
||||
switch (draft.op) {
|
||||
case "navigate":
|
||||
return { op: "navigate", ...common, to: draft.url, cause: navigationCause(draft) };
|
||||
case "click":
|
||||
return {
|
||||
op: "click",
|
||||
...common,
|
||||
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||
};
|
||||
case "hover":
|
||||
return {
|
||||
op: "hover",
|
||||
...common,
|
||||
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||
};
|
||||
case "fill":
|
||||
return {
|
||||
op: "fill",
|
||||
...common,
|
||||
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||
value: draft.value,
|
||||
commit: draft.commit ?? "blur",
|
||||
...(draft.redacted ? { redacted: true } : {}),
|
||||
};
|
||||
case "press":
|
||||
return {
|
||||
op: "press",
|
||||
...common,
|
||||
key: draft.key,
|
||||
...(draft.captureTarget || draft.matchedTarget
|
||||
? { target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget) }
|
||||
: {}),
|
||||
...(draft.modifiers?.length ? { modifiers: draft.modifiers } : {}),
|
||||
};
|
||||
case "select":
|
||||
return {
|
||||
op: "select",
|
||||
...common,
|
||||
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
|
||||
selection: selection(draft.values, draft.labels),
|
||||
};
|
||||
case "scroll":
|
||||
return { op: "scroll", ...common };
|
||||
}
|
||||
}
|
||||
|
||||
export interface ReducedTraceV3 {
|
||||
steps: StepV3[];
|
||||
stepIdByDraftId: Map<number, number>;
|
||||
}
|
||||
|
||||
export function reduceTraceStepsV3(steps: RecordingDraftStep[]): ReducedTraceV3 {
|
||||
const output: StepV3[] = [];
|
||||
const stepIdByDraftId = new Map<number, number>();
|
||||
for (const { draft, draftIds } of collapseRedirects(steps)) {
|
||||
const step = reduceDraft(draft, output.length + 1);
|
||||
if (!step) continue;
|
||||
output.push(step);
|
||||
for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id);
|
||||
}
|
||||
return { steps: output, stepIdByDraftId };
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import type { StepAnnotation } from "./types";
|
||||
|
||||
function annotationText(annotation: StepAnnotation, stepId: number): string {
|
||||
return ` ⟵ step ${stepId}: ${annotation.op}${annotation.detail ? `: ${annotation.detail}` : ""}`;
|
||||
}
|
||||
|
||||
export function formatTraceStateBody(input: {
|
||||
stateId: string;
|
||||
url: string;
|
||||
title?: string;
|
||||
stepIds: number[];
|
||||
vomText: string;
|
||||
annotations: StepAnnotation[];
|
||||
stepIdByDraftId: Map<number, number>;
|
||||
}): string {
|
||||
const lines = ["# bsk-observation 1", `state: ${JSON.stringify(input.stateId)}`];
|
||||
lines.push(`url: ${JSON.stringify(input.url)}`);
|
||||
if (input.title) lines.push(`title: ${JSON.stringify(input.title)}`);
|
||||
if (input.stepIds.length > 0) lines.push(`steps_here: [${input.stepIds.join(", ")}]`);
|
||||
lines.push("---");
|
||||
|
||||
const byLine = new Map<number, Array<{ annotation: StepAnnotation; stepId: number }>>();
|
||||
for (const annotation of input.annotations) {
|
||||
const stepId = input.stepIdByDraftId.get(annotation.draftId);
|
||||
if (stepId === undefined) continue;
|
||||
const bucket = byLine.get(annotation.line) ?? [];
|
||||
bucket.push({ annotation, stepId });
|
||||
byLine.set(annotation.line, bucket);
|
||||
}
|
||||
|
||||
input.vomText.split("\n").forEach((bodyLine, lineIndex) => {
|
||||
let line = bodyLine;
|
||||
for (const item of byLine.get(lineIndex) ?? []) {
|
||||
line += annotationText(item.annotation, item.stepId);
|
||||
}
|
||||
lines.push(line);
|
||||
});
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
import type { CaptureTargetDescriptor } from "@/lib/describe-target";
|
||||
import type {
|
||||
FillCommit,
|
||||
KeyModifier,
|
||||
NavigationCause,
|
||||
StepV3,
|
||||
TargetDescriptorV3,
|
||||
} from "@/transport/types";
|
||||
|
||||
export interface TargetGeometry {
|
||||
/** Top-level viewport-relative CSS pixels, as defined by the geometry module. */
|
||||
rect: { x: number; y: number; w: number; h: number };
|
||||
tag: string;
|
||||
}
|
||||
|
||||
export interface TargetMatchHint {
|
||||
geometry?: TargetGeometry;
|
||||
/** Missing means the current top frame, never an unrestricted frame search. */
|
||||
frameId?: string;
|
||||
}
|
||||
|
||||
export interface StepAnnotation {
|
||||
draftId: number;
|
||||
op: StepV3["op"];
|
||||
line: number;
|
||||
stateId: string;
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
interface DraftStateLink {
|
||||
pageUrl?: string;
|
||||
preStateId?: string;
|
||||
postStateId?: string;
|
||||
}
|
||||
|
||||
interface DraftTarget {
|
||||
captureTarget?: CaptureTargetDescriptor;
|
||||
targetHint?: TargetMatchHint;
|
||||
matchedTarget?: TargetDescriptorV3;
|
||||
}
|
||||
|
||||
interface DraftNavigationEffect {
|
||||
navigatedTo?: string;
|
||||
}
|
||||
|
||||
export type RecordingDraftStep =
|
||||
| ({ op: "click" } & DraftStateLink & DraftTarget & DraftNavigationEffect)
|
||||
| ({ op: "hover" } & DraftStateLink & DraftTarget)
|
||||
| ({
|
||||
op: "fill";
|
||||
value: string;
|
||||
commit?: FillCommit;
|
||||
redacted?: boolean;
|
||||
} & DraftStateLink &
|
||||
DraftTarget &
|
||||
DraftNavigationEffect)
|
||||
| ({
|
||||
op: "press";
|
||||
key: string;
|
||||
modifiers?: KeyModifier[];
|
||||
} & DraftStateLink &
|
||||
DraftTarget &
|
||||
DraftNavigationEffect)
|
||||
| ({
|
||||
op: "select";
|
||||
values: string[];
|
||||
labels?: string[];
|
||||
} & DraftStateLink &
|
||||
DraftTarget &
|
||||
DraftNavigationEffect)
|
||||
| ({ op: "scroll" } & DraftStateLink)
|
||||
| ({
|
||||
op: "navigate";
|
||||
url: string;
|
||||
cause?: NavigationCause;
|
||||
transitionType?: string;
|
||||
transitionQualifiers?: string[];
|
||||
} & DraftStateLink);
|
||||
|
||||
export type TargetedRecordingDraft = Extract<
|
||||
RecordingDraftStep,
|
||||
{ op: "click" | "hover" | "fill" | "press" | "select" }
|
||||
>;
|
||||
@@ -203,6 +203,55 @@ describe("ToolDispatcher", () => {
|
||||
expect(sent[0]).toEqual({ id: "r-1", result: {} });
|
||||
});
|
||||
|
||||
it("forwards daemon cancellation into an in-flight session stop", async () => {
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const remove = vi.fn(async () => {});
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove,
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
let finishDetach: () => void = () => {};
|
||||
const detach = new Promise<void>((resolve) => {
|
||||
finishDetach = resolve;
|
||||
});
|
||||
const cdp = {
|
||||
send: vi.fn(),
|
||||
detachSession: vi.fn(() => detach),
|
||||
ensureNetworkCapture: vi.fn(async () => {}),
|
||||
networkEntriesSince: vi.fn(() => ({
|
||||
tab_id: 7,
|
||||
entries: [],
|
||||
next_since: 0,
|
||||
truncated: false,
|
||||
})),
|
||||
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
setUserAgentOverride: vi.fn(async () => {}),
|
||||
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||
dispatcher.start();
|
||||
|
||||
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
|
||||
await vi.waitFor(() => expect(cdp.detachSession).toHaveBeenCalledWith("aa11"));
|
||||
deliver({ id: "cancel-stop", method: "cancel", params: { rpc_id: "r-1" } });
|
||||
await flushMicrotasks();
|
||||
finishDetach();
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(sent).toContainEqual({ id: "cancel-stop", result: { cancelled: true } });
|
||||
expect(sent).toContainEqual({
|
||||
id: "r-1",
|
||||
error: expect.objectContaining({ code: "cancelled" }),
|
||||
});
|
||||
expect(sessions.has("aa11")).toBe(true);
|
||||
expect(remove).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns not_found when stopping an unknown session", async () => {
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
buildFrameVomScene,
|
||||
buildVomScene,
|
||||
type CdpAxNode,
|
||||
captureVomObservation,
|
||||
handleGetHtml,
|
||||
handleObserve,
|
||||
handleScreenshot,
|
||||
@@ -1329,7 +1330,7 @@ describe("buildVomScene", () => {
|
||||
expect(rendered).toContain(' @e2 textbox "验证码"');
|
||||
});
|
||||
|
||||
it("does not mark non-password inputs sensitive from password-like labels", () => {
|
||||
it("marks current-password autocomplete as sensitive even for text inputs", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "1",
|
||||
@@ -1364,7 +1365,7 @@ describe("buildVomScene", () => {
|
||||
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
id: 10,
|
||||
sensitive: false,
|
||||
sensitive: true,
|
||||
}),
|
||||
);
|
||||
});
|
||||
@@ -1659,7 +1660,9 @@ describe("buildVomScene", () => {
|
||||
);
|
||||
const rendered = renderVom(scene);
|
||||
expect(rendered.text).toContain('@e1 button "close"');
|
||||
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
|
||||
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
|
||||
{ ref: "e1", backendNodeId: 20 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not promote a clickable container that wraps a real interactive control", () => {
|
||||
@@ -1803,7 +1806,9 @@ describe("buildVomScene", () => {
|
||||
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic");
|
||||
const rendered = renderVom(scene);
|
||||
expect(rendered.text).toContain('@e1 button "收藏"');
|
||||
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
|
||||
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
|
||||
{ ref: "e1", backendNodeId: 20 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("builds active scope blocks from active aria-controls relationships", () => {
|
||||
@@ -2904,10 +2909,360 @@ describe("handleSnapshot", () => {
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function secretsSnapshotReply(secrets: {
|
||||
email: string;
|
||||
password: string;
|
||||
otp: string;
|
||||
card: string;
|
||||
}) {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"input",
|
||||
"position",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"type",
|
||||
"text",
|
||||
"password",
|
||||
"autocomplete",
|
||||
"one-time-code",
|
||||
"cc-number",
|
||||
"value",
|
||||
secrets.email,
|
||||
secrets.password,
|
||||
secrets.otp,
|
||||
secrets.card,
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const style = [i("static"), i("auto"), i("auto")];
|
||||
return {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 1, 1, 1],
|
||||
nodeName: [i("html"), i("body"), i("input"), i("input"), i("input"), i("input")],
|
||||
backendNodeId: [10, 11, 12, 13, 14, 15],
|
||||
attributes: [
|
||||
[],
|
||||
[],
|
||||
[i("type"), i("text"), i("value"), i(secrets.email)],
|
||||
[i("type"), i("password"), i("value"), i(secrets.password)],
|
||||
[
|
||||
i("type"),
|
||||
i("text"),
|
||||
i("autocomplete"),
|
||||
i("one-time-code"),
|
||||
i("value"),
|
||||
i(secrets.otp),
|
||||
],
|
||||
[
|
||||
i("type"),
|
||||
i("text"),
|
||||
i("autocomplete"),
|
||||
i("cc-number"),
|
||||
i("value"),
|
||||
i(secrets.card),
|
||||
],
|
||||
],
|
||||
inputValue: {
|
||||
index: [2, 3, 4, 5],
|
||||
value: [i(secrets.email), i(secrets.password), i(secrets.otp), i(secrets.card)],
|
||||
},
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3, 4, 5],
|
||||
styles: [style, style, style, style, style],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[10, 10, 200, 32],
|
||||
[10, 50, 200, 32],
|
||||
[10, 90, 200, 32],
|
||||
[10, 130, 200, 32],
|
||||
],
|
||||
paintOrders: [0, 1, 2, 3, 4],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
const VP_METRICS = {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
|
||||
function makeFrameAwareDeps() {
|
||||
const strings = ["html", "body", "iframe", "button", "title", "Remote", "static", "auto"];
|
||||
const i = (value: string) => strings.indexOf(value);
|
||||
const styles = [i("static"), i("auto"), i("auto")];
|
||||
const snapshot = {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
frameId: "main",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("iframe")],
|
||||
backendNodeId: [10, 11, 12],
|
||||
attributes: [[], [], [i("title"), i("Remote")]],
|
||||
contentDocumentIndex: { index: [2], value: [1] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1, 2],
|
||||
styles: [styles, styles, styles],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[0, 0, 1000, 800],
|
||||
[100, 100, 400, 300],
|
||||
],
|
||||
paintOrders: [0, 0, 1],
|
||||
},
|
||||
},
|
||||
{
|
||||
frameId: "child",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("button")],
|
||||
backendNodeId: [20, 21, 22],
|
||||
attributes: [[], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1, 2],
|
||||
styles: [styles, styles, styles],
|
||||
bounds: [
|
||||
[0, 0, 400, 300],
|
||||
[0, 0, 400, 300],
|
||||
[20, 30, 120, 40],
|
||||
],
|
||||
paintOrders: [0, 0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const mainAx: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "main-root",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 11,
|
||||
childIds: ["frame-owner"],
|
||||
},
|
||||
{
|
||||
nodeId: "frame-owner",
|
||||
parentId: "main-root",
|
||||
role: { type: "role", value: "Iframe" },
|
||||
name: { type: "computedString", value: "Remote" },
|
||||
backendDOMNodeId: 12,
|
||||
},
|
||||
];
|
||||
const childAx: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "child-root",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 21,
|
||||
childIds: ["child-button"],
|
||||
},
|
||||
{
|
||||
nodeId: "child-button",
|
||||
parentId: "child-root",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Frame action" },
|
||||
backendDOMNodeId: 22,
|
||||
},
|
||||
];
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Page.getLayoutMetrics") return VP_METRICS;
|
||||
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: mainAx };
|
||||
throw new Error(`unexpected root CDP method ${method}`);
|
||||
});
|
||||
const sendToTarget = vi.fn(async (_target, method: string) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: childAx };
|
||||
throw new Error(`unexpected child CDP method ${method}`);
|
||||
});
|
||||
const cdp = {
|
||||
send: send as unknown as CdpRunner["send"],
|
||||
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 12,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
trackSessionTab: vi.fn(),
|
||||
} satisfies CdpRunner;
|
||||
return {
|
||||
cdp,
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it("exposes the frame-aware observation result for recording", async () => {
|
||||
const ax: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "root",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 11,
|
||||
childIds: ["password"],
|
||||
},
|
||||
{
|
||||
nodeId: "password",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Password" },
|
||||
value: { value: "hunter2" },
|
||||
backendDOMNodeId: 13,
|
||||
},
|
||||
];
|
||||
const deps = makeOverlayDeps(ax, loginSnapshotReply(), VP_METRICS);
|
||||
|
||||
const result = await captureVomObservation(deps.cdp, 4, "https://example.com", {
|
||||
redactValues: true,
|
||||
});
|
||||
|
||||
expect(result.text).not.toContain("hunter2");
|
||||
expect(result.text).toContain("•••");
|
||||
expect(result.refs[0]).toMatchObject({
|
||||
backendNodeId: 13,
|
||||
role: "textbox",
|
||||
name: "Password",
|
||||
line: expect.any(Number),
|
||||
});
|
||||
expect(result.rootFrameId).toBe("root");
|
||||
expect(result).not.toHaveProperty("frameDocuments");
|
||||
expect(result.frames).toEqual([{ frameId: "root", target: { tabId: 4 } }]);
|
||||
expect(result.matchNodes.find((node) => node.backendNodeId === 13)).toMatchObject({
|
||||
frameId: "root",
|
||||
backendNodeId: 13,
|
||||
tag: "input",
|
||||
rect: { x: 400, y: 300, w: 200, h: 40 },
|
||||
localRect: { x: 400, y: 300, w: 200, h: 40 },
|
||||
});
|
||||
});
|
||||
|
||||
it("does not leak form secrets through the record-safe observation payload", async () => {
|
||||
const secrets = {
|
||||
email: "user@example.com",
|
||||
password: "hunter2",
|
||||
otp: "847291",
|
||||
card: "4111111111111111",
|
||||
};
|
||||
const ax: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "root",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 11,
|
||||
childIds: ["email", "password", "otp", "card"],
|
||||
},
|
||||
{
|
||||
nodeId: "email",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Email" },
|
||||
value: { value: secrets.email },
|
||||
backendDOMNodeId: 12,
|
||||
},
|
||||
{
|
||||
nodeId: "password",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Password" },
|
||||
value: { value: secrets.password },
|
||||
backendDOMNodeId: 13,
|
||||
},
|
||||
{
|
||||
nodeId: "otp",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Code" },
|
||||
value: { value: secrets.otp },
|
||||
backendDOMNodeId: 14,
|
||||
},
|
||||
{
|
||||
nodeId: "card",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Card" },
|
||||
value: { value: secrets.card },
|
||||
backendDOMNodeId: 15,
|
||||
},
|
||||
];
|
||||
const deps = makeOverlayDeps(ax, secretsSnapshotReply(secrets), VP_METRICS);
|
||||
|
||||
const result = await captureVomObservation(deps.cdp, 4, "https://example.com", {
|
||||
redactValues: true,
|
||||
});
|
||||
const dumped = JSON.stringify(result);
|
||||
|
||||
expect(result).not.toHaveProperty("frameDocuments");
|
||||
expect(dumped).not.toContain(secrets.email);
|
||||
expect(dumped).not.toContain(secrets.password);
|
||||
expect(dumped).not.toContain(secrets.otp);
|
||||
expect(dumped).not.toContain(secrets.card);
|
||||
expect(dumped).not.toContain("formValue");
|
||||
expect(dumped).not.toContain("formDefaultValue");
|
||||
expect(dumped).not.toContain("axNodes");
|
||||
expect(dumped).not.toContain("domNodes");
|
||||
expect(dumped).not.toContain("attrs");
|
||||
expect(result.text).toContain("•••");
|
||||
});
|
||||
|
||||
it("keeps frames and rendered refs on the same frame identity", async () => {
|
||||
const deps = makeFrameAwareDeps();
|
||||
|
||||
const result = await captureVomObservation(deps.cdp, 4, "https://example.com");
|
||||
|
||||
expect(result.rootFrameId).toBe("main");
|
||||
expect(result.frames).toHaveLength(2);
|
||||
expect(result.frames.find((frame) => frame.frameId === "child")).toEqual({
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 12,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
});
|
||||
expect(result.matchNodes.find((node) => node.backendNodeId === 22)).toMatchObject({
|
||||
frameId: "child",
|
||||
tag: "button",
|
||||
localRect: { x: 20, y: 30, w: 120, h: 40 },
|
||||
});
|
||||
expect(result.refs.find((ref) => ref.backendNodeId === 22)).toMatchObject({
|
||||
frameId: "child",
|
||||
name: "Frame action",
|
||||
});
|
||||
});
|
||||
|
||||
it("stores iframe refs with their owning CDP session", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
const deps = makeFrameAwareDeps();
|
||||
|
||||
const result = await handleSnapshot(sm, { session_id: "aa11" }, deps);
|
||||
|
||||
if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`);
|
||||
expect(result.text).toContain('@e1 button "Frame action"');
|
||||
const frameRef = [...ctx.refStore.entries()].find(([, entry]) => entry.backendNodeId === 22);
|
||||
expect(frameRef?.[1]).toMatchObject({
|
||||
tabId: 4,
|
||||
frameId: "child",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
});
|
||||
|
||||
it("renders a blocking login overlay as the focused top layer", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
|
||||
@@ -66,6 +66,52 @@ function makeApis(
|
||||
}
|
||||
|
||||
describe("handleSessionStop with auto-return", () => {
|
||||
it("leaves the session untouched when cancellation arrived before teardown", async () => {
|
||||
const aw = fakeAgentWindow([100]);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
await sm.start("aa11");
|
||||
const cdp = { detachSession: vi.fn(async () => {}) };
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ cdp, signal: controller.signal },
|
||||
);
|
||||
|
||||
expect(res).toMatchObject({ code: "cancelled" });
|
||||
expect(sm.has("aa11")).toBe(true);
|
||||
expect(cdp.detachSession).not.toHaveBeenCalled();
|
||||
expect(aw.remove).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not close the Agent Window when cancellation lands during CDP detach", async () => {
|
||||
const aw = fakeAgentWindow([100]);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
await sm.start("aa11");
|
||||
const controller = new AbortController();
|
||||
let finishDetach: () => void = () => {};
|
||||
const detach = new Promise<void>((resolve) => {
|
||||
finishDetach = resolve;
|
||||
});
|
||||
const cdp = { detachSession: vi.fn(() => detach) };
|
||||
|
||||
const stopping = handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ cdp, signal: controller.signal },
|
||||
);
|
||||
await vi.waitFor(() => expect(cdp.detachSession).toHaveBeenCalledWith("aa11"));
|
||||
controller.abort();
|
||||
finishDetach();
|
||||
|
||||
const res = await stopping;
|
||||
expect(res).toMatchObject({ code: "cancelled" });
|
||||
expect(sm.has("aa11")).toBe(true);
|
||||
expect(aw.remove).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns every borrowed tab and closes the Agent Window in the right order", async () => {
|
||||
const aw = fakeAgentWindow([100]);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
export {
|
||||
type CaptureVomObservationOptions,
|
||||
captureVomObservation,
|
||||
} from "./observation";
|
||||
export {
|
||||
type CaptureVomFrame,
|
||||
type CaptureVomMatchNode,
|
||||
type CaptureVomObservationResult,
|
||||
} from "./vom/record-safe-observation";
|
||||
@@ -290,6 +290,7 @@ export class ToolDispatcher {
|
||||
await this.releaseHoverLatch((req.params as SessionStopParams).session_id);
|
||||
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
|
||||
cdp: this.cdp,
|
||||
signal,
|
||||
});
|
||||
}
|
||||
case "tool.tab_list":
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
isVomReferenceNode,
|
||||
renderVom,
|
||||
type VomNode,
|
||||
type VomOptions,
|
||||
type VomScene,
|
||||
} from "@browser-skill/vom";
|
||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
@@ -53,6 +54,10 @@ import {
|
||||
collectOverlayExcludedBackendIds,
|
||||
} from "./vom/capture";
|
||||
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
|
||||
import {
|
||||
type CaptureVomObservationResult,
|
||||
projectRecordSafeObservation,
|
||||
} from "./vom/record-safe-observation";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared helpers (legacy aliases — observation.ts kept exporting these
|
||||
@@ -411,7 +416,15 @@ function isModalSignal(
|
||||
|
||||
function isSensitive(capturedNode: CapturedNode | undefined): boolean {
|
||||
const attrs = capturedNode?.attrs ?? {};
|
||||
return (attrs.type ?? "").toLowerCase() === "password";
|
||||
const type = (attrs.type ?? "").toLowerCase();
|
||||
if (type === "password") return true;
|
||||
const autocomplete = (attrs.autocomplete ?? "").toLowerCase();
|
||||
return (
|
||||
autocomplete.startsWith("cc-") ||
|
||||
autocomplete === "one-time-code" ||
|
||||
autocomplete === "current-password" ||
|
||||
autocomplete === "new-password"
|
||||
);
|
||||
}
|
||||
|
||||
interface AxNodeSignals {
|
||||
@@ -461,7 +474,13 @@ const AX_TEXT_STOP_ROLES = new Set([
|
||||
"columnheader",
|
||||
"rowheader",
|
||||
]);
|
||||
const SENSITIVE_AX_INPUT_TYPES = new Set(["password", "credit-card"]);
|
||||
const SENSITIVE_AX_INPUT_TYPES = new Set([
|
||||
"password",
|
||||
"credit-card",
|
||||
"one-time-code",
|
||||
"current-password",
|
||||
"new-password",
|
||||
]);
|
||||
|
||||
function axPropertyString(axNode: CdpAxNode, name: string): string | undefined {
|
||||
const prop = axNode.properties?.find((item) => item.name === name);
|
||||
@@ -1537,14 +1556,13 @@ async function fallbackCapturedViewModel(
|
||||
try {
|
||||
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
throwIfAborted(signal, "observation");
|
||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
const source = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
viewport = {
|
||||
width: vpSrc.clientWidth ?? 0,
|
||||
height: vpSrc.clientHeight ?? 0,
|
||||
width: source.clientWidth ?? 0,
|
||||
height: source.clientHeight ?? 0,
|
||||
};
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
// viewport stays zero-sized
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
}
|
||||
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal);
|
||||
throwIfAborted(signal, "observation");
|
||||
@@ -1554,22 +1572,58 @@ async function fallbackCapturedViewModel(
|
||||
async function captureForVom(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
conditionalSurfaceProbe: boolean,
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>,
|
||||
signal?: AbortSignal,
|
||||
options: CaptureVomObservationOptions,
|
||||
): Promise<CapturedViewModel> {
|
||||
try {
|
||||
return await captureViewModel(cdp, tabId, {
|
||||
conditionalSurfaceProbe,
|
||||
hoverProbeBypassOverlay,
|
||||
signal,
|
||||
conditionalSurfaceProbe: options.conditionalSurfaceProbe ?? false,
|
||||
hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
|
||||
signal: options.signal,
|
||||
});
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
return fallbackCapturedViewModel(cdp, tabId, signal);
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
return fallbackCapturedViewModel(cdp, tabId, options.signal);
|
||||
}
|
||||
}
|
||||
|
||||
export interface CaptureVomObservationOptions extends VomOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export async function captureVomObservation(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
url: string | undefined,
|
||||
options: CaptureVomObservationOptions = {},
|
||||
): Promise<CaptureVomObservationResult> {
|
||||
throwIfAborted(options.signal, "observation");
|
||||
await cdp.ensureAttachedToUrl?.(tabId, url);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const captured = await captureForVom(cdp, tabId, options);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const documents = await captureFrameData<CdpAxNode>(cdp, tabId, captured, options.signal);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const scene =
|
||||
captured.rootFrameId && captured.frameNodes?.size
|
||||
? buildFrameVomScene(documents, captured, { pageUrl: url })
|
||||
: buildVomScene(documents[0]?.axNodes ?? [], captured, { pageUrl: url });
|
||||
const rendered = renderVom(scene, {
|
||||
maxDepth: options.maxDepth,
|
||||
maxTokens: options.maxTokens,
|
||||
redactValues: options.redactValues,
|
||||
activeRegionPolicy: options.activeRegionPolicy,
|
||||
});
|
||||
throwIfAborted(options.signal, "observation");
|
||||
return projectRecordSafeObservation({
|
||||
rootFrameId: captured.rootFrameId ?? documents[0]?.frameId ?? "root",
|
||||
frameDocuments: documents,
|
||||
rendered,
|
||||
surfaceProbes: captured.surfaceProbes,
|
||||
});
|
||||
}
|
||||
|
||||
async function handleVomObservation(
|
||||
manager: SessionManager,
|
||||
params: SnapshotParams | ObserveParams,
|
||||
@@ -1599,40 +1653,22 @@ async function handleVomObservation(
|
||||
try {
|
||||
throwIfAborted(signal, toolName);
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
throwIfAborted(signal, toolName);
|
||||
const effectiveConditionalSurfaceProbe =
|
||||
deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
|
||||
const captured = await captureForVom(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
effectiveConditionalSurfaceProbe,
|
||||
deps.hoverProbeBypassOverlay,
|
||||
signal,
|
||||
);
|
||||
throwIfAborted(signal, toolName);
|
||||
const frameDocuments = await captureFrameData<CdpAxNode>(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
captured,
|
||||
signal,
|
||||
);
|
||||
throwIfAborted(signal, toolName);
|
||||
const scene =
|
||||
captured.rootFrameId && captured.frameNodes?.size
|
||||
? buildFrameVomScene(frameDocuments, captured, { pageUrl: target.url })
|
||||
: buildVomScene(frameDocuments[0]?.axNodes ?? [], captured, { pageUrl: target.url });
|
||||
const rendered = renderVom(scene, {
|
||||
const observation = await captureVomObservation(deps.cdp, target.tabId, target.url, {
|
||||
maxDepth: params.max_depth,
|
||||
maxTokens: params.max_tokens,
|
||||
activeRegionPolicy: true,
|
||||
conditionalSurfaceProbe: effectiveConditionalSurfaceProbe,
|
||||
hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay,
|
||||
signal,
|
||||
});
|
||||
throwIfAborted(signal, toolName);
|
||||
const targetByFrameId = new Map(
|
||||
frameDocuments.map((document) => [document.frameId, document.target]),
|
||||
observation.frames.map((frame) => [frame.frameId, frame.target]),
|
||||
);
|
||||
ctx.refStore.replace(
|
||||
rendered.refs.map((ref) => {
|
||||
observation.refs.map((ref) => {
|
||||
const refTarget = ref.frameId ? targetByFrameId.get(ref.frameId) : undefined;
|
||||
return [
|
||||
ref.ref,
|
||||
@@ -1646,14 +1682,14 @@ async function handleVomObservation(
|
||||
}),
|
||||
);
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
text: rendered.text,
|
||||
ref_count: rendered.refs.length,
|
||||
text: observation.text,
|
||||
ref_count: observation.refs.length,
|
||||
tab_id: target.tabId,
|
||||
truncated: rendered.truncated,
|
||||
truncated: observation.truncated,
|
||||
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
|
||||
? {
|
||||
debug: {
|
||||
surface_probes: (captured.surfaceProbes ?? []).map((probe) => ({
|
||||
surface_probes: (observation.surfaceProbes ?? []).map((probe) => ({
|
||||
trigger_backend_node_id: probe.triggerBackendNodeId,
|
||||
...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}),
|
||||
trigger_action: probe.triggerAction,
|
||||
|
||||
@@ -17,11 +17,11 @@ import {
|
||||
type RecordStartMessage,
|
||||
type RecordStopMessage,
|
||||
} from "@/lib/record-bridge";
|
||||
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer";
|
||||
import { reduceTraceSteps, resolveTraceStartUrl } from "@/lib/trace-reducer";
|
||||
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording/step-buffer";
|
||||
import { buildTraceV2 } from "@/lib/recording/trace-reducer-v2";
|
||||
import type { RecordingDraftStep } from "@/lib/recording/types";
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
DraftTraceStep,
|
||||
RecordAwaitParams,
|
||||
RecordAwaitResult,
|
||||
RecordStartParams,
|
||||
@@ -29,7 +29,7 @@ import type {
|
||||
RecordStopParams,
|
||||
RecordStopResult,
|
||||
RpcError,
|
||||
Trace,
|
||||
TraceV2,
|
||||
} from "@/transport/types";
|
||||
import { handleNavigate } from "./navigation";
|
||||
import {
|
||||
@@ -47,11 +47,11 @@ interface ActiveRecording {
|
||||
agentWindowId: number;
|
||||
startUrl?: string;
|
||||
purpose?: string;
|
||||
steps: DraftTraceStep[];
|
||||
steps: RecordingDraftStep[];
|
||||
startedAt: string;
|
||||
startedAtMs: number;
|
||||
finishPromise: Promise<Trace>;
|
||||
resolveFinish: (trace: Trace) => void;
|
||||
finishPromise: Promise<TraceV2>;
|
||||
resolveFinish: (trace: TraceV2) => void;
|
||||
rejectFinish: (err: Error) => void;
|
||||
settled: boolean;
|
||||
finishing: boolean;
|
||||
@@ -154,17 +154,13 @@ async function sendRecordStartWithAck(
|
||||
throw lastError ?? new Error("failed to start recording in content script");
|
||||
}
|
||||
|
||||
function buildTrace(recording: ActiveRecording): Trace {
|
||||
const { pages, steps } = reduceTraceSteps(recording.steps, recording.startUrl);
|
||||
const startUrl = resolveTraceStartUrl(recording.steps, recording.startUrl, pages);
|
||||
return {
|
||||
function buildTrace(recording: ActiveRecording): TraceV2 {
|
||||
return buildTraceV2({
|
||||
steps: recording.steps,
|
||||
startedAt: recording.startedAt,
|
||||
...(recording.startUrl ? { startUrl: recording.startUrl } : {}),
|
||||
...(recording.purpose ? { purpose: recording.purpose } : {}),
|
||||
recorded_at: new Date().toISOString(),
|
||||
started_at: recording.startedAt,
|
||||
entry: { start_url: startUrl },
|
||||
pages,
|
||||
steps,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export interface RecordDeps {
|
||||
@@ -532,7 +528,7 @@ async function finishRecordingByRequest(
|
||||
}
|
||||
}
|
||||
|
||||
async function finishRecording(sessionId: string, deps: RecordDeps): Promise<Trace | null> {
|
||||
async function finishRecording(sessionId: string, deps: RecordDeps): Promise<TraceV2 | null> {
|
||||
const recording = recordings.get(sessionId);
|
||||
if (!recording || recording.settled || recording.finishing) return null;
|
||||
recording.finishing = true;
|
||||
@@ -574,9 +570,9 @@ export async function handleRecordStart(
|
||||
// on the destination page can RECORD_QUERY → rearm → show RecordOverlay
|
||||
// instead of flashing ControlOverlay ("Agent 正在控制").
|
||||
const requestId = makeRequestId(target.tabId);
|
||||
let resolveFinish!: (trace: Trace) => void;
|
||||
let resolveFinish!: (trace: TraceV2) => void;
|
||||
let rejectFinish!: (err: Error) => void;
|
||||
const finishPromise = new Promise<Trace>((resolve, reject) => {
|
||||
const finishPromise = new Promise<TraceV2>((resolve, reject) => {
|
||||
resolveFinish = resolve;
|
||||
rejectFinish = reject;
|
||||
});
|
||||
@@ -785,9 +781,9 @@ export async function handleRecordAwait(
|
||||
return { code: "cancelled", message: "record_await aborted" };
|
||||
}
|
||||
|
||||
const outcome = await new Promise<{ trace: Trace } | { error: RpcError }>((resolve) => {
|
||||
const outcome = await new Promise<{ trace: TraceV2 } | { error: RpcError }>((resolve) => {
|
||||
let settled = false;
|
||||
const finish = (result: { trace: Trace } | { error: RpcError }) => {
|
||||
const finish = (result: { trace: TraceV2 } | { error: RpcError }) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (timer) clearTimeout(timer);
|
||||
|
||||
@@ -76,6 +76,7 @@ export interface SessionStopResult {
|
||||
}
|
||||
|
||||
export interface SessionStopDeps {
|
||||
signal?: AbortSignal;
|
||||
cdp?: {
|
||||
detachSession(sessionId: string): Promise<void>;
|
||||
};
|
||||
@@ -177,6 +178,9 @@ export async function handleSessionStop(
|
||||
message: `session ${params.session_id} unknown`,
|
||||
};
|
||||
}
|
||||
if (deps.signal?.aborted) {
|
||||
return { code: "cancelled", message: "session_stop aborted before teardown" };
|
||||
}
|
||||
|
||||
// Step 1: auto-return borrowed tabs. Iterate over a snapshot of the
|
||||
// ids so deletions during iteration do not break the Map iterator.
|
||||
@@ -187,6 +191,7 @@ export async function handleSessionStop(
|
||||
try {
|
||||
const tabManagement = {
|
||||
...(deps.tabManagement ?? {}),
|
||||
signal: deps.signal,
|
||||
isAgentWindowId:
|
||||
deps.tabManagement?.isAgentWindowId ??
|
||||
((windowId: number) => manager.findByWindowId(windowId) !== null),
|
||||
@@ -214,6 +219,17 @@ export async function handleSessionStop(
|
||||
}
|
||||
}
|
||||
|
||||
if (deps.signal?.aborted) {
|
||||
const nonCancelFailures = returnFailures?.filter((failure) => failure.code !== "cancelled");
|
||||
if (nonCancelFailures && nonCancelFailures.length > 0) {
|
||||
return {
|
||||
...(returnedTabIds.length > 0 ? { returned_tab_ids: returnedTabIds } : {}),
|
||||
return_failures: nonCancelFailures,
|
||||
};
|
||||
}
|
||||
return { code: "cancelled", message: "session_stop aborted during tab return" };
|
||||
}
|
||||
|
||||
const result: SessionStopResult = {};
|
||||
if (returnedTabIds.length > 0) result.returned_tab_ids = returnedTabIds;
|
||||
if (returnFailures && returnFailures.length > 0) {
|
||||
@@ -233,6 +249,10 @@ export async function handleSessionStop(
|
||||
// Step 3: detach CDP sessions this session opened (no-op if none).
|
||||
await deps.cdp?.detachSession(params.session_id);
|
||||
|
||||
if (deps.signal?.aborted) {
|
||||
return { code: "cancelled", message: "session_stop aborted before window close" };
|
||||
}
|
||||
|
||||
// Step 4: close the Agent Window and drop the context.
|
||||
await manager.stop(params.session_id);
|
||||
|
||||
|
||||
@@ -77,9 +77,12 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
): FrameDocument<T>[] {
|
||||
const frames = frameList(graph, batches, captured);
|
||||
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
|
||||
const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId;
|
||||
const domNodesForFrame = (frameId: string): CapturedNode[] =>
|
||||
captured.frameNodes?.get(frameId) ?? (frameId === rootFrameId ? captured.nodes : []);
|
||||
const backendOwner = new Map<string, string>();
|
||||
for (const frame of frames) {
|
||||
for (const node of captured.frameNodes?.get(frame.frameId) ?? []) {
|
||||
for (const node of domNodesForFrame(frame.frameId)) {
|
||||
backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId);
|
||||
}
|
||||
}
|
||||
@@ -165,6 +168,6 @@ export function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
...frame,
|
||||
contextScopeId: frame.frameId,
|
||||
axNodes: axNodesByFrame.get(frame.frameId) ?? [],
|
||||
domNodes: captured.frameNodes?.get(frame.frameId) ?? [],
|
||||
domNodes: domNodesForFrame(frame.frameId),
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { CapturedSurfaceProbe } from "./capture";
|
||||
import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture";
|
||||
|
||||
/** Frame identity needed to resolve an `@eN` that lives in an iframe. */
|
||||
export interface CaptureVomFrame {
|
||||
frameId: string;
|
||||
target: CdpTarget;
|
||||
parentFrameId?: string;
|
||||
ownerBackendNodeId?: number;
|
||||
}
|
||||
|
||||
/** Allowlisted geometry used to match a recorded action to a rendered ref. */
|
||||
export interface CaptureVomMatchNode {
|
||||
frameId: string;
|
||||
backendNodeId: number;
|
||||
tag: string;
|
||||
/** Top-level viewport-relative geometry. */
|
||||
rect: Rect | null;
|
||||
/** Frame-local viewport-relative geometry, when available. */
|
||||
localRect?: Rect | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Raw AX/DOM trees are never returned. When `redactValues` is enabled by the
|
||||
* caller, form values in `text` are masked as well.
|
||||
*/
|
||||
export interface CaptureVomObservationResult {
|
||||
text: string;
|
||||
refs: RenderedRef[];
|
||||
truncated: boolean;
|
||||
rootFrameId: string;
|
||||
frames: CaptureVomFrame[];
|
||||
matchNodes: CaptureVomMatchNode[];
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
}
|
||||
|
||||
function projectFrames(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomFrame[] {
|
||||
return documents.map((document) => ({
|
||||
frameId: document.frameId,
|
||||
target: document.target,
|
||||
...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
|
||||
...(document.ownerBackendNodeId !== undefined
|
||||
? { ownerBackendNodeId: document.ownerBackendNodeId }
|
||||
: {}),
|
||||
}));
|
||||
}
|
||||
|
||||
function projectMatchNodes(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomMatchNode[] {
|
||||
return documents.flatMap((document) =>
|
||||
document.domNodes.map((node) => ({
|
||||
frameId: document.frameId,
|
||||
backendNodeId: node.backendNodeId,
|
||||
tag: node.tag,
|
||||
rect: node.rect,
|
||||
...(node.localRect !== undefined ? { localRect: node.localRect } : {}),
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
export function projectRecordSafeObservation(input: {
|
||||
rootFrameId: string;
|
||||
frameDocuments: CapturedFrameDocument<FrameAxNode>[];
|
||||
rendered: VomResult;
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
}): CaptureVomObservationResult {
|
||||
return {
|
||||
text: input.rendered.text,
|
||||
refs: input.rendered.refs,
|
||||
truncated: input.rendered.truncated,
|
||||
rootFrameId: input.rootFrameId,
|
||||
frames: projectFrames(input.frameDocuments),
|
||||
matchNodes: projectMatchNodes(input.frameDocuments),
|
||||
surfaceProbes: input.surfaceProbes,
|
||||
};
|
||||
}
|
||||
@@ -657,10 +657,60 @@ export interface EmulateResult {
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Semantic record payloads — mirror bsk-protocol record.rs (Trace v2)
|
||||
// Semantic record payloads mirror the versioned Rust protocol models.
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
export interface TargetDescriptor {
|
||||
export const TRACE_VERSION_V3 = 3;
|
||||
/** Logical v2 identifier. Not a wire field — v2 envelopes omit `version`. */
|
||||
export const TRACE_VERSION_V2 = 2;
|
||||
export const VOM_FORMAT_VERSION = 1;
|
||||
|
||||
export interface TargetDescriptorV3 {
|
||||
ref?: string;
|
||||
role?: string;
|
||||
name?: string;
|
||||
ctx?: string;
|
||||
unmatched?: boolean;
|
||||
}
|
||||
|
||||
export interface RecorderInfo {
|
||||
bsk: string;
|
||||
vom: number;
|
||||
}
|
||||
|
||||
export type StopReason = "user_finish" | "cli_stop";
|
||||
|
||||
export interface TraceStateV3 {
|
||||
id: string;
|
||||
url: string;
|
||||
title?: string;
|
||||
body: string;
|
||||
truncated?: boolean;
|
||||
}
|
||||
|
||||
export interface StepResultV3 {
|
||||
state: string;
|
||||
}
|
||||
|
||||
export interface StepCommonV3 {
|
||||
id: number;
|
||||
state: string;
|
||||
result: StepResultV3;
|
||||
}
|
||||
|
||||
export type NavigationCause =
|
||||
| "user_typed"
|
||||
| "link"
|
||||
| "form_submit"
|
||||
| "reload"
|
||||
| "history"
|
||||
| "script"
|
||||
| "browser";
|
||||
|
||||
export type FillCommit = "enter" | "suggestion" | "blur";
|
||||
|
||||
/** Legacy v2 target shape retained for existing record producers. */
|
||||
export interface TargetDescriptorV2 {
|
||||
role?: string;
|
||||
name?: string;
|
||||
tag: string;
|
||||
@@ -673,98 +723,97 @@ export interface TraceEntry {
|
||||
start_url: string;
|
||||
}
|
||||
|
||||
export interface PageRef {
|
||||
export interface PageRefV2 {
|
||||
id: string;
|
||||
url: string;
|
||||
title?: string;
|
||||
}
|
||||
|
||||
export interface SelectedOption {
|
||||
export interface SelectedOptionV2 {
|
||||
value: string;
|
||||
label?: string;
|
||||
}
|
||||
|
||||
export interface StepEffect {
|
||||
export interface StepEffectV2 {
|
||||
navigated_to: string;
|
||||
}
|
||||
|
||||
export interface StepCommon {
|
||||
export interface StepCommonV2 {
|
||||
id: number;
|
||||
page: string;
|
||||
effect?: StepEffect;
|
||||
effect?: StepEffectV2;
|
||||
}
|
||||
|
||||
/** Capture/buffer draft before v2 reduction. */
|
||||
export type DraftTraceStep =
|
||||
| {
|
||||
op: "click";
|
||||
target: TargetDescriptor;
|
||||
navigated_to?: string;
|
||||
page_url?: string;
|
||||
}
|
||||
| {
|
||||
op: "hover";
|
||||
target: TargetDescriptor;
|
||||
page_url?: string;
|
||||
}
|
||||
| {
|
||||
op: "fill";
|
||||
target: TargetDescriptor;
|
||||
value: string;
|
||||
redacted?: boolean;
|
||||
page_url?: string;
|
||||
}
|
||||
| {
|
||||
op: "press";
|
||||
key: string;
|
||||
target?: TargetDescriptor;
|
||||
modifiers?: KeyModifier[];
|
||||
navigated_to?: string;
|
||||
page_url?: string;
|
||||
}
|
||||
| {
|
||||
op: "select";
|
||||
target: TargetDescriptor;
|
||||
values: string[];
|
||||
labels?: string[];
|
||||
navigated_to?: string;
|
||||
page_url?: string;
|
||||
}
|
||||
| {
|
||||
op: "navigate";
|
||||
url: string;
|
||||
page_url?: string;
|
||||
};
|
||||
|
||||
/** Exported record-only step (trace v2). */
|
||||
export type Step =
|
||||
| ({ op: "navigate" } & StepCommon & { to: string })
|
||||
| ({ op: "click" } & StepCommon & { target: TargetDescriptor })
|
||||
| ({ op: "hover" } & StepCommon & { target: TargetDescriptor })
|
||||
| ({ op: "fill" } & StepCommon & {
|
||||
target: TargetDescriptor;
|
||||
export type StepV2 =
|
||||
| ({ op: "navigate" } & StepCommonV2 & { to: string })
|
||||
| ({ op: "click" } & StepCommonV2 & { target: TargetDescriptorV2 })
|
||||
| ({ op: "hover" } & StepCommonV2 & { target: TargetDescriptorV2 })
|
||||
| ({ op: "fill" } & StepCommonV2 & {
|
||||
target: TargetDescriptorV2;
|
||||
value: string;
|
||||
redacted?: boolean;
|
||||
})
|
||||
| ({ op: "select" } & StepCommon & {
|
||||
target: TargetDescriptor;
|
||||
selection: SelectedOption[];
|
||||
| ({ op: "select" } & StepCommonV2 & {
|
||||
target: TargetDescriptorV2;
|
||||
selection: SelectedOptionV2[];
|
||||
})
|
||||
| ({ op: "press" } & StepCommon & {
|
||||
| ({ op: "press" } & StepCommonV2 & {
|
||||
key: string;
|
||||
modifiers?: KeyModifier[];
|
||||
target?: TargetDescriptor;
|
||||
target?: TargetDescriptorV2;
|
||||
});
|
||||
|
||||
export interface Trace {
|
||||
export interface TraceV2 {
|
||||
recorded_at: string;
|
||||
started_at?: string;
|
||||
purpose?: string;
|
||||
entry: TraceEntry;
|
||||
pages: PageRef[];
|
||||
steps: Step[];
|
||||
pages: PageRefV2[];
|
||||
steps: StepV2[];
|
||||
}
|
||||
|
||||
export interface SelectedOptionV3 {
|
||||
value: string;
|
||||
label?: string;
|
||||
}
|
||||
|
||||
export type StepV3 =
|
||||
| ({ op: "navigate" } & StepCommonV3 & { to: string; cause: NavigationCause })
|
||||
| ({ op: "click" } & StepCommonV3 & { target: TargetDescriptorV3 })
|
||||
| ({ op: "hover" } & StepCommonV3 & { target: TargetDescriptorV3 })
|
||||
| ({ op: "fill" } & StepCommonV3 & {
|
||||
target: TargetDescriptorV3;
|
||||
value: string;
|
||||
commit: FillCommit;
|
||||
redacted?: boolean;
|
||||
})
|
||||
| ({ op: "select" } & StepCommonV3 & {
|
||||
target: TargetDescriptorV3;
|
||||
selection?: SelectedOptionV3[];
|
||||
})
|
||||
| ({ op: "press" } & StepCommonV3 & {
|
||||
key: string;
|
||||
modifiers?: KeyModifier[];
|
||||
target?: TargetDescriptorV3;
|
||||
})
|
||||
| ({ op: "scroll" } & StepCommonV3);
|
||||
|
||||
export interface TraceV3 {
|
||||
version: typeof TRACE_VERSION_V3;
|
||||
recorded_at: string;
|
||||
started_at?: string;
|
||||
purpose?: string;
|
||||
stopped_by: StopReason;
|
||||
entry: TraceEntry;
|
||||
recorder: RecorderInfo;
|
||||
states: TraceStateV3[];
|
||||
steps: StepV3[];
|
||||
}
|
||||
|
||||
export type RecordedTrace = TraceV2 | TraceV3;
|
||||
export type RecordedStep = StepV2 | StepV3;
|
||||
|
||||
export interface RecordStartParams {
|
||||
session_id: string;
|
||||
tab_id?: number;
|
||||
@@ -782,7 +831,7 @@ export interface RecordStopParams {
|
||||
}
|
||||
|
||||
export interface RecordStopResult {
|
||||
trace: Trace;
|
||||
trace: RecordedTrace;
|
||||
}
|
||||
|
||||
export interface RecordAwaitParams {
|
||||
@@ -791,5 +840,5 @@ export interface RecordAwaitParams {
|
||||
}
|
||||
|
||||
export interface RecordAwaitResult {
|
||||
trace: Trace;
|
||||
trace: RecordedTrace;
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ use anyhow::Context;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{
|
||||
RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult, RecordStopParams,
|
||||
RecordStopResult, Trace,
|
||||
RecordStopResult, RecordedTrace,
|
||||
};
|
||||
use clap::{Args, Subcommand};
|
||||
|
||||
@@ -190,7 +190,7 @@ fn record_await_ipc_timeout(timeout_ms: u32) -> Duration {
|
||||
.unwrap_or(Duration::from_secs(u64::from(timeout_ms / 1_000) + 15))
|
||||
}
|
||||
|
||||
fn write_trace_file(output: &PathBuf, trace: &Trace) -> Result<(), CliError> {
|
||||
fn write_trace_file(output: &PathBuf, trace: &RecordedTrace) -> Result<(), CliError> {
|
||||
let json = serde_json::to_string_pretty(trace)
|
||||
.context("serialize trace JSON")
|
||||
.map_err(CliError::Local)?;
|
||||
@@ -207,7 +207,7 @@ fn write_trace_file(output: &PathBuf, trace: &Trace) -> Result<(), CliError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn render_finish(trace: &Trace, output: &PathBuf, format: Format) -> Result<(), CliError> {
|
||||
fn render_finish(trace: &RecordedTrace, output: &PathBuf, format: Format) -> Result<(), CliError> {
|
||||
match format {
|
||||
Format::Json => {
|
||||
println!(
|
||||
@@ -221,7 +221,11 @@ fn render_finish(trace: &Trace, output: &PathBuf, format: Format) -> Result<(),
|
||||
);
|
||||
}
|
||||
Format::Human => {
|
||||
println!("saved {} steps to {}", trace.steps.len(), output.display());
|
||||
let step_count = match trace {
|
||||
RecordedTrace::V2(trace) => trace.steps.len(),
|
||||
RecordedTrace::V3(trace) => trace.steps.len(),
|
||||
};
|
||||
println!("saved {step_count} steps to {}", output.display());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
|
||||
@@ -206,15 +206,15 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
||||
Ok(v) => ResponseBody::Ok(v),
|
||||
Err(e) => ResponseBody::Err(e),
|
||||
},
|
||||
Method::SessionStart => match handle_session_start(&state, params).await {
|
||||
Method::SessionStart => match handle_session_start(&state, rpc_id, params).await {
|
||||
Ok(v) => ResponseBody::Ok(v),
|
||||
Err(e) => ResponseBody::Err(e),
|
||||
},
|
||||
Method::SessionStop => match handle_session_stop(&state, params).await {
|
||||
Method::SessionStop => match handle_session_stop(&state, rpc_id, params).await {
|
||||
Ok(v) => ResponseBody::Ok(v),
|
||||
Err(e) => ResponseBody::Err(e),
|
||||
},
|
||||
Method::SessionStopAll => match handle_session_stop_all(&state).await {
|
||||
Method::SessionStopAll => match handle_session_stop_all_rpc(&state, rpc_id).await {
|
||||
Ok(v) => ResponseBody::Ok(v),
|
||||
Err(e) => ResponseBody::Err(e),
|
||||
},
|
||||
@@ -434,8 +434,9 @@ async fn handle_wait_ms(
|
||||
/// cancellation surfaces (M10.2 + review C2):
|
||||
///
|
||||
/// 1. [`AbortRegistry`] — answers daemon-local cancellable runners
|
||||
/// (currently `tool.wait_ms`). If a token is registered we trip it
|
||||
/// and stop.
|
||||
/// (`tool.wait_ms` and `session.*` lifecycle calls). If a token is
|
||||
/// registered we trip it and stop. Lifecycle handlers forward their
|
||||
/// own WS cancel after preserving request-before-cancel ordering.
|
||||
/// 2. [`super::inflight::ToolInflightRegistry`] — every IPC-tracked
|
||||
/// `tool.*` RPC, registered the moment the IPC handler accepts the
|
||||
/// request. Trips the entry's cancel token regardless of whether
|
||||
@@ -634,7 +635,20 @@ fn clamp_browser_wait(wait_ms: Option<u64>) -> Option<Duration> {
|
||||
))
|
||||
}
|
||||
|
||||
async fn handle_session_start(state: &Arc<DaemonState>, params: Value) -> Result<Value, RpcError> {
|
||||
async fn handle_session_start(
|
||||
state: &Arc<DaemonState>,
|
||||
rpc_id: RpcId,
|
||||
params: Value,
|
||||
) -> Result<Value, RpcError> {
|
||||
let abort_guard = state
|
||||
.abort_registry
|
||||
.register(rpc_id)
|
||||
.map_err(|err| RpcError {
|
||||
code: ErrorCode::ProtocolError,
|
||||
message: format!("session.start cancellation registration failed: {err:?}"),
|
||||
data: None,
|
||||
})?;
|
||||
let cancel = abort_guard.token().clone();
|
||||
let params: CliSessionStartParams = if params.is_null() {
|
||||
CliSessionStartParams {
|
||||
browser_instance_id: None,
|
||||
@@ -673,6 +687,7 @@ async fn handle_session_start(state: &Arc<DaemonState>, params: Value) -> Result
|
||||
},
|
||||
state.config.extension_connect_wait,
|
||||
DEFAULT_RPC_TIMEOUT,
|
||||
Some(cancel),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -696,6 +711,8 @@ fn map_start_error(err: StartSessionError) -> RpcError {
|
||||
StartSessionError::AmbiguousBrowserLabel { .. } => ErrorCode::InvalidParams,
|
||||
StartSessionError::IdExhausted => ErrorCode::ProtocolError,
|
||||
StartSessionError::Timeout => ErrorCode::Timeout,
|
||||
StartSessionError::Cancelled => ErrorCode::Cancelled,
|
||||
StartSessionError::CleanupFailed { .. } => ErrorCode::ProtocolError,
|
||||
StartSessionError::TransportClosed => ErrorCode::ProtocolError,
|
||||
StartSessionError::ExtensionError(inner) => inner.code,
|
||||
};
|
||||
@@ -711,6 +728,16 @@ fn map_start_error(err: StartSessionError) -> RpcError {
|
||||
"label": label,
|
||||
"instance_ids": instance_ids,
|
||||
})),
|
||||
StartSessionError::CleanupFailed {
|
||||
session_id,
|
||||
agent_window_id,
|
||||
..
|
||||
} => Some(serde_json::json!({
|
||||
"reason": "cleanup_failed",
|
||||
"session_id": session_id.0,
|
||||
"agent_window_id": agent_window_id,
|
||||
})),
|
||||
StartSessionError::ExtensionError(inner) => inner.data.clone(),
|
||||
_ => None,
|
||||
};
|
||||
RpcError {
|
||||
@@ -720,7 +747,20 @@ fn map_start_error(err: StartSessionError) -> RpcError {
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_session_stop(state: &Arc<DaemonState>, params: Value) -> Result<Value, RpcError> {
|
||||
async fn handle_session_stop(
|
||||
state: &Arc<DaemonState>,
|
||||
rpc_id: RpcId,
|
||||
params: Value,
|
||||
) -> Result<Value, RpcError> {
|
||||
let abort_guard = state
|
||||
.abort_registry
|
||||
.register(rpc_id)
|
||||
.map_err(|err| RpcError {
|
||||
code: ErrorCode::ProtocolError,
|
||||
message: format!("session.stop cancellation registration failed: {err:?}"),
|
||||
data: None,
|
||||
})?;
|
||||
let cancel = abort_guard.token().clone();
|
||||
let params: CliSessionStopParams = if params.is_null() {
|
||||
CliSessionStopParams {
|
||||
session_id: None,
|
||||
@@ -734,7 +774,7 @@ async fn handle_session_stop(state: &Arc<DaemonState>, params: Value) -> Result<
|
||||
})?
|
||||
};
|
||||
if params.all {
|
||||
return handle_session_stop_all(state).await;
|
||||
return handle_session_stop_all(state, Some(cancel)).await;
|
||||
}
|
||||
let session_id = match params.session_id {
|
||||
Some(s) => SessionId(s),
|
||||
@@ -753,6 +793,7 @@ async fn handle_session_stop(state: &Arc<DaemonState>, params: Value) -> Result<
|
||||
&state.session_interrupts,
|
||||
&session_id,
|
||||
DEFAULT_SESSION_STOP_TIMEOUT,
|
||||
Some(cancel),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -804,7 +845,25 @@ fn map_stop_error(err: StopSessionError) -> RpcError {
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_session_stop_all(state: &Arc<DaemonState>) -> Result<Value, RpcError> {
|
||||
async fn handle_session_stop_all_rpc(
|
||||
state: &Arc<DaemonState>,
|
||||
rpc_id: RpcId,
|
||||
) -> Result<Value, RpcError> {
|
||||
let abort_guard = state
|
||||
.abort_registry
|
||||
.register(rpc_id)
|
||||
.map_err(|err| RpcError {
|
||||
code: ErrorCode::ProtocolError,
|
||||
message: format!("session.stop_all cancellation registration failed: {err:?}"),
|
||||
data: None,
|
||||
})?;
|
||||
handle_session_stop_all(state, Some(abort_guard.token().clone())).await
|
||||
}
|
||||
|
||||
async fn handle_session_stop_all(
|
||||
state: &Arc<DaemonState>,
|
||||
cancel: Option<super::abort::AbortToken>,
|
||||
) -> Result<Value, RpcError> {
|
||||
let ids: Vec<SessionId> = state
|
||||
.sessions
|
||||
.snapshot()
|
||||
@@ -816,6 +875,16 @@ async fn handle_session_stop_all(state: &Arc<DaemonState>) -> Result<Value, RpcE
|
||||
let mut returned_tab_ids = Vec::new();
|
||||
let mut return_failures = Vec::new();
|
||||
for id in ids {
|
||||
if cancel
|
||||
.as_ref()
|
||||
.is_some_and(super::abort::AbortToken::is_cancelled)
|
||||
{
|
||||
return Err(RpcError {
|
||||
code: ErrorCode::Cancelled,
|
||||
message: "session.stop_all was cancelled".into(),
|
||||
data: None,
|
||||
});
|
||||
}
|
||||
match stop_session(
|
||||
&state.browsers,
|
||||
&state.sessions,
|
||||
@@ -823,6 +892,7 @@ async fn handle_session_stop_all(state: &Arc<DaemonState>) -> Result<Value, RpcE
|
||||
&state.session_interrupts,
|
||||
&id,
|
||||
DEFAULT_SESSION_STOP_TIMEOUT,
|
||||
cancel.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -848,6 +918,15 @@ async fn handle_session_stop_all(state: &Arc<DaemonState>) -> Result<Value, RpcE
|
||||
returned_tab_ids.extend(stop.returned_tab_ids);
|
||||
return_failures.extend(stop.return_failures);
|
||||
}
|
||||
Err(err)
|
||||
if matches!(
|
||||
&err,
|
||||
StopSessionError::ExtensionError(inner)
|
||||
if inner.code == ErrorCode::Cancelled
|
||||
) =>
|
||||
{
|
||||
return Err(map_stop_error(err));
|
||||
}
|
||||
Err(err) => {
|
||||
debug!(session = %id, ?err, "session.stop_all: failure (continuing)");
|
||||
let rpc = map_stop_error(err);
|
||||
|
||||
@@ -43,6 +43,7 @@ use serde_json::Value;
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use super::abort::AbortToken;
|
||||
use super::browsers::BrowserRegistry;
|
||||
use super::inflight::{PromoteOutcome, ToolInflightEntry};
|
||||
use super::sessions::{SessionId, SessionRegistry};
|
||||
@@ -83,6 +84,14 @@ pub struct ToolJob {
|
||||
/// IPC request id (e.g. `session.stop`'s queued teardown call —
|
||||
/// those already carry their own bespoke retry / abort path).
|
||||
pub inflight: Option<Arc<ToolInflightEntry>>,
|
||||
/// Cancellation token for session-lifecycle work registered in the
|
||||
/// daemon-local abort registry. Unlike `inflight`, lifecycle callers
|
||||
/// forward their own WS cancel after the request has been queued.
|
||||
pub lifecycle_cancel: Option<AbortToken>,
|
||||
/// How long cancellation keeps the worker busy while the extension
|
||||
/// finishes compensation. Session teardown uses its full RPC budget
|
||||
/// so daemon state is reconciled even after the CLI exits.
|
||||
pub cancel_cleanup_timeout: Duration,
|
||||
/// Owning session — used by `ToolInflightRegistry::cancel_session`
|
||||
/// to drain queued jobs that have not yet been promoted to
|
||||
/// "forwarded". Worker callers also have access to it via the
|
||||
@@ -322,6 +331,8 @@ impl ToolQueueRegistry {
|
||||
timeout,
|
||||
false,
|
||||
inflight,
|
||||
None,
|
||||
CANCEL_CLEANUP_TIMEOUT,
|
||||
)
|
||||
.await;
|
||||
// A long-running request may outlive the idle threshold. Touching
|
||||
@@ -360,6 +371,8 @@ impl ToolQueueRegistry {
|
||||
timeout,
|
||||
respond: respond_tx,
|
||||
inflight,
|
||||
lifecycle_cancel: None,
|
||||
cancel_cleanup_timeout: CANCEL_CLEANUP_TIMEOUT,
|
||||
session_id: sid.clone(),
|
||||
};
|
||||
match forward_one(sid, &self.browsers, &self.sessions, &job).await {
|
||||
@@ -378,6 +391,7 @@ impl ToolQueueRegistry {
|
||||
method: Method,
|
||||
params: Value,
|
||||
timeout: Duration,
|
||||
cancel: Option<AbortToken>,
|
||||
) -> Result<Value, DispatchError> {
|
||||
let (sender, state) = {
|
||||
let mut guard = self.queues.lock().expect("tool queue registry poisoned");
|
||||
@@ -404,6 +418,8 @@ impl ToolQueueRegistry {
|
||||
timeout,
|
||||
true,
|
||||
None,
|
||||
cancel,
|
||||
timeout,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -419,6 +435,8 @@ async fn dispatch_with_sender(
|
||||
timeout: Duration,
|
||||
wait_for_capacity: bool,
|
||||
inflight: Option<Arc<ToolInflightEntry>>,
|
||||
lifecycle_cancel: Option<AbortToken>,
|
||||
cancel_cleanup_timeout: Duration,
|
||||
) -> Result<Value, DispatchError> {
|
||||
let (respond_tx, respond_rx) = oneshot::channel();
|
||||
let job = ToolJob {
|
||||
@@ -427,8 +445,11 @@ async fn dispatch_with_sender(
|
||||
timeout,
|
||||
respond: respond_tx,
|
||||
inflight,
|
||||
lifecycle_cancel,
|
||||
cancel_cleanup_timeout,
|
||||
session_id,
|
||||
};
|
||||
let lifecycle_cancellable = job.lifecycle_cancel.is_some();
|
||||
if wait_for_capacity {
|
||||
let waited = tokio::time::timeout(
|
||||
timeout.saturating_add(Duration::from_secs(1)),
|
||||
@@ -453,8 +474,15 @@ async fn dispatch_with_sender(
|
||||
mpsc::error::TrySendError::Closed(_) => DispatchError::QueueClosed,
|
||||
});
|
||||
}
|
||||
let waited =
|
||||
tokio::time::timeout(timeout.saturating_add(Duration::from_secs(1)), respond_rx).await;
|
||||
let mut response_timeout = timeout.saturating_add(Duration::from_secs(1));
|
||||
if lifecycle_cancellable {
|
||||
// A lifecycle cancel can arrive at the original RPC deadline and
|
||||
// intentionally keeps the worker alive while extension cleanup
|
||||
// settles. Do not let this outer waiter abandon that reconciliation
|
||||
// early, which could reopen a daemon session whose window is gone.
|
||||
response_timeout = response_timeout.saturating_add(cancel_cleanup_timeout);
|
||||
}
|
||||
let waited = tokio::time::timeout(response_timeout, respond_rx).await;
|
||||
match waited {
|
||||
Ok(Ok(Ok(v))) => Ok(v),
|
||||
Ok(Ok(Err(rpc))) => Err(DispatchError::Rpc(rpc)),
|
||||
@@ -492,11 +520,17 @@ async fn forward_one(
|
||||
// / browser resolution work, and before any WS frame leaves the
|
||||
// daemon (review C2). We keep the `tokio::select!` later so
|
||||
// cancels arriving mid-WS-hop also unblock the worker.
|
||||
if let Some(entry) = job.inflight.as_ref()
|
||||
&& entry.is_cancelled()
|
||||
if job
|
||||
.inflight
|
||||
.as_ref()
|
||||
.is_some_and(|entry| entry.is_cancelled())
|
||||
|| job
|
||||
.lifecycle_cancel
|
||||
.as_ref()
|
||||
.is_some_and(AbortToken::is_cancelled)
|
||||
{
|
||||
return Err(cancelled_error(
|
||||
Some(entry),
|
||||
job.inflight.as_deref(),
|
||||
"tool dispatch cancelled before forwarding",
|
||||
));
|
||||
}
|
||||
@@ -560,11 +594,20 @@ async fn forward_one(
|
||||
}
|
||||
}
|
||||
None => {
|
||||
// Daemon-internal callers (e.g. queued session.stop drain)
|
||||
// don't carry an inflight entry; they cannot be cancelled
|
||||
// through the IPC `cancel` surface, so a plain sink send
|
||||
// is sufficient and keeps the existing transport-error
|
||||
// behaviour.
|
||||
// Daemon-internal callers do not carry a tool-inflight entry.
|
||||
// Session lifecycle jobs instead use the abort-registry token
|
||||
// below; other internal callers keep the plain send path.
|
||||
if job
|
||||
.lifecycle_cancel
|
||||
.as_ref()
|
||||
.is_some_and(AbortToken::is_cancelled)
|
||||
{
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(cancelled_error(
|
||||
None,
|
||||
"session lifecycle cancelled before forwarding",
|
||||
));
|
||||
}
|
||||
if client.sink.send(request).is_err() {
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(RpcError {
|
||||
@@ -573,19 +616,45 @@ async fn forward_one(
|
||||
data: None,
|
||||
});
|
||||
}
|
||||
None
|
||||
job.lifecycle_cancel.clone()
|
||||
}
|
||||
};
|
||||
let forward_lifecycle_cancel = || {
|
||||
if job.lifecycle_cancel.is_none() {
|
||||
return;
|
||||
}
|
||||
let cancel = Frame::Request(RequestFrame {
|
||||
id: format!("cancel-{rpc_id}"),
|
||||
method: Method::Cancel,
|
||||
params: Some(serde_json::json!({ "rpc_id": rpc_id })),
|
||||
});
|
||||
if let Err(err) = client.sink.send(cancel) {
|
||||
warn!(%rpc_id, ?err, "failed to forward session lifecycle cancel");
|
||||
}
|
||||
};
|
||||
let on_abort: Option<&(dyn Fn() + Sync)> = job
|
||||
.lifecycle_cancel
|
||||
.as_ref()
|
||||
.map(|_| &forward_lifecycle_cancel as &(dyn Fn() + Sync));
|
||||
let waited = await_with_optional_cancel(
|
||||
job.timeout,
|
||||
CANCEL_CLEANUP_TIMEOUT,
|
||||
job.cancel_cleanup_timeout,
|
||||
waiter,
|
||||
cancel_token.as_ref(),
|
||||
on_abort,
|
||||
)
|
||||
.await;
|
||||
let response = match waited {
|
||||
WaitOutcome::Response(resp) => resp,
|
||||
WaitOutcome::CancelledAfterResponse(resp) => {
|
||||
if job.method == Method::ToolSessionStop
|
||||
&& let ResponseBody::Ok(value) = resp.body
|
||||
{
|
||||
// Teardown crossed its final irreversible boundary before
|
||||
// the cancel landed. Commit the real extension result so the
|
||||
// daemon does not retain a session whose window is gone.
|
||||
return Ok(value);
|
||||
}
|
||||
// Cancellation wins the external verdict, but only after the
|
||||
// extension's original RPC has settled. Preserve any non-cancel
|
||||
// error so compensation failures remain explicit instead of
|
||||
@@ -600,13 +669,33 @@ async fn forward_one(
|
||||
"tool dispatch cancelled after extension cleanup",
|
||||
));
|
||||
}
|
||||
WaitOutcome::TimedOutAfterResponse(resp) => match resp.body {
|
||||
ResponseBody::Ok(value) if job.method == Method::ToolSessionStop => {
|
||||
// The close crossed its irreversible boundary during the
|
||||
// timeout cleanup grace. Reconcile daemon state instead of
|
||||
// reopening a session whose window is already gone.
|
||||
return Ok(value);
|
||||
}
|
||||
ResponseBody::Err(err)
|
||||
if !matches!(err.code, ErrorCode::Cancelled | ErrorCode::UserAborted) =>
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
_ => {
|
||||
return Err(RpcError {
|
||||
code: ErrorCode::Timeout,
|
||||
message: format!("tool RPC timed out after {:?}", job.timeout),
|
||||
data: None,
|
||||
});
|
||||
}
|
||||
},
|
||||
WaitOutcome::CleanupTimeout => {
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(RpcError {
|
||||
code: ErrorCode::Timeout,
|
||||
message: format!(
|
||||
"cancelled tool did not finish cleanup within {:?}",
|
||||
CANCEL_CLEANUP_TIMEOUT
|
||||
job.cancel_cleanup_timeout
|
||||
),
|
||||
data: Some(serde_json::json!({ "reason": "cancel_cleanup_timeout" })),
|
||||
});
|
||||
@@ -638,6 +727,7 @@ async fn forward_one(
|
||||
enum WaitOutcome {
|
||||
Response(bsk_protocol::ResponseFrame),
|
||||
CancelledAfterResponse(bsk_protocol::ResponseFrame),
|
||||
TimedOutAfterResponse(bsk_protocol::ResponseFrame),
|
||||
CleanupTimeout,
|
||||
WaiterClosed,
|
||||
Timeout,
|
||||
@@ -648,6 +738,7 @@ async fn await_with_optional_cancel(
|
||||
cleanup_timeout: Duration,
|
||||
mut waiter: oneshot::Receiver<bsk_protocol::ResponseFrame>,
|
||||
cancel: Option<&super::abort::AbortToken>,
|
||||
on_abort: Option<&(dyn Fn() + Sync)>,
|
||||
) -> WaitOutcome {
|
||||
match cancel {
|
||||
Some(token) => {
|
||||
@@ -659,6 +750,9 @@ async fn await_with_optional_cancel(
|
||||
// after compensation or the cleanup deadline expires.
|
||||
biased;
|
||||
_ = token.cancelled() => {
|
||||
if let Some(on_abort) = on_abort {
|
||||
on_abort();
|
||||
}
|
||||
match tokio::time::timeout(cleanup_timeout, &mut waiter).await {
|
||||
Ok(Ok(resp)) => WaitOutcome::CancelledAfterResponse(resp),
|
||||
Ok(Err(_)) => WaitOutcome::WaiterClosed,
|
||||
@@ -669,7 +763,17 @@ async fn await_with_optional_cancel(
|
||||
Ok(resp) => WaitOutcome::Response(resp),
|
||||
Err(_) => WaitOutcome::WaiterClosed,
|
||||
},
|
||||
_ = &mut deadline => WaitOutcome::Timeout,
|
||||
_ = &mut deadline => {
|
||||
let Some(on_abort) = on_abort else {
|
||||
return WaitOutcome::Timeout;
|
||||
};
|
||||
on_abort();
|
||||
match tokio::time::timeout(cleanup_timeout, &mut waiter).await {
|
||||
Ok(Ok(resp)) => WaitOutcome::TimedOutAfterResponse(resp),
|
||||
Ok(Err(_)) => WaitOutcome::WaiterClosed,
|
||||
Err(_) => WaitOutcome::CleanupTimeout,
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
None => match tokio::time::timeout(timeout, &mut waiter).await {
|
||||
@@ -759,6 +863,7 @@ mod await_with_optional_cancel_tests {
|
||||
Duration::from_secs(1),
|
||||
rx,
|
||||
Some(&token),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
@@ -781,6 +886,7 @@ mod await_with_optional_cancel_tests {
|
||||
Duration::from_secs(1),
|
||||
rx,
|
||||
Some(&token),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
match outcome {
|
||||
@@ -795,9 +901,14 @@ mod await_with_optional_cancel_tests {
|
||||
async fn no_cancel_token_still_returns_response() {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
tx.send(dummy_response()).unwrap();
|
||||
let outcome =
|
||||
await_with_optional_cancel(Duration::from_secs(10), Duration::from_secs(1), rx, None)
|
||||
.await;
|
||||
let outcome = await_with_optional_cancel(
|
||||
Duration::from_secs(10),
|
||||
Duration::from_secs(1),
|
||||
rx,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(outcome, WaitOutcome::Response(_)));
|
||||
}
|
||||
|
||||
@@ -812,6 +923,7 @@ mod await_with_optional_cancel_tests {
|
||||
Duration::from_secs(1),
|
||||
rx,
|
||||
Some(&token),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
});
|
||||
@@ -835,10 +947,37 @@ mod await_with_optional_cancel_tests {
|
||||
Duration::from_millis(10),
|
||||
rx,
|
||||
Some(&token),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(outcome, WaitOutcome::CleanupTimeout));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lifecycle_timeout_forwards_abort_and_waits_for_cleanup_response() {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
let token = AbortToken::new();
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let abort_forwarded = AtomicBool::new(false);
|
||||
let forward_abort = || abort_forwarded.store(true, Ordering::SeqCst);
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
tx.send(dummy_response()).unwrap();
|
||||
});
|
||||
|
||||
let outcome = await_with_optional_cancel(
|
||||
Duration::from_millis(5),
|
||||
Duration::from_millis(100),
|
||||
rx,
|
||||
Some(&token),
|
||||
Some(&forward_abort),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(abort_forwarded.load(Ordering::SeqCst));
|
||||
assert!(matches!(outcome, WaitOutcome::TimedOutAfterResponse(_)));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -901,6 +1040,8 @@ mod tool_job_session_id_tests {
|
||||
timeout: Duration::from_secs(1),
|
||||
respond: tx,
|
||||
inflight: None,
|
||||
lifecycle_cancel: None,
|
||||
cancel_cleanup_timeout: CANCEL_CLEANUP_TIMEOUT,
|
||||
session_id: SessionId("sess-A".into()),
|
||||
};
|
||||
assert_eq!(job.session_id.0, "sess-A");
|
||||
|
||||
@@ -12,13 +12,14 @@ use bsk_protocol::system::{BrowserStatusEntry, SessionStatusEntry};
|
||||
use bsk_protocol::tools::{
|
||||
SessionStartParams, SessionStartResult, SessionStopParams, SessionStopResult,
|
||||
};
|
||||
use bsk_protocol::{Frame, RequestFrame, ResponseBody, RpcError, RpcId};
|
||||
use bsk_protocol::{ErrorCode, Frame, Method, RequestFrame, ResponseBody, RpcError, RpcId};
|
||||
use rand::Rng;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::time::{Duration, timeout};
|
||||
|
||||
use super::abort::AbortToken;
|
||||
use super::browsers::{BrowserClient, BrowserId, BrowserRegistry, SelectError};
|
||||
use super::queue::{DispatchError, ToolQueueRegistry};
|
||||
use super::queue::{CANCEL_CLEANUP_TIMEOUT, DispatchError, ToolQueueRegistry};
|
||||
use super::session_interrupt::SessionInterruptRegistry;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
@@ -339,6 +340,14 @@ pub enum StartSessionError {
|
||||
IdExhausted,
|
||||
#[error("session creation timed out waiting for extension")]
|
||||
Timeout,
|
||||
#[error("session creation was cancelled")]
|
||||
Cancelled,
|
||||
#[error("session creation cancellation cleanup failed for {session_id}: {message}")]
|
||||
CleanupFailed {
|
||||
session_id: SessionId,
|
||||
agent_window_id: Option<i64>,
|
||||
message: String,
|
||||
},
|
||||
#[error("extension rejected tool.session_start: {0:?}")]
|
||||
ExtensionError(RpcError),
|
||||
#[error("transport closed while waiting for extension response")]
|
||||
@@ -354,6 +363,8 @@ impl StartSessionError {
|
||||
StartSessionError::AmbiguousBrowserLabel { .. } => "invalid_params",
|
||||
StartSessionError::IdExhausted => "protocol_error",
|
||||
StartSessionError::Timeout => "timeout",
|
||||
StartSessionError::Cancelled => "cancelled",
|
||||
StartSessionError::CleanupFailed { .. } => "protocol_error",
|
||||
StartSessionError::TransportClosed => "protocol_error",
|
||||
StartSessionError::ExtensionError(err) => match err.code {
|
||||
bsk_protocol::ErrorCode::Timeout => "timeout",
|
||||
@@ -407,6 +418,7 @@ pub struct AgentWindowOptions {
|
||||
/// On success the matching per-session dispatch queue (M6.5,
|
||||
/// design §5) is also spawned so that subsequent `tool.*` RPCs serialise
|
||||
/// against this session's Agent Window / ref-store.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn start_session(
|
||||
registry: &Arc<BrowserRegistry>,
|
||||
sessions: &Arc<SessionRegistry>,
|
||||
@@ -415,24 +427,32 @@ pub async fn start_session(
|
||||
window: AgentWindowOptions,
|
||||
connect_wait: Duration,
|
||||
timeout_dur: Duration,
|
||||
cancel: Option<AbortToken>,
|
||||
) -> Result<Session, StartSessionError> {
|
||||
let client: Arc<BrowserClient> = registry
|
||||
.select_with_connect_wait(requested, connect_wait)
|
||||
.await
|
||||
.map_err(|e| match e {
|
||||
SelectError::NoBrowserConnected => StartSessionError::NoBrowserConnected,
|
||||
SelectError::MultipleBrowsersOnline => StartSessionError::MultipleBrowsersOnline {
|
||||
browsers: snapshot_status_entries(registry, sessions),
|
||||
},
|
||||
SelectError::NotFound => StartSessionError::BrowserNotFound,
|
||||
SelectError::AmbiguousLabel {
|
||||
label,
|
||||
instance_ids,
|
||||
} => StartSessionError::AmbiguousBrowserLabel {
|
||||
label,
|
||||
instance_ids,
|
||||
},
|
||||
})?;
|
||||
let selection = registry.select_with_connect_wait(requested, connect_wait);
|
||||
tokio::pin!(selection);
|
||||
let selected = match cancel.as_ref() {
|
||||
Some(token) => tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => return Err(StartSessionError::Cancelled),
|
||||
selected = &mut selection => selected,
|
||||
},
|
||||
None => selection.await,
|
||||
};
|
||||
let client: Arc<BrowserClient> = selected.map_err(|e| match e {
|
||||
SelectError::NoBrowserConnected => StartSessionError::NoBrowserConnected,
|
||||
SelectError::MultipleBrowsersOnline => StartSessionError::MultipleBrowsersOnline {
|
||||
browsers: snapshot_status_entries(registry, sessions),
|
||||
},
|
||||
SelectError::NotFound => StartSessionError::BrowserNotFound,
|
||||
SelectError::AmbiguousLabel {
|
||||
label,
|
||||
instance_ids,
|
||||
} => StartSessionError::AmbiguousBrowserLabel {
|
||||
label,
|
||||
instance_ids,
|
||||
},
|
||||
})?;
|
||||
let session_id = sessions
|
||||
.reserve_id(client.id.clone(), SESSION_ID_MAX_RESERVE_ATTEMPTS, now_ms)
|
||||
.ok_or(StartSessionError::IdExhausted)?;
|
||||
@@ -453,22 +473,71 @@ pub async fn start_session(
|
||||
let mut pending = client.pending.lock().unwrap();
|
||||
pending.register(rpc_id.clone())
|
||||
};
|
||||
if cancel.as_ref().is_some_and(AbortToken::is_cancelled) {
|
||||
sessions.cancel_reservation(&session_id);
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(StartSessionError::Cancelled);
|
||||
}
|
||||
if client.sink.send(Frame::Request(request)).is_err() {
|
||||
sessions.cancel_reservation(&session_id);
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(StartSessionError::TransportClosed);
|
||||
}
|
||||
let response = match timeout(timeout_dur, waiter).await {
|
||||
Ok(Ok(resp)) => resp,
|
||||
Ok(Err(_)) => {
|
||||
let mut waiter = waiter;
|
||||
enum StartWaitOutcome {
|
||||
Response(bsk_protocol::ResponseFrame),
|
||||
WaiterClosed,
|
||||
Cancelled,
|
||||
Timeout,
|
||||
}
|
||||
let wait_outcome = match cancel.as_ref() {
|
||||
Some(token) => {
|
||||
let deadline = tokio::time::sleep(timeout_dur);
|
||||
tokio::pin!(deadline);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => StartWaitOutcome::Cancelled,
|
||||
response = &mut waiter => match response {
|
||||
Ok(response) => StartWaitOutcome::Response(response),
|
||||
Err(_) => StartWaitOutcome::WaiterClosed,
|
||||
},
|
||||
_ = &mut deadline => StartWaitOutcome::Timeout,
|
||||
}
|
||||
}
|
||||
None => match timeout(timeout_dur, &mut waiter).await {
|
||||
Ok(Ok(response)) => StartWaitOutcome::Response(response),
|
||||
Ok(Err(_)) => StartWaitOutcome::WaiterClosed,
|
||||
Err(_) => StartWaitOutcome::Timeout,
|
||||
},
|
||||
};
|
||||
let response = match wait_outcome {
|
||||
StartWaitOutcome::Response(response) => response,
|
||||
StartWaitOutcome::WaiterClosed => {
|
||||
sessions.cancel_reservation(&session_id);
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(StartSessionError::TransportClosed);
|
||||
}
|
||||
Err(_) => {
|
||||
sessions.cancel_reservation(&session_id);
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(StartSessionError::Timeout);
|
||||
StartWaitOutcome::Cancelled => {
|
||||
return Err(finish_aborted_start(
|
||||
&client,
|
||||
sessions,
|
||||
&session_id,
|
||||
&rpc_id,
|
||||
waiter,
|
||||
StartAbortReason::Cancelled,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
StartWaitOutcome::Timeout => {
|
||||
return Err(finish_aborted_start(
|
||||
&client,
|
||||
sessions,
|
||||
&session_id,
|
||||
&rpc_id,
|
||||
waiter,
|
||||
StartAbortReason::Timeout,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
};
|
||||
let agent_window_id = match response.body {
|
||||
@@ -504,6 +573,124 @@ pub async fn start_session(
|
||||
Ok(session)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum StartAbortReason {
|
||||
Cancelled,
|
||||
Timeout,
|
||||
}
|
||||
|
||||
impl StartAbortReason {
|
||||
fn error(self) -> StartSessionError {
|
||||
match self {
|
||||
Self::Cancelled => StartSessionError::Cancelled,
|
||||
Self::Timeout => StartSessionError::Timeout,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn finish_aborted_start(
|
||||
client: &Arc<BrowserClient>,
|
||||
sessions: &Arc<SessionRegistry>,
|
||||
session_id: &SessionId,
|
||||
rpc_id: &RpcId,
|
||||
mut waiter: tokio::sync::oneshot::Receiver<bsk_protocol::ResponseFrame>,
|
||||
reason: StartAbortReason,
|
||||
) -> StartSessionError {
|
||||
sessions.cancel_reservation(session_id);
|
||||
let cancel = Frame::Request(RequestFrame {
|
||||
id: format!("cancel-{rpc_id}"),
|
||||
method: Method::Cancel,
|
||||
params: Some(serde_json::json!({ "rpc_id": rpc_id })),
|
||||
});
|
||||
if client.sink.send(cancel).is_err() {
|
||||
client.pending.lock().unwrap().cancel(rpc_id);
|
||||
return StartSessionError::TransportClosed;
|
||||
}
|
||||
|
||||
match timeout(CANCEL_CLEANUP_TIMEOUT, &mut waiter).await {
|
||||
Ok(Ok(response)) => match response.body {
|
||||
ResponseBody::Err(err)
|
||||
if matches!(err.code, ErrorCode::Cancelled | ErrorCode::UserAborted) =>
|
||||
{
|
||||
reason.error()
|
||||
}
|
||||
ResponseBody::Err(err) => StartSessionError::ExtensionError(err),
|
||||
ResponseBody::Ok(value) => {
|
||||
let agent_window_id = serde_json::from_value::<SessionStartResult>(value)
|
||||
.ok()
|
||||
.and_then(|result| result.agent_window_id);
|
||||
match rollback_extension_session(client, session_id).await {
|
||||
Ok(()) => reason.error(),
|
||||
Err(message) => StartSessionError::CleanupFailed {
|
||||
session_id: session_id.clone(),
|
||||
agent_window_id,
|
||||
message,
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
Ok(Err(_)) => {
|
||||
client.pending.lock().unwrap().cancel(rpc_id);
|
||||
StartSessionError::TransportClosed
|
||||
}
|
||||
Err(_) => {
|
||||
// The cancel frame is already queued behind session_start. Even
|
||||
// if chrome.windows.create is still blocked, the extension will
|
||||
// observe the aborted signal and compensate before replying.
|
||||
client.pending.lock().unwrap().cancel(rpc_id);
|
||||
reason.error()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn rollback_extension_session(
|
||||
client: &Arc<BrowserClient>,
|
||||
session_id: &SessionId,
|
||||
) -> Result<(), String> {
|
||||
let rollback_id = next_rpc_id("sess-start-rollback");
|
||||
let waiter = {
|
||||
let mut pending = client.pending.lock().unwrap();
|
||||
pending.register(rollback_id.clone())
|
||||
};
|
||||
let request = RequestFrame {
|
||||
id: rollback_id.clone(),
|
||||
method: Method::ToolSessionStop,
|
||||
params: Some(
|
||||
serde_json::to_value(SessionStopParams {
|
||||
session_id: session_id.0.clone(),
|
||||
})
|
||||
.expect("serialize session_start rollback params"),
|
||||
),
|
||||
};
|
||||
if client.sink.send(Frame::Request(request)).is_err() {
|
||||
client.pending.lock().unwrap().cancel(&rollback_id);
|
||||
return Err("browser disconnected before rollback was queued".into());
|
||||
}
|
||||
let response = match timeout(CANCEL_CLEANUP_TIMEOUT, waiter).await {
|
||||
Ok(Ok(response)) => response,
|
||||
Ok(Err(_)) => return Err("browser disconnected during rollback".into()),
|
||||
Err(_) => {
|
||||
client.pending.lock().unwrap().cancel(&rollback_id);
|
||||
return Err(format!(
|
||||
"extension did not finish rollback within {CANCEL_CLEANUP_TIMEOUT:?}"
|
||||
));
|
||||
}
|
||||
};
|
||||
match response.body {
|
||||
ResponseBody::Ok(value) => {
|
||||
let result: SessionStopResult = serde_json::from_value(value)
|
||||
.map_err(|err| format!("invalid rollback response: {err}"))?;
|
||||
if result.return_failures.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err("rollback reported borrowed-tab return failures".into())
|
||||
}
|
||||
}
|
||||
ResponseBody::Err(err) if err.code == ErrorCode::NotFound => Ok(()),
|
||||
ResponseBody::Err(err) => Err(format!("rollback failed: {} ({:?})", err.message, err.code)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Tear down a single session id (round-trips `tool.session_stop` to the
|
||||
/// owning extension and unregisters on success). Also tears down the
|
||||
/// per-session dispatch queue so the worker task exits.
|
||||
@@ -514,6 +701,7 @@ pub async fn stop_session(
|
||||
interrupts: &Arc<SessionInterruptRegistry>,
|
||||
session_id: &SessionId,
|
||||
timeout_dur: Duration,
|
||||
cancel: Option<AbortToken>,
|
||||
) -> Result<SessionStopResult, StopSessionError> {
|
||||
let session = sessions.get(session_id).ok_or(StopSessionError::NotFound)?;
|
||||
if registry.get(&session.browser_id).is_none() {
|
||||
@@ -524,12 +712,17 @@ pub async fn stop_session(
|
||||
session_id: session_id.0.clone(),
|
||||
})
|
||||
.unwrap();
|
||||
// Internal callers (idle reaping and shutdown) do not have a CLI-owned
|
||||
// token, but still need timeout-driven WS cancellation so a late window
|
||||
// close cannot leave daemon state pointing at a vanished session.
|
||||
let lifecycle_cancel = Some(cancel.unwrap_or_default());
|
||||
match queues
|
||||
.dispatch_after_closing(
|
||||
session_id,
|
||||
bsk_protocol::Method::ToolSessionStop,
|
||||
params,
|
||||
timeout_dur,
|
||||
lifecycle_cancel,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -567,6 +760,10 @@ pub async fn stop_session(
|
||||
drop_session_local(queues, interrupts, session_id);
|
||||
return Ok(SessionStopResult::default());
|
||||
}
|
||||
if err.code == bsk_protocol::ErrorCode::Timeout {
|
||||
queues.reopen(session_id);
|
||||
return Err(StopSessionError::Timeout);
|
||||
}
|
||||
queues.reopen(session_id);
|
||||
Err(StopSessionError::ExtensionError(err))
|
||||
}
|
||||
|
||||
@@ -15,7 +15,6 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
@@ -291,8 +290,7 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
|
||||
});
|
||||
|
||||
let started_at = Instant::now();
|
||||
let activity: Arc<Mutex<Instant>> = Arc::new(Mutex::new(started_at));
|
||||
let active_ipc_connections = Arc::new(AtomicUsize::new(0));
|
||||
let activity = Arc::new(Mutex::new(IpcActivityState::new(started_at)));
|
||||
let (ipc_shutdown_tx, ipc_shutdown_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let status = ipc::DaemonStatus {
|
||||
started_at,
|
||||
@@ -314,18 +312,14 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
|
||||
let handler = handler.clone();
|
||||
let ipc_open = {
|
||||
let activity = activity.clone();
|
||||
let active = active_ipc_connections.clone();
|
||||
move || {
|
||||
active.fetch_add(1, Ordering::SeqCst);
|
||||
record_activity(&activity);
|
||||
record_ipc_open(&activity);
|
||||
}
|
||||
};
|
||||
let ipc_close = {
|
||||
let activity = activity.clone();
|
||||
let active = active_ipc_connections.clone();
|
||||
move || {
|
||||
active.fetch_sub(1, Ordering::SeqCst);
|
||||
record_activity(&activity);
|
||||
record_ipc_close(&activity);
|
||||
}
|
||||
};
|
||||
tokio::spawn(ipc::serve(
|
||||
@@ -344,7 +338,6 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
|
||||
let idle_task = {
|
||||
let activity = activity.clone();
|
||||
let state = Arc::clone(&state);
|
||||
let active_ipc_connections = active_ipc_connections.clone();
|
||||
let daemon_idle = cfg.daemon_idle;
|
||||
tokio::spawn(async move {
|
||||
let tick = (daemon_idle / 4).max(Duration::from_millis(250));
|
||||
@@ -352,22 +345,19 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
|
||||
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
loop {
|
||||
ticker.tick().await;
|
||||
let last = match activity.lock() {
|
||||
Ok(a) => *a,
|
||||
Err(p) => *p.into_inner(),
|
||||
};
|
||||
if last.elapsed() < daemon_idle {
|
||||
let ipc_is_idle =
|
||||
lock_activity(&activity).should_exit_for_idle(Instant::now(), daemon_idle);
|
||||
if !ipc_is_idle {
|
||||
continue;
|
||||
}
|
||||
// Bridge from M2/M3 (which only knew about connection
|
||||
// counts) to M4/M5 (browser + session registries):
|
||||
// hold the daemon alive while any IPC client is
|
||||
// connected, any browser is paired, or any session
|
||||
// is live (design §3.2).
|
||||
if active_ipc_connections.load(Ordering::SeqCst) > 0
|
||||
|| !state.browsers.is_empty()
|
||||
|| !state.sessions.is_empty()
|
||||
{
|
||||
// IPC liveness (open-connection count + idle window) is already
|
||||
// enforced above by `should_exit_for_idle`, which reads
|
||||
// `IpcActivityState` under one lock and returns true only when
|
||||
// there are zero connections AND the idle interval has elapsed.
|
||||
// Here we additionally hold the daemon alive while any browser
|
||||
// is paired or any session is live (design §3.2, M4/M5
|
||||
// registries).
|
||||
if !state.browsers.is_empty() || !state.sessions.is_empty() {
|
||||
continue;
|
||||
}
|
||||
info!(
|
||||
@@ -491,6 +481,7 @@ pub(crate) fn spawn_session_idle_reaper(state: Arc<DaemonState>) -> tokio::task:
|
||||
&state.session_interrupts,
|
||||
&session_id,
|
||||
Duration::from_secs(10),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -681,10 +672,55 @@ fn restart_start_args(cfg: &DaemonConfig) -> StartArgs {
|
||||
}
|
||||
}
|
||||
|
||||
fn record_activity(activity: &Arc<Mutex<Instant>>) {
|
||||
if let Ok(mut a) = activity.lock() {
|
||||
*a = Instant::now();
|
||||
#[derive(Debug)]
|
||||
struct IpcActivityState {
|
||||
last_activity: Instant,
|
||||
active_connections: usize,
|
||||
}
|
||||
|
||||
impl IpcActivityState {
|
||||
fn new(now: Instant) -> Self {
|
||||
Self {
|
||||
last_activity: now,
|
||||
active_connections: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn touch(&mut self, now: Instant) {
|
||||
self.last_activity = now;
|
||||
}
|
||||
|
||||
fn opened(&mut self, now: Instant) {
|
||||
self.active_connections = self.active_connections.saturating_add(1);
|
||||
self.last_activity = now;
|
||||
}
|
||||
|
||||
fn closed(&mut self, now: Instant) {
|
||||
self.active_connections = self.active_connections.saturating_sub(1);
|
||||
self.last_activity = now;
|
||||
}
|
||||
|
||||
fn should_exit_for_idle(&self, now: Instant, idle: Duration) -> bool {
|
||||
self.active_connections == 0 && now.saturating_duration_since(self.last_activity) >= idle
|
||||
}
|
||||
}
|
||||
|
||||
fn lock_activity(
|
||||
activity: &Arc<Mutex<IpcActivityState>>,
|
||||
) -> std::sync::MutexGuard<'_, IpcActivityState> {
|
||||
activity.lock().unwrap_or_else(|poison| poison.into_inner())
|
||||
}
|
||||
|
||||
fn record_activity(activity: &Arc<Mutex<IpcActivityState>>) {
|
||||
lock_activity(activity).touch(Instant::now());
|
||||
}
|
||||
|
||||
fn record_ipc_open(activity: &Arc<Mutex<IpcActivityState>>) {
|
||||
lock_activity(activity).opened(Instant::now());
|
||||
}
|
||||
|
||||
fn record_ipc_close(activity: &Arc<Mutex<IpcActivityState>>) {
|
||||
lock_activity(activity).closed(Instant::now());
|
||||
}
|
||||
|
||||
/// Initialise tracing for the daemon: write a daily-rotated file in
|
||||
@@ -1067,4 +1103,37 @@ mod tests {
|
||||
assert_eq!(args.session_idle, Some(Duration::from_secs(11)));
|
||||
assert_eq!(args.daemon_idle, Some(Duration::from_secs(22)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ipc_close_restarts_the_idle_window_atomically() {
|
||||
let started = Instant::now();
|
||||
let idle = Duration::from_secs(10);
|
||||
let mut activity = IpcActivityState::new(started);
|
||||
|
||||
activity.opened(started);
|
||||
let closed_at = started + Duration::from_secs(30);
|
||||
assert!(
|
||||
!activity.should_exit_for_idle(closed_at, idle),
|
||||
"an open IPC connection must hold the daemon alive"
|
||||
);
|
||||
|
||||
activity.closed(closed_at);
|
||||
assert_eq!(activity.active_connections, 0);
|
||||
assert_eq!(activity.last_activity, closed_at);
|
||||
assert!(!activity.should_exit_for_idle(closed_at, idle));
|
||||
assert!(!activity.should_exit_for_idle(closed_at + idle - Duration::from_millis(1), idle));
|
||||
assert!(activity.should_exit_for_idle(closed_at + idle, idle));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ipc_activity_keeps_the_daemon_alive_until_a_full_idle_interval_passes() {
|
||||
let started = Instant::now();
|
||||
let idle = Duration::from_secs(10);
|
||||
let mut activity = IpcActivityState::new(started);
|
||||
let request_at = started + Duration::from_secs(20);
|
||||
|
||||
activity.touch(request_at);
|
||||
assert!(!activity.should_exit_for_idle(request_at + Duration::from_secs(9), idle));
|
||||
assert!(activity.should_exit_for_idle(request_at + idle, idle));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,8 +34,8 @@ pub struct DaemonState {
|
||||
/// `stop_session` / browser disconnect.
|
||||
pub tool_queues: Arc<ToolQueueRegistry>,
|
||||
/// Per-rpc-id cancellation tokens for daemon-side long-runners
|
||||
/// (M9.3 — currently only `tool.wait_ms`). The CLI's `cancel
|
||||
/// { rpc_id }` consults this registry first.
|
||||
/// (`tool.wait_ms` plus `session.*` lifecycle calls). The CLI's
|
||||
/// `cancel { rpc_id }` consults this registry first.
|
||||
pub abort_registry: Arc<AbortRegistry>,
|
||||
/// Tracks `tool.*` RPCs that have been forwarded to an extension
|
||||
/// over WS but have not yet received a response. Indexed by the
|
||||
|
||||
@@ -325,6 +325,7 @@ async fn session_stop_fast_fails_while_tool_is_in_flight() {
|
||||
&state.session_interrupts,
|
||||
&sid,
|
||||
Duration::from_secs(5),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
@@ -366,6 +367,7 @@ async fn session_stop_fast_fails_while_tool_is_in_flight() {
|
||||
&state.session_interrupts,
|
||||
&sid,
|
||||
Duration::from_secs(5),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("session.stop ok after tool completes");
|
||||
|
||||
@@ -10,9 +10,14 @@ use std::time::Duration;
|
||||
|
||||
use bsk::daemon::{self, DaemonConfig};
|
||||
use bsk::ipc_client::IpcClient;
|
||||
use bsk_protocol::cancel::{CancelParams, CancelResult};
|
||||
use bsk_protocol::system::{HandshakeParams, HandshakeResult, StatusResult};
|
||||
use bsk_protocol::tools::{SessionStartParams, SessionStartResult, SessionStopParams};
|
||||
use bsk_protocol::{BrowserPeerInfo, Frame, Method, RequestFrame, ResponseBody, ResponseFrame};
|
||||
use bsk_protocol::tools::{
|
||||
SessionStartParams, SessionStartResult, SessionStopParams, SessionStopResult,
|
||||
};
|
||||
use bsk_protocol::{
|
||||
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
|
||||
};
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use rand::Rng;
|
||||
use tokio_tungstenite::tungstenite::handshake::client::generate_key;
|
||||
@@ -23,6 +28,9 @@ use support::{wait_for_browser_count, wait_for_no_sessions};
|
||||
|
||||
const TEST_EXT_ID: &str = "abcdefghijklmnopabcdefghijklmnop";
|
||||
|
||||
type TestWs =
|
||||
tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>>;
|
||||
|
||||
fn tempfile_path(prefix: &str) -> PathBuf {
|
||||
let mut p = std::env::temp_dir();
|
||||
let mut rng = rand::thread_rng();
|
||||
@@ -56,9 +64,7 @@ async fn spawn_daemon_with_session_idle(session_idle: Duration) -> (daemon::Daem
|
||||
(handle, sock)
|
||||
}
|
||||
|
||||
async fn connect_ext(
|
||||
addr: std::net::SocketAddr,
|
||||
) -> tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>> {
|
||||
async fn connect_ext(addr: std::net::SocketAddr) -> TestWs {
|
||||
let origin = format!("chrome-extension://{TEST_EXT_ID}");
|
||||
let url = format!("ws://{addr}/");
|
||||
let req = Request::builder()
|
||||
@@ -76,11 +82,7 @@ async fn connect_ext(
|
||||
ws
|
||||
}
|
||||
|
||||
async fn handshake_as_ext(
|
||||
ws: &mut tokio_tungstenite::WebSocketStream<
|
||||
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
|
||||
>,
|
||||
) -> HandshakeResult {
|
||||
async fn handshake_as_ext(ws: &mut TestWs) -> HandshakeResult {
|
||||
let params = HandshakeParams {
|
||||
client: "browser-skill-extension".into(),
|
||||
version: "0.1.0-dev.0".parse().unwrap(),
|
||||
@@ -114,6 +116,128 @@ async fn handshake_as_ext(
|
||||
}
|
||||
}
|
||||
|
||||
async fn next_extension_request(ws: &mut TestWs) -> RequestFrame {
|
||||
loop {
|
||||
let message = ws.next().await.expect("daemon closed websocket").unwrap();
|
||||
match message {
|
||||
Message::Text(text) => match serde_json::from_str::<Frame>(&text).unwrap() {
|
||||
Frame::Request(request) => return request,
|
||||
Frame::Response(_) | Frame::Event(_) => continue,
|
||||
},
|
||||
Message::Ping(payload) => ws.send(Message::Pong(payload)).await.unwrap(),
|
||||
Message::Close(_) => panic!("daemon closed websocket before sending request"),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_extension_response(ws: &mut TestWs, response: ResponseFrame) {
|
||||
ws.send(Message::Text(serde_json::to_string(&response).unwrap()))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn acknowledge_extension_cancel(ws: &mut TestWs, cancel: RequestFrame, target_rpc_id: &str) {
|
||||
assert_eq!(cancel.method, Method::Cancel);
|
||||
let params: CancelParams = serde_json::from_value(cancel.params.unwrap()).unwrap();
|
||||
assert_eq!(params.rpc_id, target_rpc_id);
|
||||
send_extension_response(
|
||||
ws,
|
||||
ResponseFrame {
|
||||
id: cancel.id,
|
||||
body: ResponseBody::Ok(serde_json::to_value(CancelResult { cancelled: true }).unwrap()),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn respond_to_aborted_start(
|
||||
ws: &mut TestWs,
|
||||
start_seen: Option<tokio::sync::oneshot::Sender<()>>,
|
||||
agent_window_id: i64,
|
||||
) {
|
||||
let start = next_extension_request(ws).await;
|
||||
assert_eq!(start.method, Method::ToolSessionStart);
|
||||
let params: SessionStartParams = serde_json::from_value(start.params.clone().unwrap()).unwrap();
|
||||
if let Some(start_seen) = start_seen {
|
||||
start_seen.send(()).unwrap();
|
||||
}
|
||||
|
||||
let cancel = next_extension_request(ws).await;
|
||||
acknowledge_extension_cancel(ws, cancel, &start.id).await;
|
||||
|
||||
// Model the hardest race: chrome.windows.create completed just as
|
||||
// cancellation arrived, so the extension's original call succeeds.
|
||||
send_extension_response(
|
||||
ws,
|
||||
ResponseFrame {
|
||||
id: start.id,
|
||||
body: ResponseBody::Ok(
|
||||
serde_json::to_value(SessionStartResult {
|
||||
agent_window_id: Some(agent_window_id),
|
||||
})
|
||||
.unwrap(),
|
||||
),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
let rollback = next_extension_request(ws).await;
|
||||
assert_eq!(rollback.method, Method::ToolSessionStop);
|
||||
let rollback_params: SessionStopParams =
|
||||
serde_json::from_value(rollback.params.unwrap()).unwrap();
|
||||
assert_eq!(rollback_params.session_id, params.session_id);
|
||||
send_extension_response(
|
||||
ws,
|
||||
ResponseFrame {
|
||||
id: rollback.id,
|
||||
body: ResponseBody::Ok(serde_json::to_value(SessionStopResult::default()).unwrap()),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn respond_to_aborted_stop(
|
||||
ws: &mut TestWs,
|
||||
stop_seen: Option<tokio::sync::oneshot::Sender<()>>,
|
||||
) {
|
||||
let start = next_extension_request(ws).await;
|
||||
assert_eq!(start.method, Method::ToolSessionStart);
|
||||
send_extension_response(
|
||||
ws,
|
||||
ResponseFrame {
|
||||
id: start.id,
|
||||
body: ResponseBody::Ok(
|
||||
serde_json::to_value(SessionStartResult {
|
||||
agent_window_id: Some(4242),
|
||||
})
|
||||
.unwrap(),
|
||||
),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
let stop = next_extension_request(ws).await;
|
||||
assert_eq!(stop.method, Method::ToolSessionStop);
|
||||
if let Some(stop_seen) = stop_seen {
|
||||
stop_seen.send(()).unwrap();
|
||||
}
|
||||
let cancel = next_extension_request(ws).await;
|
||||
acknowledge_extension_cancel(ws, cancel, &stop.id).await;
|
||||
send_extension_response(
|
||||
ws,
|
||||
ResponseFrame {
|
||||
id: stop.id,
|
||||
body: ResponseBody::Err(RpcError {
|
||||
code: ErrorCode::Cancelled,
|
||||
message: "session_stop aborted before window close".into(),
|
||||
data: None,
|
||||
}),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn session_start_stop_round_trip_via_ipc() {
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
@@ -250,6 +374,240 @@ async fn session_start_stop_round_trip_via_ipc() {
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelling_session_start_rolls_back_a_late_extension_success() {
|
||||
const START_RPC_ID: &str = "session-start-cancel";
|
||||
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
let mut ws = connect_ext(handle.ws_addr()).await;
|
||||
let _ = handshake_as_ext(&mut ws).await;
|
||||
let (start_seen_tx, start_seen_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
|
||||
let responder = tokio::spawn(async move {
|
||||
respond_to_aborted_start(&mut ws, Some(start_seen_tx), 4242).await;
|
||||
let _ = release_rx.await;
|
||||
});
|
||||
|
||||
let mut start_ipc = IpcClient::connect(&sock).await.unwrap();
|
||||
let start_call = tokio::spawn(async move {
|
||||
let outcome: Result<serde_json::Value, RpcError> = start_ipc
|
||||
.call_with_id(
|
||||
START_RPC_ID.to_string(),
|
||||
Method::SessionStart,
|
||||
Some(serde_json::json!({ "focused": false })),
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
outcome
|
||||
});
|
||||
|
||||
start_seen_rx.await.unwrap();
|
||||
let mut cancel_ipc = IpcClient::connect(&sock).await.unwrap();
|
||||
let cancel: CancelResult = cancel_ipc
|
||||
.call(
|
||||
"cancel-start",
|
||||
Method::Cancel,
|
||||
Some(CancelParams {
|
||||
rpc_id: START_RPC_ID.to_string(),
|
||||
}),
|
||||
Duration::from_secs(2),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(cancel.cancelled);
|
||||
|
||||
let start_error = start_call.await.unwrap().unwrap_err();
|
||||
assert_eq!(start_error.code, ErrorCode::Cancelled);
|
||||
|
||||
let mut status_ipc = IpcClient::connect(&sock).await.unwrap();
|
||||
let status: StatusResult = status_ipc
|
||||
.call::<(), _>(
|
||||
"status-after-start-cancel",
|
||||
Method::SystemStatus,
|
||||
None,
|
||||
Duration::from_secs(2),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(status.sessions.is_empty());
|
||||
assert_eq!(status.browsers[0].session_count, 0);
|
||||
|
||||
let _ = release_tx.send(());
|
||||
responder.await.unwrap();
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn timing_out_session_start_rolls_back_a_late_extension_success() {
|
||||
let (handle, _sock) = spawn_daemon().await;
|
||||
let state = handle.state();
|
||||
let mut ws = connect_ext(handle.ws_addr()).await;
|
||||
let _ = handshake_as_ext(&mut ws).await;
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
|
||||
let responder = tokio::spawn(async move {
|
||||
respond_to_aborted_start(&mut ws, None, 4343).await;
|
||||
let _ = release_rx.await;
|
||||
});
|
||||
|
||||
let start = bsk::daemon::sessions::start_session(
|
||||
&state.browsers,
|
||||
&state.sessions,
|
||||
&state.tool_queues,
|
||||
None,
|
||||
bsk::daemon::sessions::AgentWindowOptions::default(),
|
||||
Duration::ZERO,
|
||||
Duration::from_millis(20),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(
|
||||
start,
|
||||
Err(bsk::daemon::sessions::StartSessionError::Timeout)
|
||||
));
|
||||
assert!(state.sessions.is_empty());
|
||||
|
||||
let _ = release_tx.send(());
|
||||
responder.await.unwrap();
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelling_session_stop_keeps_the_session_retryable() {
|
||||
const STOP_RPC_ID: &str = "session-stop-cancel";
|
||||
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
let mut ws = connect_ext(handle.ws_addr()).await;
|
||||
let _ = handshake_as_ext(&mut ws).await;
|
||||
let (stop_seen_tx, stop_seen_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
|
||||
let responder = tokio::spawn(async move {
|
||||
respond_to_aborted_stop(&mut ws, Some(stop_seen_tx)).await;
|
||||
let _ = release_rx.await;
|
||||
});
|
||||
|
||||
let mut start_ipc = IpcClient::connect(&sock).await.unwrap();
|
||||
let start: serde_json::Value = start_ipc
|
||||
.call(
|
||||
"start-before-stop-cancel",
|
||||
Method::SessionStart,
|
||||
Some(serde_json::json!({ "focused": false })),
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let session_id = start["session_id"].as_str().unwrap().to_string();
|
||||
|
||||
let mut stop_ipc = IpcClient::connect(&sock).await.unwrap();
|
||||
let stop_session_id = session_id.clone();
|
||||
let stop_call = tokio::spawn(async move {
|
||||
let outcome: Result<serde_json::Value, RpcError> = stop_ipc
|
||||
.call_with_id(
|
||||
STOP_RPC_ID.to_string(),
|
||||
Method::SessionStop,
|
||||
Some(serde_json::json!({
|
||||
"session_id": stop_session_id,
|
||||
"all": false,
|
||||
})),
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
outcome
|
||||
});
|
||||
|
||||
stop_seen_rx.await.unwrap();
|
||||
let mut cancel_ipc = IpcClient::connect(&sock).await.unwrap();
|
||||
let cancel: CancelResult = cancel_ipc
|
||||
.call(
|
||||
"cancel-stop",
|
||||
Method::Cancel,
|
||||
Some(CancelParams {
|
||||
rpc_id: STOP_RPC_ID.to_string(),
|
||||
}),
|
||||
Duration::from_secs(2),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(cancel.cancelled);
|
||||
|
||||
let stop_error = stop_call.await.unwrap().unwrap_err();
|
||||
assert_eq!(stop_error.code, ErrorCode::Cancelled);
|
||||
|
||||
let mut status_ipc = IpcClient::connect(&sock).await.unwrap();
|
||||
let status: StatusResult = status_ipc
|
||||
.call::<(), _>(
|
||||
"status-after-stop-cancel",
|
||||
Method::SystemStatus,
|
||||
None,
|
||||
Duration::from_secs(2),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(status.sessions.len(), 1);
|
||||
assert_eq!(status.sessions[0].session_id, session_id);
|
||||
|
||||
let _ = release_tx.send(());
|
||||
responder.await.unwrap();
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn timing_out_session_stop_cancels_extension_teardown_and_keeps_session() {
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
let state = handle.state();
|
||||
let mut ws = connect_ext(handle.ws_addr()).await;
|
||||
let _ = handshake_as_ext(&mut ws).await;
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
|
||||
let responder = tokio::spawn(async move {
|
||||
respond_to_aborted_stop(&mut ws, None).await;
|
||||
let _ = release_rx.await;
|
||||
});
|
||||
|
||||
let mut start_ipc = IpcClient::connect(&sock).await.unwrap();
|
||||
let start: serde_json::Value = start_ipc
|
||||
.call(
|
||||
"start-before-stop-timeout",
|
||||
Method::SessionStart,
|
||||
Some(serde_json::json!({ "focused": false })),
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let session_id =
|
||||
bsk::daemon::sessions::SessionId(start["session_id"].as_str().unwrap().to_string());
|
||||
|
||||
let stop = bsk::daemon::sessions::stop_session(
|
||||
&state.browsers,
|
||||
&state.sessions,
|
||||
&state.tool_queues,
|
||||
&state.session_interrupts,
|
||||
&session_id,
|
||||
Duration::from_millis(20),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(
|
||||
stop,
|
||||
Err(bsk::daemon::sessions::StopSessionError::Timeout)
|
||||
));
|
||||
assert!(state.sessions.get(&session_id).is_some());
|
||||
|
||||
let _ = release_tx.send(());
|
||||
responder.await.unwrap();
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn session_idle_timeout_stops_and_unregisters_session() {
|
||||
let (handle, _sock) = spawn_daemon_with_session_idle(Duration::from_millis(50)).await;
|
||||
|
||||
@@ -7,10 +7,18 @@
|
||||
],
|
||||
"properties": {
|
||||
"trace": {
|
||||
"$ref": "#/definitions/Trace"
|
||||
"$ref": "#/definitions/RecordedTrace"
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"FillCommit": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"enter",
|
||||
"suggestion",
|
||||
"blur"
|
||||
]
|
||||
},
|
||||
"KeyModifier": {
|
||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||
"type": "string",
|
||||
@@ -21,7 +29,19 @@
|
||||
"shift"
|
||||
]
|
||||
},
|
||||
"PageRef": {
|
||||
"NavigationCause": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"user_typed",
|
||||
"link",
|
||||
"form_submit",
|
||||
"reload",
|
||||
"history",
|
||||
"script",
|
||||
"browser"
|
||||
]
|
||||
},
|
||||
"PageRefV2": {
|
||||
"description": "Page context dictionary entry — referenced by steps via `page` id.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
@@ -43,7 +63,42 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"SelectedOption": {
|
||||
"RecordedTrace": {
|
||||
"if": {
|
||||
"required": [
|
||||
"version"
|
||||
],
|
||||
"properties": {
|
||||
"version": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"then": {
|
||||
"$ref": "#/definitions/TraceV3"
|
||||
},
|
||||
"else": {
|
||||
"$ref": "#/definitions/TraceV2"
|
||||
}
|
||||
},
|
||||
"RecorderInfo": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"bsk",
|
||||
"vom"
|
||||
],
|
||||
"properties": {
|
||||
"bsk": {
|
||||
"type": "string"
|
||||
},
|
||||
"vom": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
}
|
||||
}
|
||||
},
|
||||
"SelectedOptionV2": {
|
||||
"description": "One selected option (`select` op).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
@@ -61,11 +116,53 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"Step": {
|
||||
"description": "One recorded user action — discriminated union by `op`.",
|
||||
"SelectedOptionV3": {
|
||||
"description": "One selected option (`select` op).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepEffectV2": {
|
||||
"description": "Observed navigation after a step (objective fact only).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"navigated_to"
|
||||
],
|
||||
"properties": {
|
||||
"navigated_to": {
|
||||
"description": "Reference into `pages[]` for the destination page.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepResultV3": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"state": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepV2": {
|
||||
"description": "One recorded user action — discriminated union by `op` (v2).",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -77,7 +174,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -105,7 +202,7 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -117,7 +214,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -140,12 +237,12 @@
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -157,7 +254,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -180,12 +277,12 @@
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -198,7 +295,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -227,7 +324,7 @@
|
||||
]
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
@@ -235,7 +332,7 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -248,7 +345,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -273,16 +370,16 @@
|
||||
"selection": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/SelectedOption"
|
||||
"$ref": "#/definitions/SelectedOptionV2"
|
||||
}
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -294,7 +391,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -331,7 +428,7 @@
|
||||
"target": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -342,21 +439,294 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"StepEffect": {
|
||||
"description": "Observed navigation after a step (objective fact only).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"navigated_to"
|
||||
],
|
||||
"properties": {
|
||||
"navigated_to": {
|
||||
"description": "Reference into `pages[]` for the destination page.",
|
||||
"type": "string"
|
||||
"StepV3": {
|
||||
"description": "One recorded user action — discriminated union by `op`.",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"cause",
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"to"
|
||||
],
|
||||
"properties": {
|
||||
"cause": {
|
||||
"$ref": "#/definitions/NavigationCause"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"navigate"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"to": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"click"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"commit",
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target",
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"commit": {
|
||||
"$ref": "#/definitions/FillCommit"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"fill"
|
||||
]
|
||||
},
|
||||
"redacted": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"select"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"selection": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/SelectedOptionV3"
|
||||
}
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"key",
|
||||
"op",
|
||||
"result",
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"key": {
|
||||
"type": "string"
|
||||
},
|
||||
"modifiers": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/KeyModifier"
|
||||
}
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"press"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"scroll"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"TargetDescriptor": {
|
||||
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
|
||||
"StopReason": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"user_finish",
|
||||
"cli_stop"
|
||||
]
|
||||
},
|
||||
"TargetDescriptorV2": {
|
||||
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"tag"
|
||||
@@ -397,8 +767,83 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"Trace": {
|
||||
"description": "Persisted user-action trace exported by `tool.record_stop` / `await`.",
|
||||
"TargetDescriptorV3": {
|
||||
"description": "Stable semantic handle for an interacted element within a page observation.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"ctx": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"ref": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"role": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"unmatched": {
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceEntry": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"start_url"
|
||||
],
|
||||
"properties": {
|
||||
"start_url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceStateV3": {
|
||||
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"body",
|
||||
"id",
|
||||
"url"
|
||||
],
|
||||
"properties": {
|
||||
"body": {
|
||||
"description": "Full page observation (front matter + VOM body + annotations).",
|
||||
"type": "string"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"title": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"truncated": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceV2": {
|
||||
"title": "TraceV2",
|
||||
"description": "Persisted user-action trace exported by legacy `tool.record_stop` / `await`.\n\nUnknown extension fields are ignored so older traces remain readable. `states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace` classification.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"entry",
|
||||
@@ -413,11 +858,10 @@
|
||||
"pages": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/PageRef"
|
||||
"$ref": "#/definitions/PageRefV2"
|
||||
}
|
||||
},
|
||||
"purpose": {
|
||||
"description": "Optional user-provided goal from `--purpose` (metadata only).",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
@@ -433,25 +877,76 @@
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"states": false,
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/Step"
|
||||
"$ref": "#/definitions/StepV2"
|
||||
}
|
||||
},
|
||||
"version": {
|
||||
"description": "Numeric version selects Trace v3. Legacy v2 envelopes omit this field.",
|
||||
"not": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceEntry": {
|
||||
"description": "Recording entry point — first URL the flow starts from.",
|
||||
"TraceV3": {
|
||||
"description": "Wire trace returned by `tool.record_stop` / `await`.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"start_url"
|
||||
"entry",
|
||||
"recorded_at",
|
||||
"recorder",
|
||||
"states",
|
||||
"steps",
|
||||
"stopped_by",
|
||||
"version"
|
||||
],
|
||||
"properties": {
|
||||
"start_url": {
|
||||
"entry": {
|
||||
"$ref": "#/definitions/TraceEntry"
|
||||
},
|
||||
"purpose": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"recorded_at": {
|
||||
"type": "string"
|
||||
},
|
||||
"recorder": {
|
||||
"$ref": "#/definitions/RecorderInfo"
|
||||
},
|
||||
"started_at": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"states": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/TraceStateV3"
|
||||
}
|
||||
},
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/StepV3"
|
||||
}
|
||||
},
|
||||
"stopped_by": {
|
||||
"$ref": "#/definitions/StopReason"
|
||||
},
|
||||
"version": {
|
||||
"type": "integer",
|
||||
"const": 3
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,10 +7,18 @@
|
||||
],
|
||||
"properties": {
|
||||
"trace": {
|
||||
"$ref": "#/definitions/Trace"
|
||||
"$ref": "#/definitions/RecordedTrace"
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"FillCommit": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"enter",
|
||||
"suggestion",
|
||||
"blur"
|
||||
]
|
||||
},
|
||||
"KeyModifier": {
|
||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||
"type": "string",
|
||||
@@ -21,7 +29,19 @@
|
||||
"shift"
|
||||
]
|
||||
},
|
||||
"PageRef": {
|
||||
"NavigationCause": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"user_typed",
|
||||
"link",
|
||||
"form_submit",
|
||||
"reload",
|
||||
"history",
|
||||
"script",
|
||||
"browser"
|
||||
]
|
||||
},
|
||||
"PageRefV2": {
|
||||
"description": "Page context dictionary entry — referenced by steps via `page` id.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
@@ -43,7 +63,42 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"SelectedOption": {
|
||||
"RecordedTrace": {
|
||||
"if": {
|
||||
"required": [
|
||||
"version"
|
||||
],
|
||||
"properties": {
|
||||
"version": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"then": {
|
||||
"$ref": "#/definitions/TraceV3"
|
||||
},
|
||||
"else": {
|
||||
"$ref": "#/definitions/TraceV2"
|
||||
}
|
||||
},
|
||||
"RecorderInfo": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"bsk",
|
||||
"vom"
|
||||
],
|
||||
"properties": {
|
||||
"bsk": {
|
||||
"type": "string"
|
||||
},
|
||||
"vom": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
}
|
||||
}
|
||||
},
|
||||
"SelectedOptionV2": {
|
||||
"description": "One selected option (`select` op).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
@@ -61,11 +116,53 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"Step": {
|
||||
"description": "One recorded user action — discriminated union by `op`.",
|
||||
"SelectedOptionV3": {
|
||||
"description": "One selected option (`select` op).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepEffectV2": {
|
||||
"description": "Observed navigation after a step (objective fact only).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"navigated_to"
|
||||
],
|
||||
"properties": {
|
||||
"navigated_to": {
|
||||
"description": "Reference into `pages[]` for the destination page.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepResultV3": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"state": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepV2": {
|
||||
"description": "One recorded user action — discriminated union by `op` (v2).",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -77,7 +174,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -105,7 +202,7 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -117,7 +214,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -140,12 +237,12 @@
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -157,7 +254,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -180,12 +277,12 @@
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -198,7 +295,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -227,7 +324,7 @@
|
||||
]
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
@@ -235,7 +332,7 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -248,7 +345,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -273,16 +370,16 @@
|
||||
"selection": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/SelectedOption"
|
||||
"$ref": "#/definitions/SelectedOptionV2"
|
||||
}
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -294,7 +391,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -331,7 +428,7 @@
|
||||
"target": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -342,21 +439,294 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"StepEffect": {
|
||||
"description": "Observed navigation after a step (objective fact only).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"navigated_to"
|
||||
],
|
||||
"properties": {
|
||||
"navigated_to": {
|
||||
"description": "Reference into `pages[]` for the destination page.",
|
||||
"type": "string"
|
||||
"StepV3": {
|
||||
"description": "One recorded user action — discriminated union by `op`.",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"cause",
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"to"
|
||||
],
|
||||
"properties": {
|
||||
"cause": {
|
||||
"$ref": "#/definitions/NavigationCause"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"navigate"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"to": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"click"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"commit",
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target",
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"commit": {
|
||||
"$ref": "#/definitions/FillCommit"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"fill"
|
||||
]
|
||||
},
|
||||
"redacted": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"select"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"selection": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/SelectedOptionV3"
|
||||
}
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"key",
|
||||
"op",
|
||||
"result",
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"key": {
|
||||
"type": "string"
|
||||
},
|
||||
"modifiers": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/KeyModifier"
|
||||
}
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"press"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"scroll"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"TargetDescriptor": {
|
||||
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
|
||||
"StopReason": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"user_finish",
|
||||
"cli_stop"
|
||||
]
|
||||
},
|
||||
"TargetDescriptorV2": {
|
||||
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"tag"
|
||||
@@ -397,8 +767,83 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"Trace": {
|
||||
"description": "Persisted user-action trace exported by `tool.record_stop` / `await`.",
|
||||
"TargetDescriptorV3": {
|
||||
"description": "Stable semantic handle for an interacted element within a page observation.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"ctx": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"ref": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"role": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"unmatched": {
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceEntry": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"start_url"
|
||||
],
|
||||
"properties": {
|
||||
"start_url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceStateV3": {
|
||||
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"body",
|
||||
"id",
|
||||
"url"
|
||||
],
|
||||
"properties": {
|
||||
"body": {
|
||||
"description": "Full page observation (front matter + VOM body + annotations).",
|
||||
"type": "string"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"title": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"truncated": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceV2": {
|
||||
"title": "TraceV2",
|
||||
"description": "Persisted user-action trace exported by legacy `tool.record_stop` / `await`.\n\nUnknown extension fields are ignored so older traces remain readable. `states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace` classification.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"entry",
|
||||
@@ -413,11 +858,10 @@
|
||||
"pages": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/PageRef"
|
||||
"$ref": "#/definitions/PageRefV2"
|
||||
}
|
||||
},
|
||||
"purpose": {
|
||||
"description": "Optional user-provided goal from `--purpose` (metadata only).",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
@@ -433,25 +877,76 @@
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"states": false,
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/Step"
|
||||
"$ref": "#/definitions/StepV2"
|
||||
}
|
||||
},
|
||||
"version": {
|
||||
"description": "Numeric version selects Trace v3. Legacy v2 envelopes omit this field.",
|
||||
"not": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceEntry": {
|
||||
"description": "Recording entry point — first URL the flow starts from.",
|
||||
"TraceV3": {
|
||||
"description": "Wire trace returned by `tool.record_stop` / `await`.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"start_url"
|
||||
"entry",
|
||||
"recorded_at",
|
||||
"recorder",
|
||||
"states",
|
||||
"steps",
|
||||
"stopped_by",
|
||||
"version"
|
||||
],
|
||||
"properties": {
|
||||
"start_url": {
|
||||
"entry": {
|
||||
"$ref": "#/definitions/TraceEntry"
|
||||
},
|
||||
"purpose": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"recorded_at": {
|
||||
"type": "string"
|
||||
},
|
||||
"recorder": {
|
||||
"$ref": "#/definitions/RecorderInfo"
|
||||
},
|
||||
"started_at": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"states": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/TraceStateV3"
|
||||
}
|
||||
},
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/StepV3"
|
||||
}
|
||||
},
|
||||
"stopped_by": {
|
||||
"$ref": "#/definitions/StopReason"
|
||||
},
|
||||
"version": {
|
||||
"type": "integer",
|
||||
"const": 3
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,49 +1,31 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "Trace",
|
||||
"description": "Persisted user-action trace exported by `tool.record_stop` / `await`.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"entry",
|
||||
"pages",
|
||||
"recorded_at",
|
||||
"steps"
|
||||
],
|
||||
"properties": {
|
||||
"entry": {
|
||||
"$ref": "#/definitions/TraceEntry"
|
||||
},
|
||||
"pages": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/PageRef"
|
||||
}
|
||||
},
|
||||
"purpose": {
|
||||
"description": "Optional user-provided goal from `--purpose` (metadata only).",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"recorded_at": {
|
||||
"description": "RFC 3339 timestamp when recording stopped.",
|
||||
"type": "string"
|
||||
},
|
||||
"started_at": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/Step"
|
||||
"title": "RecordedTrace",
|
||||
"if": {
|
||||
"required": [
|
||||
"version"
|
||||
],
|
||||
"properties": {
|
||||
"version": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"then": {
|
||||
"$ref": "#/definitions/TraceV3"
|
||||
},
|
||||
"else": {
|
||||
"$ref": "#/definitions/TraceV2"
|
||||
},
|
||||
"definitions": {
|
||||
"FillCommit": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"enter",
|
||||
"suggestion",
|
||||
"blur"
|
||||
]
|
||||
},
|
||||
"KeyModifier": {
|
||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||
"type": "string",
|
||||
@@ -54,7 +36,19 @@
|
||||
"shift"
|
||||
]
|
||||
},
|
||||
"PageRef": {
|
||||
"NavigationCause": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"user_typed",
|
||||
"link",
|
||||
"form_submit",
|
||||
"reload",
|
||||
"history",
|
||||
"script",
|
||||
"browser"
|
||||
]
|
||||
},
|
||||
"PageRefV2": {
|
||||
"description": "Page context dictionary entry — referenced by steps via `page` id.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
@@ -76,7 +70,24 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"SelectedOption": {
|
||||
"RecorderInfo": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"bsk",
|
||||
"vom"
|
||||
],
|
||||
"properties": {
|
||||
"bsk": {
|
||||
"type": "string"
|
||||
},
|
||||
"vom": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
}
|
||||
}
|
||||
},
|
||||
"SelectedOptionV2": {
|
||||
"description": "One selected option (`select` op).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
@@ -94,11 +105,53 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"Step": {
|
||||
"description": "One recorded user action — discriminated union by `op`.",
|
||||
"SelectedOptionV3": {
|
||||
"description": "One selected option (`select` op).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepEffectV2": {
|
||||
"description": "Observed navigation after a step (objective fact only).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"navigated_to"
|
||||
],
|
||||
"properties": {
|
||||
"navigated_to": {
|
||||
"description": "Reference into `pages[]` for the destination page.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepResultV3": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"state": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepV2": {
|
||||
"description": "One recorded user action — discriminated union by `op` (v2).",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -110,7 +163,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -138,7 +191,7 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -150,7 +203,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -173,12 +226,12 @@
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -190,7 +243,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -213,12 +266,12 @@
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -231,7 +284,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -260,7 +313,7 @@
|
||||
]
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
@@ -268,7 +321,7 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -281,7 +334,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -306,16 +359,16 @@
|
||||
"selection": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/SelectedOption"
|
||||
"$ref": "#/definitions/SelectedOptionV2"
|
||||
}
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -327,7 +380,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -364,7 +417,7 @@
|
||||
"target": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -375,21 +428,294 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"StepEffect": {
|
||||
"description": "Observed navigation after a step (objective fact only).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"navigated_to"
|
||||
],
|
||||
"properties": {
|
||||
"navigated_to": {
|
||||
"description": "Reference into `pages[]` for the destination page.",
|
||||
"type": "string"
|
||||
"StepV3": {
|
||||
"description": "One recorded user action — discriminated union by `op`.",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"cause",
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"to"
|
||||
],
|
||||
"properties": {
|
||||
"cause": {
|
||||
"$ref": "#/definitions/NavigationCause"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"navigate"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"to": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"click"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"commit",
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target",
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"commit": {
|
||||
"$ref": "#/definitions/FillCommit"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"fill"
|
||||
]
|
||||
},
|
||||
"redacted": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"select"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"selection": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/SelectedOptionV3"
|
||||
}
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"key",
|
||||
"op",
|
||||
"result",
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"key": {
|
||||
"type": "string"
|
||||
},
|
||||
"modifiers": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/KeyModifier"
|
||||
}
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"press"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"scroll"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"TargetDescriptor": {
|
||||
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
|
||||
"StopReason": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"user_finish",
|
||||
"cli_stop"
|
||||
]
|
||||
},
|
||||
"TargetDescriptorV2": {
|
||||
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"tag"
|
||||
@@ -430,8 +756,40 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"TargetDescriptorV3": {
|
||||
"description": "Stable semantic handle for an interacted element within a page observation.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"ctx": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"ref": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"role": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"unmatched": {
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceEntry": {
|
||||
"description": "Recording entry point — first URL the flow starts from.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"start_url"
|
||||
@@ -441,6 +799,143 @@
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceStateV3": {
|
||||
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"body",
|
||||
"id",
|
||||
"url"
|
||||
],
|
||||
"properties": {
|
||||
"body": {
|
||||
"description": "Full page observation (front matter + VOM body + annotations).",
|
||||
"type": "string"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"title": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"truncated": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceV2": {
|
||||
"title": "TraceV2",
|
||||
"description": "Persisted user-action trace exported by legacy `tool.record_stop` / `await`.\n\nUnknown extension fields are ignored so older traces remain readable. `states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace` classification.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"entry",
|
||||
"pages",
|
||||
"recorded_at",
|
||||
"steps"
|
||||
],
|
||||
"properties": {
|
||||
"entry": {
|
||||
"$ref": "#/definitions/TraceEntry"
|
||||
},
|
||||
"pages": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/PageRefV2"
|
||||
}
|
||||
},
|
||||
"purpose": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"recorded_at": {
|
||||
"description": "RFC 3339 timestamp when recording stopped.",
|
||||
"type": "string"
|
||||
},
|
||||
"started_at": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"states": false,
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/StepV2"
|
||||
}
|
||||
},
|
||||
"version": {
|
||||
"description": "Numeric version selects Trace v3. Legacy v2 envelopes omit this field.",
|
||||
"not": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceV3": {
|
||||
"description": "Wire trace returned by `tool.record_stop` / `await`.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"entry",
|
||||
"recorded_at",
|
||||
"recorder",
|
||||
"states",
|
||||
"steps",
|
||||
"stopped_by",
|
||||
"version"
|
||||
],
|
||||
"properties": {
|
||||
"entry": {
|
||||
"$ref": "#/definitions/TraceEntry"
|
||||
},
|
||||
"purpose": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"recorded_at": {
|
||||
"type": "string"
|
||||
},
|
||||
"recorder": {
|
||||
"$ref": "#/definitions/RecorderInfo"
|
||||
},
|
||||
"started_at": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"states": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/TraceStateV3"
|
||||
}
|
||||
},
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/StepV3"
|
||||
}
|
||||
},
|
||||
"stopped_by": {
|
||||
"$ref": "#/definitions/StopReason"
|
||||
},
|
||||
"version": {
|
||||
"type": "integer",
|
||||
"const": 3
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,286 +1,32 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "Step",
|
||||
"description": "One recorded user action — discriminated union by `op`.",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"to"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"navigate"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"to": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"click"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target",
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"fill"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"redacted": {
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"selection",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"select"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"selection": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/SelectedOption"
|
||||
}
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"key",
|
||||
"op",
|
||||
"page"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"key": {
|
||||
"type": "string"
|
||||
},
|
||||
"modifiers": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/KeyModifier"
|
||||
}
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"press"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"title": "RecordedStep",
|
||||
"not": {
|
||||
"required": [
|
||||
"page",
|
||||
"state"
|
||||
]
|
||||
},
|
||||
"if": {
|
||||
"required": [
|
||||
"state"
|
||||
]
|
||||
},
|
||||
"then": {
|
||||
"$ref": "#/definitions/StepV3"
|
||||
},
|
||||
"else": {
|
||||
"$ref": "#/definitions/StepV2"
|
||||
},
|
||||
"definitions": {
|
||||
"FillCommit": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"enter",
|
||||
"suggestion",
|
||||
"blur"
|
||||
]
|
||||
},
|
||||
"KeyModifier": {
|
||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||
"type": "string",
|
||||
@@ -291,7 +37,19 @@
|
||||
"shift"
|
||||
]
|
||||
},
|
||||
"SelectedOption": {
|
||||
"NavigationCause": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"user_typed",
|
||||
"link",
|
||||
"form_submit",
|
||||
"reload",
|
||||
"history",
|
||||
"script",
|
||||
"browser"
|
||||
]
|
||||
},
|
||||
"SelectedOptionV2": {
|
||||
"description": "One selected option (`select` op).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
@@ -309,7 +67,25 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepEffect": {
|
||||
"SelectedOptionV3": {
|
||||
"description": "One selected option (`select` op).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepEffectV2": {
|
||||
"description": "Observed navigation after a step (objective fact only).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
@@ -322,8 +98,579 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"TargetDescriptor": {
|
||||
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
|
||||
"StepResultV3": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"state": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepV2": {
|
||||
"description": "One recorded user action — discriminated union by `op` (v2).",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"to"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"navigate"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"to": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"click"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target",
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"fill"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"redacted": {
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"selection",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"select"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"selection": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/SelectedOptionV2"
|
||||
}
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"key",
|
||||
"op",
|
||||
"page"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"key": {
|
||||
"type": "string"
|
||||
},
|
||||
"modifiers": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/KeyModifier"
|
||||
}
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"press"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"StepV3": {
|
||||
"description": "One recorded user action — discriminated union by `op`.",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"cause",
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"to"
|
||||
],
|
||||
"properties": {
|
||||
"cause": {
|
||||
"$ref": "#/definitions/NavigationCause"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"navigate"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"to": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"click"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"commit",
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target",
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"commit": {
|
||||
"$ref": "#/definitions/FillCommit"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"fill"
|
||||
]
|
||||
},
|
||||
"redacted": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"select"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"selection": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/SelectedOptionV3"
|
||||
}
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"key",
|
||||
"op",
|
||||
"result",
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"key": {
|
||||
"type": "string"
|
||||
},
|
||||
"modifiers": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/KeyModifier"
|
||||
}
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"press"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"scroll"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"TargetDescriptorV2": {
|
||||
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"tag"
|
||||
@@ -363,6 +710,39 @@
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TargetDescriptorV3": {
|
||||
"description": "Stable semantic handle for an interacted element within a page observation.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"ctx": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"ref": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"role": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"unmatched": {
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,368 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "StepV2",
|
||||
"description": "One recorded user action — discriminated union by `op` (v2).",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"to"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"navigate"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"to": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"click"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target",
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"fill"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"redacted": {
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"selection",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"select"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"selection": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/SelectedOptionV2"
|
||||
}
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"key",
|
||||
"op",
|
||||
"page"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"key": {
|
||||
"type": "string"
|
||||
},
|
||||
"modifiers": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/KeyModifier"
|
||||
}
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"press"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"definitions": {
|
||||
"KeyModifier": {
|
||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"alt",
|
||||
"ctrl",
|
||||
"meta",
|
||||
"shift"
|
||||
]
|
||||
},
|
||||
"SelectedOptionV2": {
|
||||
"description": "One selected option (`select` op).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepEffectV2": {
|
||||
"description": "Observed navigation after a step (objective fact only).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"navigated_to"
|
||||
],
|
||||
"properties": {
|
||||
"navigated_to": {
|
||||
"description": "Reference into `pages[]` for the destination page.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TargetDescriptorV2": {
|
||||
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"tag"
|
||||
],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"name_attr": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"nearby_label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"placeholder": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"role": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"tag": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,375 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "StepV3",
|
||||
"description": "One recorded user action — discriminated union by `op`.",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"cause",
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"to"
|
||||
],
|
||||
"properties": {
|
||||
"cause": {
|
||||
"$ref": "#/definitions/NavigationCause"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"navigate"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"to": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"click"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"commit",
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target",
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"commit": {
|
||||
"$ref": "#/definitions/FillCommit"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"fill"
|
||||
]
|
||||
},
|
||||
"redacted": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"select"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"selection": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/SelectedOptionV3"
|
||||
}
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"key",
|
||||
"op",
|
||||
"result",
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"key": {
|
||||
"type": "string"
|
||||
},
|
||||
"modifiers": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/KeyModifier"
|
||||
}
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"press"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"scroll"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"definitions": {
|
||||
"FillCommit": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"enter",
|
||||
"suggestion",
|
||||
"blur"
|
||||
]
|
||||
},
|
||||
"KeyModifier": {
|
||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"alt",
|
||||
"ctrl",
|
||||
"meta",
|
||||
"shift"
|
||||
]
|
||||
},
|
||||
"NavigationCause": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"user_typed",
|
||||
"link",
|
||||
"form_submit",
|
||||
"reload",
|
||||
"history",
|
||||
"script",
|
||||
"browser"
|
||||
]
|
||||
},
|
||||
"SelectedOptionV3": {
|
||||
"description": "One selected option (`select` op).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepResultV3": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"state": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TargetDescriptorV3": {
|
||||
"description": "Stable semantic handle for an interacted element within a page observation.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"ctx": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"ref": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"role": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"unmatched": {
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,451 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "TraceV2",
|
||||
"description": "Persisted user-action trace exported by legacy `tool.record_stop` / `await`.\n\nUnknown extension fields are ignored so older traces remain readable. `states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace` classification.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"entry",
|
||||
"pages",
|
||||
"recorded_at",
|
||||
"steps"
|
||||
],
|
||||
"properties": {
|
||||
"entry": {
|
||||
"$ref": "#/definitions/TraceEntry"
|
||||
},
|
||||
"pages": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/PageRefV2"
|
||||
}
|
||||
},
|
||||
"purpose": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"recorded_at": {
|
||||
"description": "RFC 3339 timestamp when recording stopped.",
|
||||
"type": "string"
|
||||
},
|
||||
"started_at": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"states": false,
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/StepV2"
|
||||
}
|
||||
},
|
||||
"version": {
|
||||
"description": "Numeric version selects Trace v3. Legacy v2 envelopes omit this field.",
|
||||
"not": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"KeyModifier": {
|
||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"alt",
|
||||
"ctrl",
|
||||
"meta",
|
||||
"shift"
|
||||
]
|
||||
},
|
||||
"PageRefV2": {
|
||||
"description": "Page context dictionary entry — referenced by steps via `page` id.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"url"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"title": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"SelectedOptionV2": {
|
||||
"description": "One selected option (`select` op).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepEffectV2": {
|
||||
"description": "Observed navigation after a step (objective fact only).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"navigated_to"
|
||||
],
|
||||
"properties": {
|
||||
"navigated_to": {
|
||||
"description": "Reference into `pages[]` for the destination page.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepV2": {
|
||||
"description": "One recorded user action — discriminated union by `op` (v2).",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"to"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"navigate"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"to": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"click"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target",
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"fill"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"redacted": {
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"selection",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"select"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"selection": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/SelectedOptionV2"
|
||||
}
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"key",
|
||||
"op",
|
||||
"page"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"key": {
|
||||
"type": "string"
|
||||
},
|
||||
"modifiers": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/KeyModifier"
|
||||
}
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"press"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"TargetDescriptorV2": {
|
||||
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"tag"
|
||||
],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"name_attr": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"nearby_label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"placeholder": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"role": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"tag": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceEntry": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"start_url"
|
||||
],
|
||||
"properties": {
|
||||
"start_url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,496 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "TraceV3",
|
||||
"description": "Wire trace returned by `tool.record_stop` / `await`.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"entry",
|
||||
"recorded_at",
|
||||
"recorder",
|
||||
"states",
|
||||
"steps",
|
||||
"stopped_by",
|
||||
"version"
|
||||
],
|
||||
"properties": {
|
||||
"entry": {
|
||||
"$ref": "#/definitions/TraceEntry"
|
||||
},
|
||||
"purpose": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"recorded_at": {
|
||||
"type": "string"
|
||||
},
|
||||
"recorder": {
|
||||
"$ref": "#/definitions/RecorderInfo"
|
||||
},
|
||||
"started_at": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"states": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/TraceStateV3"
|
||||
}
|
||||
},
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/StepV3"
|
||||
}
|
||||
},
|
||||
"stopped_by": {
|
||||
"$ref": "#/definitions/StopReason"
|
||||
},
|
||||
"version": {
|
||||
"type": "integer",
|
||||
"const": 3
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"definitions": {
|
||||
"FillCommit": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"enter",
|
||||
"suggestion",
|
||||
"blur"
|
||||
]
|
||||
},
|
||||
"KeyModifier": {
|
||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"alt",
|
||||
"ctrl",
|
||||
"meta",
|
||||
"shift"
|
||||
]
|
||||
},
|
||||
"NavigationCause": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"user_typed",
|
||||
"link",
|
||||
"form_submit",
|
||||
"reload",
|
||||
"history",
|
||||
"script",
|
||||
"browser"
|
||||
]
|
||||
},
|
||||
"RecorderInfo": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"bsk",
|
||||
"vom"
|
||||
],
|
||||
"properties": {
|
||||
"bsk": {
|
||||
"type": "string"
|
||||
},
|
||||
"vom": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
}
|
||||
}
|
||||
},
|
||||
"SelectedOptionV3": {
|
||||
"description": "One selected option (`select` op).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepResultV3": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"state": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepV3": {
|
||||
"description": "One recorded user action — discriminated union by `op`.",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"cause",
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"to"
|
||||
],
|
||||
"properties": {
|
||||
"cause": {
|
||||
"$ref": "#/definitions/NavigationCause"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"navigate"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"to": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"click"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"commit",
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target",
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"commit": {
|
||||
"$ref": "#/definitions/FillCommit"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"fill"
|
||||
]
|
||||
},
|
||||
"redacted": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"select"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"selection": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/SelectedOptionV3"
|
||||
}
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"key",
|
||||
"op",
|
||||
"result",
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"key": {
|
||||
"type": "string"
|
||||
},
|
||||
"modifiers": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/KeyModifier"
|
||||
}
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"press"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TargetDescriptorV3"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"scroll"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResultV3"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"StopReason": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"user_finish",
|
||||
"cli_stop"
|
||||
]
|
||||
},
|
||||
"TargetDescriptorV3": {
|
||||
"description": "Stable semantic handle for an interacted element within a page observation.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"ctx": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"ref": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"role": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"unmatched": {
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceEntry": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"start_url"
|
||||
],
|
||||
"properties": {
|
||||
"start_url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceStateV3": {
|
||||
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"body",
|
||||
"id",
|
||||
"url"
|
||||
],
|
||||
"properties": {
|
||||
"body": {
|
||||
"description": "Full page observation (front matter + VOM body + annotations).",
|
||||
"type": "string"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"title": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"truncated": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -113,8 +113,12 @@ fn main() {
|
||||
dump!(RequestHelpParams, "tool_request_help_params");
|
||||
dump!(RequestHelpResult, "tool_request_help_result");
|
||||
|
||||
dump!(Trace, "trace");
|
||||
dump!(Step, "trace_step");
|
||||
dump!(TraceV2, "trace_v2");
|
||||
dump!(TraceV3, "trace_v3");
|
||||
dump!(RecordedTrace, "trace");
|
||||
dump!(StepV2, "trace_step_v2");
|
||||
dump!(StepV3, "trace_step_v3");
|
||||
dump!(RecordedStep, "trace_step");
|
||||
dump!(RecordStartParams, "tool_record_start_params");
|
||||
dump!(RecordStartResult, "tool_record_start_result");
|
||||
dump!(RecordStopParams, "tool_record_stop_params");
|
||||
|
||||
@@ -9,6 +9,9 @@ pub mod navigation;
|
||||
pub mod network;
|
||||
pub mod observation;
|
||||
pub mod record;
|
||||
mod record_common;
|
||||
mod record_v2;
|
||||
mod record_v3;
|
||||
pub mod script;
|
||||
pub mod session;
|
||||
pub mod tabs;
|
||||
|
||||
@@ -1,147 +1,23 @@
|
||||
//! Semantic user-action recording (`tool.record_start` / `stop` / `await`).
|
||||
//!
|
||||
//! Trace v2 is a **record-only** log of user actions: what was clicked, filled,
|
||||
//! selected, and where navigation occurred. No LLM runs during recording, so
|
||||
//! variable-vs-constant classification is **not** stored — executing agents
|
||||
//! infer that at run time from raw values and control names.
|
||||
//! Wire traces are either Trace v2 (`pages[]`) or Trace v3 (`version: 3`,
|
||||
//! `states[]`). Version-specific models live in `record_v2` / `record_v3`.
|
||||
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::interaction::KeyModifier;
|
||||
pub use super::record_common::TraceEntry;
|
||||
pub use super::record_v2::{
|
||||
PageRefV2, SelectedOptionV2, StepCommonV2, StepEffectV2, StepV2, TargetDescriptorV2, TraceV2,
|
||||
};
|
||||
pub use super::record_v3::{
|
||||
FillCommit, NavigationCause, RecorderInfo, SelectedOptionV3, StepCommonV3, StepResultV3,
|
||||
StepV3, StopReason, TRACE_VERSION_V3, TargetDescriptorV3, TraceStateV3, TraceV3,
|
||||
VOM_FORMAT_VERSION,
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Target
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Stable semantic handle for an interacted element.
|
||||
///
|
||||
/// `name` and `nearby_label` are **untrusted page text**.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TargetDescriptor {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub role: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub name: Option<String>,
|
||||
pub tag: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub name_attr: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub placeholder: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub nearby_label: Option<String>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Trace envelope
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Recording entry point — first URL the flow starts from.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TraceEntry {
|
||||
pub start_url: String,
|
||||
}
|
||||
|
||||
/// Page context dictionary entry — referenced by steps via `page` id.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct PageRef {
|
||||
pub id: String,
|
||||
pub url: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub title: Option<String>,
|
||||
}
|
||||
|
||||
/// One selected option (`select` op).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct SelectedOption {
|
||||
pub value: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub label: Option<String>,
|
||||
}
|
||||
|
||||
/// Observed navigation after a step (objective fact only).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct StepEffect {
|
||||
/// Reference into `pages[]` for the destination page.
|
||||
pub navigated_to: String,
|
||||
}
|
||||
|
||||
/// Fields shared by every step variant (flattened in JSON).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct StepCommon {
|
||||
pub id: u32,
|
||||
/// Reference into `pages[]`.
|
||||
pub page: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub effect: Option<StepEffect>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Step op-specific payloads
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// One recorded user action — discriminated union by `op`.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(tag = "op", rename_all = "snake_case")]
|
||||
pub enum Step {
|
||||
Navigate {
|
||||
#[serde(flatten)]
|
||||
common: StepCommon,
|
||||
to: String,
|
||||
},
|
||||
Click {
|
||||
#[serde(flatten)]
|
||||
common: StepCommon,
|
||||
target: TargetDescriptor,
|
||||
},
|
||||
Hover {
|
||||
#[serde(flatten)]
|
||||
common: StepCommon,
|
||||
target: TargetDescriptor,
|
||||
},
|
||||
Fill {
|
||||
#[serde(flatten)]
|
||||
common: StepCommon,
|
||||
target: TargetDescriptor,
|
||||
value: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
redacted: Option<bool>,
|
||||
},
|
||||
Select {
|
||||
#[serde(flatten)]
|
||||
common: StepCommon,
|
||||
target: TargetDescriptor,
|
||||
selection: Vec<SelectedOption>,
|
||||
},
|
||||
Press {
|
||||
#[serde(flatten)]
|
||||
common: StepCommon,
|
||||
key: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
modifiers: Option<Vec<KeyModifier>>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
target: Option<TargetDescriptor>,
|
||||
},
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Trace root
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Persisted user-action trace exported by `tool.record_stop` / `await`.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Trace {
|
||||
/// RFC 3339 timestamp when recording stopped.
|
||||
pub recorded_at: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub started_at: Option<String>,
|
||||
/// Optional user-provided goal from `--purpose` (metadata only).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub purpose: Option<String>,
|
||||
pub entry: TraceEntry,
|
||||
pub pages: Vec<PageRef>,
|
||||
pub steps: Vec<Step>,
|
||||
}
|
||||
/// Logical v2 identifier. Not a wire field — v2 envelopes omit `version`.
|
||||
pub const TRACE_VERSION_V2: u32 = 2;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// RPC params / results
|
||||
@@ -169,9 +45,209 @@ pub struct RecordStopParams {
|
||||
pub session_id: String,
|
||||
}
|
||||
|
||||
/// Wire trace payload — v2 (legacy `pages[]`) or v3 (`version: 3`, `states[]`).
|
||||
///
|
||||
/// Classification matches [`RecordedTrace::classify_value`]: a numeric `version`
|
||||
/// selects v3 (and forbids `pages[]`); otherwise the envelope is v2 and must
|
||||
/// not include `states[]`.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum RecordedTrace {
|
||||
V2(TraceV2),
|
||||
V3(TraceV3),
|
||||
}
|
||||
|
||||
impl RecordedTrace {
|
||||
pub fn classify_value(v: &serde_json::Value) -> Result<Self, String> {
|
||||
if let Some(ver) = v.get("version").and_then(|x| x.as_u64()) {
|
||||
if ver != u64::from(TRACE_VERSION_V3) {
|
||||
return Err(format!("unsupported trace version {ver}"));
|
||||
}
|
||||
if v.get("pages").is_some() {
|
||||
return Err("trace v3 must not include legacy pages[]".into());
|
||||
}
|
||||
return serde_json::from_value(v.clone())
|
||||
.map(RecordedTrace::V3)
|
||||
.map_err(|e| e.to_string());
|
||||
}
|
||||
if v.get("states").is_some() {
|
||||
return Err("trace v2 must not include states[]; set version: 3 for Trace v3".into());
|
||||
}
|
||||
if v.get("pages").is_some() {
|
||||
return serde_json::from_value(v.clone())
|
||||
.map(RecordedTrace::V2)
|
||||
.map_err(|e| e.to_string());
|
||||
}
|
||||
Err("ambiguous or unparseable trace".into())
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for RecordedTrace {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
match self {
|
||||
RecordedTrace::V2(t) => t.serialize(serializer),
|
||||
RecordedTrace::V3(t) => t.serialize(serializer),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for RecordedTrace {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let value = serde_json::Value::deserialize(deserializer)?;
|
||||
Self::classify_value(&value).map_err(serde::de::Error::custom)
|
||||
}
|
||||
}
|
||||
|
||||
impl JsonSchema for RecordedTrace {
|
||||
fn schema_name() -> String {
|
||||
"RecordedTrace".into()
|
||||
}
|
||||
|
||||
fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::schema::Schema {
|
||||
// Mirror classify_value: a numeric `version` selects v3; otherwise v2.
|
||||
// TraceV2 stays open for unknown extension fields but forbids `states[]`
|
||||
// and a numeric `version`. TraceV3 already denies `pages[]`.
|
||||
let mut version_props = schemars::Map::new();
|
||||
version_props.insert(
|
||||
"version".into(),
|
||||
schemars::schema::SchemaObject {
|
||||
instance_type: Some(schemars::schema::InstanceType::Integer.into()),
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
let mut required = schemars::Set::new();
|
||||
required.insert("version".into());
|
||||
|
||||
schemars::schema::SchemaObject {
|
||||
subschemas: Some(Box::new(schemars::schema::SubschemaValidation {
|
||||
if_schema: Some(Box::new(
|
||||
schemars::schema::SchemaObject {
|
||||
object: Some(Box::new(schemars::schema::ObjectValidation {
|
||||
properties: version_props,
|
||||
required,
|
||||
..Default::default()
|
||||
})),
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
)),
|
||||
then_schema: Some(Box::new(generator.subschema_for::<TraceV3>())),
|
||||
else_schema: Some(Box::new(generator.subschema_for::<TraceV2>())),
|
||||
..Default::default()
|
||||
})),
|
||||
..Default::default()
|
||||
}
|
||||
.into()
|
||||
}
|
||||
}
|
||||
|
||||
/// One recorded step — v2 (`page`) or v3 (`state` / `result.state`).
|
||||
///
|
||||
/// Classification matches [`RecordedStep::classify_value`]: `state` selects v3
|
||||
/// (and forbids `page`); otherwise the step is v2 and must include `page`.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum RecordedStep {
|
||||
V2(StepV2),
|
||||
V3(StepV3),
|
||||
}
|
||||
|
||||
impl RecordedStep {
|
||||
pub fn classify_value(v: &serde_json::Value) -> Result<Self, String> {
|
||||
if v.get("state").is_some() {
|
||||
if v.get("page").is_some() {
|
||||
return Err("step v3 must not include legacy page".into());
|
||||
}
|
||||
return serde_json::from_value(v.clone())
|
||||
.map(RecordedStep::V3)
|
||||
.map_err(|e| e.to_string());
|
||||
}
|
||||
if v.get("page").is_some() {
|
||||
return serde_json::from_value(v.clone())
|
||||
.map(RecordedStep::V2)
|
||||
.map_err(|e| e.to_string());
|
||||
}
|
||||
Err("ambiguous or unparseable step".into())
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for RecordedStep {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
match self {
|
||||
RecordedStep::V2(s) => s.serialize(serializer),
|
||||
RecordedStep::V3(s) => s.serialize(serializer),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for RecordedStep {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let value = serde_json::Value::deserialize(deserializer)?;
|
||||
Self::classify_value(&value).map_err(serde::de::Error::custom)
|
||||
}
|
||||
}
|
||||
|
||||
impl JsonSchema for RecordedStep {
|
||||
fn schema_name() -> String {
|
||||
"RecordedStep".into()
|
||||
}
|
||||
|
||||
fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::schema::Schema {
|
||||
// Mirror classify_value: `state` selects v3; otherwise v2.
|
||||
// Mixed page + state is rejected at this layer because StepV2/StepV3
|
||||
// still allow unknown extension fields.
|
||||
let mut required = schemars::Set::new();
|
||||
required.insert("state".into());
|
||||
let mut mixed_keys = schemars::Set::new();
|
||||
mixed_keys.insert("page".into());
|
||||
mixed_keys.insert("state".into());
|
||||
|
||||
schemars::schema::SchemaObject {
|
||||
subschemas: Some(Box::new(schemars::schema::SubschemaValidation {
|
||||
if_schema: Some(Box::new(
|
||||
schemars::schema::SchemaObject {
|
||||
object: Some(Box::new(schemars::schema::ObjectValidation {
|
||||
required,
|
||||
..Default::default()
|
||||
})),
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
)),
|
||||
then_schema: Some(Box::new(generator.subschema_for::<StepV3>())),
|
||||
else_schema: Some(Box::new(generator.subschema_for::<StepV2>())),
|
||||
not: Some(Box::new(
|
||||
schemars::schema::SchemaObject {
|
||||
object: Some(Box::new(schemars::schema::ObjectValidation {
|
||||
required: mixed_keys,
|
||||
..Default::default()
|
||||
})),
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
)),
|
||||
..Default::default()
|
||||
})),
|
||||
..Default::default()
|
||||
}
|
||||
.into()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct RecordStopResult {
|
||||
pub trace: Trace,
|
||||
pub trace: RecordedTrace,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
@@ -183,281 +259,197 @@ pub struct RecordAwaitParams {
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct RecordAwaitResult {
|
||||
pub trace: Trace,
|
||||
pub trace: RecordedTrace,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
fn sample_common(id: u32) -> StepCommon {
|
||||
StepCommon {
|
||||
id,
|
||||
page: "p1".into(),
|
||||
effect: None,
|
||||
#[test]
|
||||
fn recorded_trace_classifies_v2_and_v3() {
|
||||
let v2 = json!({
|
||||
"recorded_at": "2026-07-21T08:00:00Z",
|
||||
"entry": { "start_url": "https://example.com/" },
|
||||
"pages": [{ "id": "p1", "url": "https://example.com/" }],
|
||||
"steps": []
|
||||
});
|
||||
match RecordedTrace::classify_value(&v2).unwrap() {
|
||||
RecordedTrace::V2(_) => {}
|
||||
other => panic!("expected v2, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_target() -> TargetDescriptor {
|
||||
TargetDescriptor {
|
||||
role: Some("button".into()),
|
||||
name: Some("发布".into()),
|
||||
tag: "button".into(),
|
||||
name_attr: None,
|
||||
placeholder: None,
|
||||
nearby_label: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_trace() -> Trace {
|
||||
Trace {
|
||||
recorded_at: "2026-07-17T09:01:10Z".into(),
|
||||
started_at: Some("2026-07-17T09:00:00Z".into()),
|
||||
purpose: Some("发布一篇文章".into()),
|
||||
entry: TraceEntry {
|
||||
start_url: "https://x.com/editor".into(),
|
||||
},
|
||||
pages: vec![
|
||||
PageRef {
|
||||
id: "p1".into(),
|
||||
url: "https://x.com/editor".into(),
|
||||
title: Some("写文章".into()),
|
||||
},
|
||||
PageRef {
|
||||
id: "p2".into(),
|
||||
url: "https://x.com/p/99".into(),
|
||||
title: None,
|
||||
},
|
||||
],
|
||||
steps: vec![
|
||||
Step::Fill {
|
||||
common: sample_common(1),
|
||||
target: TargetDescriptor {
|
||||
role: Some("textbox".into()),
|
||||
name: Some("标题".into()),
|
||||
tag: "input".into(),
|
||||
name_attr: None,
|
||||
placeholder: None,
|
||||
nearby_label: None,
|
||||
},
|
||||
value: "我的第一篇文章".into(),
|
||||
redacted: None,
|
||||
},
|
||||
Step::Select {
|
||||
common: sample_common(3),
|
||||
target: TargetDescriptor {
|
||||
role: Some("combobox".into()),
|
||||
name: Some("分类".into()),
|
||||
tag: "select".into(),
|
||||
name_attr: None,
|
||||
placeholder: None,
|
||||
nearby_label: None,
|
||||
},
|
||||
selection: vec![SelectedOption {
|
||||
value: "tech".into(),
|
||||
label: Some("技术分享".into()),
|
||||
}],
|
||||
},
|
||||
Step::Click {
|
||||
common: StepCommon {
|
||||
effect: Some(StepEffect {
|
||||
navigated_to: "p2".into(),
|
||||
}),
|
||||
..sample_common(4)
|
||||
},
|
||||
target: sample_target(),
|
||||
},
|
||||
],
|
||||
let v3 = json!({
|
||||
"version": 3,
|
||||
"recorded_at": "2026-07-21T08:00:00Z",
|
||||
"stopped_by": "user_finish",
|
||||
"entry": { "start_url": "https://example.com/" },
|
||||
"recorder": { "bsk": "0.1.10", "vom": 1 },
|
||||
"states": [],
|
||||
"steps": []
|
||||
});
|
||||
match RecordedTrace::classify_value(&v3).unwrap() {
|
||||
RecordedTrace::V3(t) => assert_eq!(t.version, 3),
|
||||
other => panic!("expected v3, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn step_click_round_trips() {
|
||||
let step = Step::Click {
|
||||
common: sample_common(1),
|
||||
target: sample_target(),
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v.get("op").and_then(|v| v.as_str()), Some("click"));
|
||||
assert_eq!(v.get("page").and_then(|v| v.as_str()), Some("p1"));
|
||||
assert!(v.get("key").is_none());
|
||||
let round: Step = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, step);
|
||||
fn recorded_trace_rejects_mixed_and_unsupported_versions() {
|
||||
let v2_with_states = json!({
|
||||
"recorded_at": "2026-07-21T08:00:00Z",
|
||||
"entry": { "start_url": "https://example.com/" },
|
||||
"pages": [{ "id": "p1", "url": "https://example.com/" }],
|
||||
"states": [],
|
||||
"steps": []
|
||||
});
|
||||
assert!(RecordedTrace::classify_value(&v2_with_states).is_err());
|
||||
|
||||
let v3_with_pages = json!({
|
||||
"version": 3,
|
||||
"recorded_at": "2026-07-21T08:00:00Z",
|
||||
"stopped_by": "user_finish",
|
||||
"entry": { "start_url": "https://example.com/" },
|
||||
"recorder": { "bsk": "0.1.10", "vom": 1 },
|
||||
"pages": [{ "id": "p1", "url": "https://example.com/" }],
|
||||
"states": [],
|
||||
"steps": []
|
||||
});
|
||||
assert!(RecordedTrace::classify_value(&v3_with_pages).is_err());
|
||||
|
||||
let unsupported = json!({
|
||||
"version": 2,
|
||||
"recorded_at": "2026-07-21T08:00:00Z",
|
||||
"stopped_by": "user_finish",
|
||||
"entry": { "start_url": "https://example.com/" },
|
||||
"recorder": { "bsk": "0.1.10", "vom": 1 },
|
||||
"states": [],
|
||||
"steps": []
|
||||
});
|
||||
assert!(RecordedTrace::classify_value(&unsupported).is_err());
|
||||
assert!(serde_json::from_value::<TraceV3>(unsupported).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn step_fill_with_raw_value_round_trips() {
|
||||
let step = Step::Fill {
|
||||
common: sample_common(2),
|
||||
target: TargetDescriptor {
|
||||
role: Some("textbox".into()),
|
||||
name: Some("服务名称".into()),
|
||||
tag: "input".into(),
|
||||
name_attr: Some("serviceName".into()),
|
||||
placeholder: None,
|
||||
nearby_label: None,
|
||||
},
|
||||
value: "my-svc".into(),
|
||||
redacted: None,
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v["op"], "fill");
|
||||
assert_eq!(v["value"], "my-svc");
|
||||
let round: Step = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, step);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn step_fill_password_is_redacted() {
|
||||
let step = Step::Fill {
|
||||
common: sample_common(1),
|
||||
target: TargetDescriptor {
|
||||
role: Some("textbox".into()),
|
||||
name: Some("密码".into()),
|
||||
tag: "input".into(),
|
||||
name_attr: None,
|
||||
placeholder: None,
|
||||
nearby_label: None,
|
||||
},
|
||||
value: "***".into(),
|
||||
redacted: Some(true),
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v["value"], "***");
|
||||
assert_eq!(v["redacted"], true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn step_select_uses_object_array() {
|
||||
let step = Step::Select {
|
||||
common: sample_common(3),
|
||||
target: TargetDescriptor {
|
||||
role: Some("combobox".into()),
|
||||
name: Some("分类".into()),
|
||||
tag: "select".into(),
|
||||
name_attr: None,
|
||||
placeholder: None,
|
||||
nearby_label: None,
|
||||
},
|
||||
selection: vec![SelectedOption {
|
||||
value: "tech".into(),
|
||||
label: Some("技术分享".into()),
|
||||
}],
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v["selection"][0]["value"], "tech");
|
||||
assert_eq!(v["selection"][0]["label"], "技术分享");
|
||||
let round: Step = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, step);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn step_navigate_round_trips() {
|
||||
let step = Step::Navigate {
|
||||
common: sample_common(1),
|
||||
to: "https://example.com".into(),
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v["op"], "navigate");
|
||||
assert_eq!(v["to"], "https://example.com");
|
||||
let round: Step = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, step);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn step_press_with_modifiers_round_trips() {
|
||||
let step = Step::Press {
|
||||
common: StepCommon {
|
||||
effect: Some(StepEffect {
|
||||
navigated_to: "p2".into(),
|
||||
}),
|
||||
..sample_common(2)
|
||||
},
|
||||
key: "Enter".into(),
|
||||
modifiers: Some(vec![KeyModifier::Ctrl, KeyModifier::Shift]),
|
||||
target: Some(sample_target()),
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v["key"], "Enter");
|
||||
assert_eq!(v["modifiers"], json!(["ctrl", "shift"]));
|
||||
assert_eq!(v["effect"]["navigated_to"], "p2");
|
||||
let round: Step = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, step);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_record_only_round_trips() {
|
||||
let trace = sample_trace();
|
||||
let v = serde_json::to_value(&trace).unwrap();
|
||||
assert!(v.get("version").is_none());
|
||||
fn assert_classifies_by_integer_version(schema: &serde_json::Value) {
|
||||
assert_eq!(
|
||||
v.get("purpose").and_then(|v| v.as_str()),
|
||||
Some("发布一篇文章")
|
||||
schema["if"]["required"],
|
||||
json!(["version"]),
|
||||
"numeric version is the RecordedTrace classification key"
|
||||
);
|
||||
assert!(v.get("parameters").is_none());
|
||||
assert!(v.get("site").is_none());
|
||||
assert!(v.get("goal").is_none());
|
||||
assert_eq!(
|
||||
v.get("entry")
|
||||
.and_then(|e| e.get("start_url"))
|
||||
.and_then(|u| u.as_str()),
|
||||
Some("https://x.com/editor")
|
||||
);
|
||||
let round: Trace = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, trace);
|
||||
assert_eq!(schema["if"]["properties"]["version"]["type"], "integer");
|
||||
assert_eq!(schema["then"]["$ref"], "#/definitions/TraceV3");
|
||||
assert_eq!(schema["else"]["$ref"], "#/definitions/TraceV2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn discriminated_union_click_ignores_extra_key_field() {
|
||||
let bad = json!({
|
||||
fn recorded_trace_schema_includes_v2_and_v3() {
|
||||
let schema = serde_json::to_value(schemars::schema_for!(RecordStopResult)).unwrap();
|
||||
assert_classifies_by_integer_version(&schema["definitions"]["RecordedTrace"]);
|
||||
assert_eq!(
|
||||
schema["definitions"]["TraceV3"]["properties"]["version"]["const"],
|
||||
3
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recorded_trace_schema_follows_classify_value() {
|
||||
let standalone = serde_json::to_value(schemars::schema_for!(RecordedTrace)).unwrap();
|
||||
assert_classifies_by_integer_version(&standalone);
|
||||
assert!(standalone["definitions"].get("TraceV2").is_some());
|
||||
assert!(standalone["definitions"].get("TraceV3").is_some());
|
||||
|
||||
let stop_result = serde_json::to_value(schemars::schema_for!(RecordStopResult)).unwrap();
|
||||
assert_classifies_by_integer_version(&stop_result["definitions"]["RecordedTrace"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standalone_trace_schema_is_recorded_trace_union() {
|
||||
let schema = serde_json::to_value(schemars::schema_for!(RecordedTrace)).unwrap();
|
||||
assert_eq!(schema["title"], "RecordedTrace");
|
||||
assert_classifies_by_integer_version(&schema);
|
||||
assert!(schema["definitions"].get("TraceV2").is_some());
|
||||
assert!(schema["definitions"].get("TraceV3").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recorded_trace_accepts_v2_with_unknown_extension_fields() {
|
||||
let v2 = json!({
|
||||
"recorded_at": "2026-07-21T08:00:00Z",
|
||||
"entry": { "start_url": "https://example.com/" },
|
||||
"pages": [{ "id": "p1", "url": "https://example.com/" }],
|
||||
"steps": [],
|
||||
"meta": { "tool": "legacy-exporter" }
|
||||
});
|
||||
match RecordedTrace::classify_value(&v2).unwrap() {
|
||||
RecordedTrace::V2(trace) => assert_eq!(trace.pages.len(), 1),
|
||||
other => panic!("expected v2, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn v2_click() -> serde_json::Value {
|
||||
json!({
|
||||
"op": "click",
|
||||
"id": 1,
|
||||
"page": "p1",
|
||||
"target": { "tag": "button", "name": "OK" },
|
||||
"key": "Enter"
|
||||
});
|
||||
let step: Step = serde_json::from_value(bad).unwrap();
|
||||
assert!(matches!(step, Step::Click { .. }));
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert!(v.get("key").is_none());
|
||||
"target": { "tag": "button", "role": "button", "name": "发布" }
|
||||
})
|
||||
}
|
||||
|
||||
fn v3_click() -> serde_json::Value {
|
||||
json!({
|
||||
"op": "click",
|
||||
"id": 1,
|
||||
"state": "s1",
|
||||
"result": { "state": "s2" },
|
||||
"target": { "ref": "e1", "role": "button", "name": "发布" }
|
||||
})
|
||||
}
|
||||
|
||||
fn assert_step_classifies_by_state(schema: &serde_json::Value) {
|
||||
assert_eq!(
|
||||
schema["if"]["required"],
|
||||
json!(["state"]),
|
||||
"state is the RecordedStep classification key"
|
||||
);
|
||||
assert_eq!(schema["then"]["$ref"], "#/definitions/StepV3");
|
||||
assert_eq!(schema["else"]["$ref"], "#/definitions/StepV2");
|
||||
let mut forbidden = schema["not"]["required"]
|
||||
.as_array()
|
||||
.expect("RecordedStep schema must reject mixed page + state steps")
|
||||
.iter()
|
||||
.filter_map(|v| v.as_str())
|
||||
.collect::<Vec<_>>();
|
||||
forbidden.sort_unstable();
|
||||
assert_eq!(forbidden, ["page", "state"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extension_trace_deserializes() {
|
||||
let v = json!({
|
||||
"recorded_at": "2026-07-21T08:00:00Z",
|
||||
"started_at": "2026-07-21T07:59:00Z",
|
||||
"purpose": "demo",
|
||||
"entry": { "start_url": "https://example.com/editor" },
|
||||
"pages": [
|
||||
{ "id": "p1", "url": "https://example.com/editor" },
|
||||
{ "id": "p2", "url": "https://example.com/p/99" }
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"op": "fill",
|
||||
"id": 1,
|
||||
"page": "p1",
|
||||
"target": { "tag": "input", "role": "textbox", "name": "标题" },
|
||||
"value": "hello"
|
||||
},
|
||||
{
|
||||
"op": "click",
|
||||
"id": 2,
|
||||
"page": "p1",
|
||||
"target": { "tag": "button", "role": "button", "name": "发布" },
|
||||
"effect": { "navigated_to": "p2" }
|
||||
}
|
||||
]
|
||||
});
|
||||
let trace: Trace = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(trace.entry.start_url, "https://example.com/editor");
|
||||
assert_eq!(trace.pages.len(), 2);
|
||||
assert_eq!(trace.steps.len(), 2);
|
||||
fn recorded_step_classifies_v2_and_v3() {
|
||||
match RecordedStep::classify_value(&v2_click()).unwrap() {
|
||||
RecordedStep::V2(StepV2::Click { .. }) => {}
|
||||
other => panic!("expected v2 click, got {other:?}"),
|
||||
}
|
||||
match RecordedStep::classify_value(&v3_click()).unwrap() {
|
||||
RecordedStep::V3(StepV3::Click { .. }) => {}
|
||||
other => panic!("expected v3 click, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recorded_step_rejects_mixed_page_and_state() {
|
||||
let mut mixed = v2_click();
|
||||
mixed["state"] = json!("s1");
|
||||
mixed["result"] = json!({ "state": "s2" });
|
||||
assert!(RecordedStep::classify_value(&mixed).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recorded_step_schema_follows_classify_value() {
|
||||
let schema = serde_json::to_value(schemars::schema_for!(RecordedStep)).unwrap();
|
||||
assert_eq!(schema["title"], "RecordedStep");
|
||||
assert_step_classifies_by_state(&schema);
|
||||
assert!(schema["definitions"].get("StepV2").is_some());
|
||||
assert!(schema["definitions"].get("StepV3").is_some());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TraceEntry {
|
||||
pub start_url: String,
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
//! Trace v2 — record-only user-action log with `pages[]` and step `page` refs.
|
||||
//!
|
||||
//! Legacy wire format with no top-level `version` field.
|
||||
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::interaction::KeyModifier;
|
||||
use super::record_common::TraceEntry;
|
||||
|
||||
/// Stable semantic handle for an interacted element (v2).
|
||||
///
|
||||
/// `name` and `nearby_label` are **untrusted page text**.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TargetDescriptorV2 {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub role: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub name: Option<String>,
|
||||
pub tag: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub name_attr: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub placeholder: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub nearby_label: Option<String>,
|
||||
}
|
||||
|
||||
/// Page context dictionary entry — referenced by steps via `page` id.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct PageRefV2 {
|
||||
pub id: String,
|
||||
pub url: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub title: Option<String>,
|
||||
}
|
||||
|
||||
/// One selected option (`select` op).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct SelectedOptionV2 {
|
||||
pub value: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub label: Option<String>,
|
||||
}
|
||||
|
||||
/// Observed navigation after a step (objective fact only).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct StepEffectV2 {
|
||||
/// Reference into `pages[]` for the destination page.
|
||||
pub navigated_to: String,
|
||||
}
|
||||
|
||||
/// Fields shared by every v2 step variant (flattened in JSON).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct StepCommonV2 {
|
||||
pub id: u32,
|
||||
/// Reference into `pages[]`.
|
||||
pub page: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub effect: Option<StepEffectV2>,
|
||||
}
|
||||
|
||||
/// One recorded user action — discriminated union by `op` (v2).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(tag = "op", rename_all = "snake_case")]
|
||||
pub enum StepV2 {
|
||||
Navigate {
|
||||
#[serde(flatten)]
|
||||
common: StepCommonV2,
|
||||
to: String,
|
||||
},
|
||||
Click {
|
||||
#[serde(flatten)]
|
||||
common: StepCommonV2,
|
||||
target: TargetDescriptorV2,
|
||||
},
|
||||
Hover {
|
||||
#[serde(flatten)]
|
||||
common: StepCommonV2,
|
||||
target: TargetDescriptorV2,
|
||||
},
|
||||
Fill {
|
||||
#[serde(flatten)]
|
||||
common: StepCommonV2,
|
||||
target: TargetDescriptorV2,
|
||||
value: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
redacted: Option<bool>,
|
||||
},
|
||||
Select {
|
||||
#[serde(flatten)]
|
||||
common: StepCommonV2,
|
||||
target: TargetDescriptorV2,
|
||||
selection: Vec<SelectedOptionV2>,
|
||||
},
|
||||
Press {
|
||||
#[serde(flatten)]
|
||||
common: StepCommonV2,
|
||||
key: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
modifiers: Option<Vec<KeyModifier>>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
target: Option<TargetDescriptorV2>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Persisted user-action trace exported by legacy `tool.record_stop` / `await`.
|
||||
///
|
||||
/// Unknown extension fields are ignored so older traces remain readable.
|
||||
/// `states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace`
|
||||
/// classification — serde still ignores them, but the JSON Schema rejects them.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct TraceV2 {
|
||||
/// RFC 3339 timestamp when recording stopped.
|
||||
pub recorded_at: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub started_at: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub purpose: Option<String>,
|
||||
pub entry: TraceEntry,
|
||||
pub pages: Vec<PageRefV2>,
|
||||
pub steps: Vec<StepV2>,
|
||||
}
|
||||
|
||||
fn constrain_trace_v2_schema(mut schema: schemars::schema::Schema) -> schemars::schema::Schema {
|
||||
let schemars::schema::Schema::Object(obj) = &mut schema else {
|
||||
return schema;
|
||||
};
|
||||
obj.metadata().title = Some("TraceV2".into());
|
||||
obj.metadata().description = Some(
|
||||
"Persisted user-action trace exported by legacy `tool.record_stop` / `await`.\n\n\
|
||||
Unknown extension fields are ignored so older traces remain readable. \
|
||||
`states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace` classification."
|
||||
.into(),
|
||||
);
|
||||
|
||||
let object = obj.object.get_or_insert_with(Default::default);
|
||||
object
|
||||
.properties
|
||||
.insert("states".into(), schemars::schema::Schema::Bool(false));
|
||||
object.properties.insert(
|
||||
"version".into(),
|
||||
schemars::schema::SchemaObject {
|
||||
metadata: Some(Box::new(schemars::schema::Metadata {
|
||||
description: Some(
|
||||
"Numeric version selects Trace v3. Legacy v2 envelopes omit this field.".into(),
|
||||
),
|
||||
..Default::default()
|
||||
})),
|
||||
subschemas: Some(Box::new(schemars::schema::SubschemaValidation {
|
||||
not: Some(Box::new(
|
||||
schemars::schema::SchemaObject {
|
||||
instance_type: Some(schemars::schema::InstanceType::Integer.into()),
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
)),
|
||||
..Default::default()
|
||||
})),
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
);
|
||||
schema
|
||||
}
|
||||
|
||||
impl JsonSchema for TraceV2 {
|
||||
fn schema_name() -> String {
|
||||
"TraceV2".into()
|
||||
}
|
||||
|
||||
fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::schema::Schema {
|
||||
#[derive(JsonSchema)]
|
||||
#[allow(dead_code)]
|
||||
struct TraceV2Shape {
|
||||
/// RFC 3339 timestamp when recording stopped.
|
||||
recorded_at: String,
|
||||
started_at: Option<String>,
|
||||
purpose: Option<String>,
|
||||
entry: TraceEntry,
|
||||
pages: Vec<PageRefV2>,
|
||||
steps: Vec<StepV2>,
|
||||
}
|
||||
|
||||
constrain_trace_v2_schema(TraceV2Shape::json_schema(generator))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
fn sample_common(id: u32) -> StepCommonV2 {
|
||||
StepCommonV2 {
|
||||
id,
|
||||
page: "p1".into(),
|
||||
effect: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_target() -> TargetDescriptorV2 {
|
||||
TargetDescriptorV2 {
|
||||
role: Some("button".into()),
|
||||
name: Some("发布".into()),
|
||||
tag: "button".into(),
|
||||
name_attr: None,
|
||||
placeholder: None,
|
||||
nearby_label: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_v2_has_no_version_field() {
|
||||
let trace = TraceV2 {
|
||||
recorded_at: "2026-07-17T09:01:10Z".into(),
|
||||
started_at: None,
|
||||
purpose: None,
|
||||
entry: TraceEntry {
|
||||
start_url: "https://example.com/".into(),
|
||||
},
|
||||
pages: vec![PageRefV2 {
|
||||
id: "p1".into(),
|
||||
url: "https://example.com/".into(),
|
||||
title: None,
|
||||
}],
|
||||
steps: vec![StepV2::Click {
|
||||
common: sample_common(1),
|
||||
target: sample_target(),
|
||||
}],
|
||||
};
|
||||
let v = serde_json::to_value(&trace).unwrap();
|
||||
assert!(v.get("version").is_none());
|
||||
assert!(v.get("pages").is_some());
|
||||
assert!(v.get("states").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extension_v2_trace_deserializes() {
|
||||
let v = json!({
|
||||
"recorded_at": "2026-07-21T08:00:00Z",
|
||||
"started_at": "2026-07-21T07:59:00Z",
|
||||
"purpose": "demo",
|
||||
"entry": { "start_url": "https://example.com/editor" },
|
||||
"pages": [
|
||||
{ "id": "p1", "url": "https://example.com/editor" },
|
||||
{ "id": "p2", "url": "https://example.com/p/99" }
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"op": "fill",
|
||||
"id": 1,
|
||||
"page": "p1",
|
||||
"target": { "tag": "input", "role": "textbox", "name": "标题" },
|
||||
"value": "hello"
|
||||
},
|
||||
{
|
||||
"op": "click",
|
||||
"id": 2,
|
||||
"page": "p1",
|
||||
"target": { "tag": "button", "role": "button", "name": "发布" },
|
||||
"effect": { "navigated_to": "p2" }
|
||||
}
|
||||
]
|
||||
});
|
||||
let trace: TraceV2 = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(trace.pages.len(), 2);
|
||||
assert_eq!(trace.steps.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn v2_hover_steps_round_trip() {
|
||||
let value = json!({
|
||||
"recorded_at": "2026-07-21T08:00:00Z",
|
||||
"entry": { "start_url": "https://example.com/" },
|
||||
"pages": [{ "id": "p1", "url": "https://example.com/" }],
|
||||
"steps": [{
|
||||
"op": "hover",
|
||||
"id": 1,
|
||||
"page": "p1",
|
||||
"target": { "tag": "span", "role": "button", "name": "Account" }
|
||||
}]
|
||||
});
|
||||
|
||||
let trace: TraceV2 = serde_json::from_value(value).unwrap();
|
||||
assert!(matches!(trace.steps.as_slice(), [StepV2::Hover { .. }]));
|
||||
assert_eq!(
|
||||
serde_json::to_value(&trace).unwrap()["steps"][0]["op"],
|
||||
"hover"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_v2_ignores_unknown_extension_fields() {
|
||||
let value = json!({
|
||||
"recorded_at": "2026-07-21T08:00:00Z",
|
||||
"entry": { "start_url": "https://example.com/" },
|
||||
"pages": [{ "id": "p1", "url": "https://example.com/" }],
|
||||
"steps": [],
|
||||
"meta": { "tool": "legacy-exporter" }
|
||||
});
|
||||
let trace: TraceV2 = serde_json::from_value(value).unwrap();
|
||||
assert_eq!(trace.pages.len(), 1);
|
||||
assert!(trace.steps.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_v2_schema_allows_additional_properties() {
|
||||
let schema = serde_json::to_value(schemars::schema_for!(TraceV2)).unwrap();
|
||||
assert_ne!(
|
||||
schema.get("additionalProperties"),
|
||||
Some(&serde_json::Value::Bool(false)),
|
||||
"Trace v2 must keep accepting traces with unknown extension fields"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_v2_schema_forbids_classification_keys() {
|
||||
let schema = serde_json::to_value(schemars::schema_for!(TraceV2)).unwrap();
|
||||
assert_eq!(
|
||||
schema["properties"]["states"],
|
||||
json!(false),
|
||||
"Trace v2 schema must reject states[] (reserved for v3)"
|
||||
);
|
||||
assert_eq!(
|
||||
schema["properties"]["version"]["not"]["type"], "integer",
|
||||
"Trace v2 schema must reject a numeric version (that selects the v3 path)"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,428 @@
|
||||
//! Trace v3 — state-action-state log with `states[]` and step `state` refs.
|
||||
//!
|
||||
//! Wire format with top-level `version: 3`.
|
||||
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Deserializer, Serialize};
|
||||
|
||||
use super::interaction::KeyModifier;
|
||||
use super::record_common::TraceEntry;
|
||||
|
||||
pub const TRACE_VERSION_V3: u32 = 3;
|
||||
pub const VOM_FORMAT_VERSION: u32 = 1;
|
||||
|
||||
fn deserialize_trace_v3_version<'de, D>(deserializer: D) -> Result<u32, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let version = u32::deserialize(deserializer)?;
|
||||
if version != TRACE_VERSION_V3 {
|
||||
return Err(serde::de::Error::custom(format!(
|
||||
"unsupported trace version {version} (expected {TRACE_VERSION_V3})"
|
||||
)));
|
||||
}
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
fn trace_v3_version_schema(_: &mut schemars::r#gen::SchemaGenerator) -> schemars::schema::Schema {
|
||||
schemars::schema::SchemaObject {
|
||||
instance_type: Some(schemars::schema::InstanceType::Integer.into()),
|
||||
const_value: Some(serde_json::json!(TRACE_VERSION_V3)),
|
||||
..Default::default()
|
||||
}
|
||||
.into()
|
||||
}
|
||||
|
||||
/// Stable semantic handle for an interacted element within a page observation.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TargetDescriptorV3 {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none", rename = "ref")]
|
||||
pub element_ref: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub role: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub name: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub ctx: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
pub unmatched: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct RecorderInfo {
|
||||
pub bsk: String,
|
||||
pub vom: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum StopReason {
|
||||
UserFinish,
|
||||
CliStop,
|
||||
}
|
||||
|
||||
/// Page observation dictionary entry — referenced by steps via `state` / `result.state`.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TraceStateV3 {
|
||||
pub id: String,
|
||||
pub url: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub title: Option<String>,
|
||||
/// Full page observation (front matter + VOM body + annotations).
|
||||
pub body: String,
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
pub truncated: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct StepResultV3 {
|
||||
pub state: String,
|
||||
}
|
||||
|
||||
/// Fields shared by every step variant (flattened in JSON).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct StepCommonV3 {
|
||||
pub id: u32,
|
||||
/// Observation id immediately before this action.
|
||||
pub state: String,
|
||||
pub result: StepResultV3,
|
||||
}
|
||||
|
||||
/// One selected option (`select` op).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct SelectedOptionV3 {
|
||||
pub value: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub label: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum NavigationCause {
|
||||
UserTyped,
|
||||
Link,
|
||||
FormSubmit,
|
||||
Reload,
|
||||
History,
|
||||
Script,
|
||||
Browser,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum FillCommit {
|
||||
Enter,
|
||||
Suggestion,
|
||||
Blur,
|
||||
}
|
||||
|
||||
/// One recorded user action — discriminated union by `op`.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(tag = "op", rename_all = "snake_case")]
|
||||
pub enum StepV3 {
|
||||
Navigate {
|
||||
#[serde(flatten)]
|
||||
common: StepCommonV3,
|
||||
to: String,
|
||||
cause: NavigationCause,
|
||||
},
|
||||
Click {
|
||||
#[serde(flatten)]
|
||||
common: StepCommonV3,
|
||||
target: TargetDescriptorV3,
|
||||
},
|
||||
Hover {
|
||||
#[serde(flatten)]
|
||||
common: StepCommonV3,
|
||||
target: TargetDescriptorV3,
|
||||
},
|
||||
Fill {
|
||||
#[serde(flatten)]
|
||||
common: StepCommonV3,
|
||||
target: TargetDescriptorV3,
|
||||
value: String,
|
||||
commit: FillCommit,
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
redacted: bool,
|
||||
},
|
||||
Select {
|
||||
#[serde(flatten)]
|
||||
common: StepCommonV3,
|
||||
target: TargetDescriptorV3,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
selection: Vec<SelectedOptionV3>,
|
||||
},
|
||||
Press {
|
||||
#[serde(flatten)]
|
||||
common: StepCommonV3,
|
||||
key: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
modifiers: Option<Vec<KeyModifier>>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
target: Option<TargetDescriptorV3>,
|
||||
},
|
||||
Scroll {
|
||||
#[serde(flatten)]
|
||||
common: StepCommonV3,
|
||||
},
|
||||
}
|
||||
|
||||
/// Wire trace returned by `tool.record_stop` / `await`.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct TraceV3 {
|
||||
#[serde(deserialize_with = "deserialize_trace_v3_version")]
|
||||
#[schemars(schema_with = "trace_v3_version_schema")]
|
||||
pub version: u32,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub purpose: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub started_at: Option<String>,
|
||||
pub recorded_at: String,
|
||||
pub stopped_by: StopReason,
|
||||
pub entry: TraceEntry,
|
||||
pub recorder: RecorderInfo,
|
||||
pub states: Vec<TraceStateV3>,
|
||||
pub steps: Vec<StepV3>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
fn sample_common(id: u32, state: &str, result_state: &str) -> StepCommonV3 {
|
||||
StepCommonV3 {
|
||||
id,
|
||||
state: state.into(),
|
||||
result: StepResultV3 {
|
||||
state: result_state.into(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_target() -> TargetDescriptorV3 {
|
||||
TargetDescriptorV3 {
|
||||
element_ref: Some("e21".into()),
|
||||
role: Some("button".into()),
|
||||
name: Some("发布".into()),
|
||||
ctx: Some("金桔柠檬 6 号".into()),
|
||||
unmatched: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_trace() -> TraceV3 {
|
||||
TraceV3 {
|
||||
version: TRACE_VERSION_V3,
|
||||
purpose: Some("把草稿商品发布上架".into()),
|
||||
started_at: Some("2026-08-10T02:10:41.080Z".into()),
|
||||
recorded_at: "2026-08-10T02:12:55.360Z".into(),
|
||||
stopped_by: StopReason::UserFinish,
|
||||
entry: TraceEntry {
|
||||
start_url: "https://example.com/".into(),
|
||||
},
|
||||
recorder: RecorderInfo {
|
||||
bsk: "0.1.10".into(),
|
||||
vom: VOM_FORMAT_VERSION,
|
||||
},
|
||||
states: vec![
|
||||
TraceStateV3 {
|
||||
id: "s1".into(),
|
||||
url: "https://example.com/".into(),
|
||||
title: Some("Example Domain".into()),
|
||||
body: "@vom 1\nRootWebArea \"Example Domain\"".into(),
|
||||
truncated: false,
|
||||
},
|
||||
TraceStateV3 {
|
||||
id: "s2".into(),
|
||||
url: "https://shop.example.com/products?status=draft".into(),
|
||||
title: Some("商品管理".into()),
|
||||
body: "@vom 1\nRootWebArea \"商品管理\"".into(),
|
||||
truncated: false,
|
||||
},
|
||||
],
|
||||
steps: vec![
|
||||
StepV3::Navigate {
|
||||
common: sample_common(1, "s1", "s2"),
|
||||
to: "https://shop.example.com/products?status=draft".into(),
|
||||
cause: NavigationCause::UserTyped,
|
||||
},
|
||||
StepV3::Fill {
|
||||
common: sample_common(2, "s2", "s2"),
|
||||
target: TargetDescriptorV3 {
|
||||
element_ref: Some("e12".into()),
|
||||
role: Some("textbox".into()),
|
||||
name: Some("搜索商品".into()),
|
||||
ctx: None,
|
||||
unmatched: false,
|
||||
},
|
||||
value: "金桔柠檬".into(),
|
||||
commit: FillCommit::Enter,
|
||||
redacted: false,
|
||||
},
|
||||
StepV3::Click {
|
||||
common: sample_common(3, "s2", "s2"),
|
||||
target: sample_target(),
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn step_click_round_trips() {
|
||||
let step = StepV3::Click {
|
||||
common: sample_common(1, "s1", "s2"),
|
||||
target: sample_target(),
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v.get("op").and_then(|v| v.as_str()), Some("click"));
|
||||
assert_eq!(v.get("state").and_then(|v| v.as_str()), Some("s1"));
|
||||
assert_eq!(v["result"]["state"], "s2");
|
||||
assert_eq!(v["target"]["ref"], "e21");
|
||||
let round: StepV3 = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, step);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn step_fill_with_commit_round_trips() {
|
||||
let step = StepV3::Fill {
|
||||
common: sample_common(2, "s2", "s3"),
|
||||
target: TargetDescriptorV3 {
|
||||
element_ref: Some("e12".into()),
|
||||
role: Some("textbox".into()),
|
||||
name: Some("搜索商品".into()),
|
||||
ctx: None,
|
||||
unmatched: false,
|
||||
},
|
||||
value: "browser skill".into(),
|
||||
commit: FillCommit::Enter,
|
||||
redacted: false,
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v["op"], "fill");
|
||||
assert_eq!(v["commit"], "enter");
|
||||
assert!(v.get("redacted").is_none());
|
||||
let round: StepV3 = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, step);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn step_fill_password_is_redacted() {
|
||||
let step = StepV3::Fill {
|
||||
common: sample_common(1, "s1", "s1"),
|
||||
target: TargetDescriptorV3 {
|
||||
element_ref: Some("e3".into()),
|
||||
role: Some("textbox".into()),
|
||||
name: Some("密码".into()),
|
||||
ctx: None,
|
||||
unmatched: false,
|
||||
},
|
||||
value: "***".into(),
|
||||
commit: FillCommit::Blur,
|
||||
redacted: true,
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v["value"], "***");
|
||||
assert_eq!(v["redacted"], true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn step_navigate_with_cause_round_trips() {
|
||||
let step = StepV3::Navigate {
|
||||
common: sample_common(1, "s1", "s2"),
|
||||
to: "https://example.com".into(),
|
||||
cause: NavigationCause::UserTyped,
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v["op"], "navigate");
|
||||
assert_eq!(v["cause"], "user_typed");
|
||||
let round: StepV3 = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, step);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_v3_round_trips() {
|
||||
let trace = sample_trace();
|
||||
let v = serde_json::to_value(&trace).unwrap();
|
||||
assert_eq!(v.get("version").and_then(|v| v.as_u64()), Some(3));
|
||||
assert!(v.get("pages").is_none());
|
||||
assert_eq!(v["recorder"]["vom"], 1);
|
||||
assert_eq!(v["states"].as_array().unwrap().len(), 2);
|
||||
let round: TraceV3 = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, trace);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_fields_are_omitted() {
|
||||
let step = StepV3::Click {
|
||||
common: sample_common(1, "s1", "s1"),
|
||||
target: TargetDescriptorV3 {
|
||||
element_ref: Some("e1".into()),
|
||||
role: Some("button".into()),
|
||||
name: Some("OK".into()),
|
||||
ctx: None,
|
||||
unmatched: false,
|
||||
},
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert!(v.get("unmatched").is_none());
|
||||
assert!(v["target"].get("ctx").is_none());
|
||||
assert!(v["target"].get("unmatched").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unmatched_target_serializes_flag() {
|
||||
let step = StepV3::Click {
|
||||
common: sample_common(1, "s1", "s2"),
|
||||
target: TargetDescriptorV3 {
|
||||
element_ref: None,
|
||||
role: Some("button".into()),
|
||||
name: Some("发布".into()),
|
||||
ctx: None,
|
||||
unmatched: true,
|
||||
},
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v["target"]["unmatched"], true);
|
||||
assert!(v["target"].get("ref").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extension_trace_deserializes() {
|
||||
let v = json!({
|
||||
"version": 3,
|
||||
"recorded_at": "2026-07-21T08:00:00Z",
|
||||
"started_at": "2026-07-21T07:59:00Z",
|
||||
"purpose": "demo",
|
||||
"stopped_by": "user_finish",
|
||||
"entry": { "start_url": "https://example.com/editor" },
|
||||
"recorder": { "bsk": "0.1.10", "vom": 1 },
|
||||
"states": [
|
||||
{ "id": "s1", "url": "https://example.com/editor", "body": "@vom 1" },
|
||||
{ "id": "s2", "url": "https://example.com/p/99", "body": "@vom 1" }
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"op": "fill",
|
||||
"id": 1,
|
||||
"state": "s1",
|
||||
"result": { "state": "s1" },
|
||||
"target": { "ref": "e1", "role": "textbox", "name": "标题" },
|
||||
"value": "hello",
|
||||
"commit": "blur"
|
||||
},
|
||||
{
|
||||
"op": "click",
|
||||
"id": 2,
|
||||
"state": "s1",
|
||||
"result": { "state": "s2" },
|
||||
"target": { "ref": "e2", "role": "button", "name": "发布" }
|
||||
}
|
||||
]
|
||||
});
|
||||
let trace: TraceV3 = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(trace.version, 3);
|
||||
assert_eq!(trace.states.len(), 2);
|
||||
assert_eq!(trace.steps.len(), 2);
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,7 @@
|
||||
* events, and never move the registry's current pointer.
|
||||
*/
|
||||
|
||||
import { createHash } from "node:crypto";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { readFile, unlink } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
@@ -78,6 +78,7 @@ const FAILURE_BACKOFF_THRESHOLD = 3;
|
||||
const FRAME_RING_SIZE = 2;
|
||||
|
||||
export class ObservationService {
|
||||
private readonly scratchNamespace = randomUUID();
|
||||
private readonly observations = new Map<string, SessionObservation>();
|
||||
private readonly listeners = new Set<(event: ObservationEvent) => void>();
|
||||
private readonly captureTimers = new Map<string, unknown>();
|
||||
@@ -304,9 +305,9 @@ export class ObservationService {
|
||||
if (current === undefined || current.dead === true) return;
|
||||
if (this.captureInFlight.has(sessionId)) return;
|
||||
this.captureInFlight.add(sessionId);
|
||||
// One fixed scratch path per session (overwritten each frame) and always
|
||||
// unlinked after the bytes are read — /tmp must not grow without bound.
|
||||
const outPath = join(tmpdir(), `bsk-obs-${sessionId}.png`);
|
||||
// Stable within this service, isolated from captures owned by other instances.
|
||||
const sessionKey = createHash("sha256").update(sessionId).digest("hex").slice(0, 16);
|
||||
const outPath = join(tmpdir(), `bsk-obs-${this.scratchNamespace}-${sessionKey}.png`);
|
||||
let writtenPath = outPath;
|
||||
try {
|
||||
const result = await this.deps.queue.run(sessionId, () =>
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
import { existsSync, mkdtempSync, writeFileSync } from "node:fs";
|
||||
import type { IncomingMessage, ServerResponse } from "node:http";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { basename, dirname } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
actionForLabel,
|
||||
@@ -548,8 +548,12 @@ describe("scratch files and thumbnail references", () => {
|
||||
);
|
||||
const firstOut = runner.calls.filter((c) => c.args[0] === "screenshot").at(-1)?.args;
|
||||
const pathOf = (args: string[]) => args[args.indexOf("--out") + 1];
|
||||
expect(pathOf(firstOut ?? [])).toBe(join(tmpdir(), "bsk-obs-s1.png"));
|
||||
expect(existsSync(pathOf(firstOut ?? []))).toBe(false);
|
||||
const firstPath = pathOf(firstOut ?? []);
|
||||
expect(dirname(firstPath)).toBe(tmpdir());
|
||||
expect(basename(firstPath)).toMatch(
|
||||
/^bsk-obs-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-[0-9a-f]{16}\.png$/,
|
||||
);
|
||||
expect(existsSync(firstPath)).toBe(false);
|
||||
// second frame: same path, deleted again
|
||||
service.endAction("s1");
|
||||
scheduler.runNext();
|
||||
@@ -559,11 +563,33 @@ describe("scratch files and thumbnail references", () => {
|
||||
scheduler.pending().length === 1,
|
||||
);
|
||||
const secondOut = runner.calls.filter((c) => c.args[0] === "screenshot").at(-1)?.args;
|
||||
expect(pathOf(secondOut ?? [])).toBe(join(tmpdir(), "bsk-obs-s1.png"));
|
||||
await waitFor(() => !existsSync(pathOf(secondOut ?? [])));
|
||||
const secondPath = pathOf(secondOut ?? []);
|
||||
expect(secondPath).toBe(firstPath);
|
||||
await waitFor(() => !existsSync(secondPath));
|
||||
service.dispose();
|
||||
});
|
||||
|
||||
it("isolates scratch paths between service instances", async () => {
|
||||
const first = setup({ scheduler: fakeScheduler(), runner: fakeRunner() });
|
||||
const second = setup({ scheduler: fakeScheduler(), runner: fakeRunner() });
|
||||
first.service.addSession("s1");
|
||||
second.service.addSession("s1");
|
||||
first.scheduler.runNext();
|
||||
second.scheduler.runNext();
|
||||
await waitFor(
|
||||
() =>
|
||||
first.service.getState()[0]?.thumbnailAttachmentId !== undefined &&
|
||||
second.service.getState()[0]?.thumbnailAttachmentId !== undefined,
|
||||
);
|
||||
const pathOf = (runner: FakeRunner) => {
|
||||
const args = runner.calls.find((call) => call.args[0] === "screenshot")?.args ?? [];
|
||||
return args[args.indexOf("--out") + 1];
|
||||
};
|
||||
expect(pathOf(first.runner)).not.toBe(pathOf(second.runner));
|
||||
first.service.dispose();
|
||||
second.service.dispose();
|
||||
});
|
||||
|
||||
it("keeps the last two distinct frames and evicts the third", async () => {
|
||||
let n = 0;
|
||||
const runner = fakeRunner({
|
||||
|
||||
@@ -24,6 +24,10 @@ function scene(nodes: VomNode[]): VomScene {
|
||||
};
|
||||
}
|
||||
|
||||
function coreRefs(refs: ReturnType<typeof renderVom>["refs"]) {
|
||||
return refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }));
|
||||
}
|
||||
|
||||
describe("renderVom single-layer page", () => {
|
||||
it("does not derive handle context across frame scopes", () => {
|
||||
const out = renderVom(
|
||||
@@ -130,7 +134,7 @@ describe("renderVom single-layer page", () => {
|
||||
' @e2 button "加入购物车"',
|
||||
].join("\n"),
|
||||
);
|
||||
expect(out.refs).toEqual([
|
||||
expect(coreRefs(out.refs)).toEqual([
|
||||
{ ref: "e1", backendNodeId: 3 },
|
||||
{ ref: "e2", backendNodeId: 6 },
|
||||
]);
|
||||
@@ -151,7 +155,7 @@ describe("renderVom single-layer page", () => {
|
||||
expect(out.text).toContain(' heading "Title"');
|
||||
expect(out.text).toContain(' img "Logo"');
|
||||
expect(out.text).toContain(' @e1 button "Continue"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
});
|
||||
|
||||
it("assigns refs to listbox controls", () => {
|
||||
@@ -163,7 +167,7 @@ describe("renderVom single-layer page", () => {
|
||||
);
|
||||
|
||||
expect(out.text).toContain(' @e1 listbox "Choices"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("annotates external links regardless of role casing", () => {
|
||||
@@ -200,7 +204,7 @@ describe("renderVom single-layer page", () => {
|
||||
expect(out.text).toContain(" alertdialog");
|
||||
expect(out.text).toContain(" section");
|
||||
expect(out.text).toContain(' @e1 button "Close"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 5 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 5 }]);
|
||||
});
|
||||
|
||||
it("skips generic nodes without adding an extra indentation level", () => {
|
||||
@@ -291,7 +295,7 @@ describe("renderVom single-layer page", () => {
|
||||
expect(out.text).toContain(" section");
|
||||
expect(out.text).not.toContain("Too deep");
|
||||
expect(out.text).toContain(' @e1 button "Later"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
});
|
||||
|
||||
it("truncates when maxTokens is exceeded while keeping refs in sync with rendered text", () => {
|
||||
@@ -326,7 +330,7 @@ describe("renderVom single-layer page", () => {
|
||||
);
|
||||
|
||||
expect(out.text).toContain('@e1 button "Open settings"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("keeps the deepest custom control when wrapper and child both look clickable", () => {
|
||||
@@ -356,7 +360,7 @@ describe("renderVom single-layer page", () => {
|
||||
|
||||
expect(out.text).not.toContain('@e1 button "Card"');
|
||||
expect(out.text).toContain('@e1 button "Details"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
||||
});
|
||||
|
||||
it("does not recover clickable wrappers around native controls", () => {
|
||||
@@ -379,7 +383,7 @@ describe("renderVom single-layer page", () => {
|
||||
|
||||
expect(out.text).not.toContain('@e1 button "Checkout"');
|
||||
expect(out.text).toContain('@e1 button "Pay"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
||||
});
|
||||
|
||||
it("adds context to duplicate weak action labels", () => {
|
||||
@@ -442,7 +446,7 @@ describe("renderVom single-layer page", () => {
|
||||
|
||||
expect(out.text).toContain("@layers 1 focus=L1");
|
||||
expect(out.text).toContain('@e1 button "Background"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("can filter refs blocked by active positioned regions without changing layer output", () => {
|
||||
@@ -485,7 +489,7 @@ describe("renderVom single-layer page", () => {
|
||||
expect(out.text).not.toContain("active-region");
|
||||
expect(out.text).not.toContain("Background");
|
||||
expect(out.text).toContain('@e1 button "Foreground"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
});
|
||||
|
||||
it("keeps paint-order comparisons inside their frame document", () => {
|
||||
@@ -600,7 +604,7 @@ describe("renderVom single-layer page", () => {
|
||||
});
|
||||
|
||||
expect(out.text).toContain('@e1 button "Products" [hover first: Shoes | Bags | Accessories]');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("injects active scope blocks immediately after their trigger without refs", () => {
|
||||
@@ -626,7 +630,7 @@ describe("renderVom single-layer page", () => {
|
||||
" Bob - great sound",
|
||||
].join("\n"),
|
||||
);
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("does not render redundant children inside named ref controls", () => {
|
||||
@@ -640,7 +644,7 @@ describe("renderVom single-layer page", () => {
|
||||
|
||||
expect(out.text).toContain('@e1 button "Save"');
|
||||
expect(out.text).not.toContain("disk icon");
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("does not render redundant children inside native text inputs", () => {
|
||||
@@ -807,7 +811,7 @@ describe("renderVom double-layer page", () => {
|
||||
].join("\n"),
|
||||
);
|
||||
expect(out.text).not.toContain("底层按钮");
|
||||
expect(out.refs).toEqual([
|
||||
expect(coreRefs(out.refs)).toEqual([
|
||||
{ ref: "e1", backendNodeId: 4 },
|
||||
{ ref: "e2", backendNodeId: 5 },
|
||||
]);
|
||||
@@ -859,4 +863,36 @@ describe("renderVom double-layer page", () => {
|
||||
expect(out.text).toContain(' @e2 textbox "密码" [filled] ="•••"');
|
||||
expect(out.refs.map((r) => r.backendNodeId)).toEqual([101, 102]);
|
||||
});
|
||||
|
||||
it("redacts form values and returns semantic metadata for rendered refs", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Form" }),
|
||||
node({
|
||||
id: 2,
|
||||
backendNodeId: 42,
|
||||
frameId: "child-frame",
|
||||
parentId: 1,
|
||||
role: "textbox",
|
||||
name: "Email",
|
||||
value: "user@example.com",
|
||||
inputState: "filled",
|
||||
tag: "input",
|
||||
}),
|
||||
]),
|
||||
{ redactValues: true },
|
||||
);
|
||||
|
||||
expect(out.text).toContain('[filled] ="•••"');
|
||||
expect(out.text).not.toContain("user@example.com");
|
||||
expect(out.refs[0]).toMatchObject({
|
||||
ref: "e1",
|
||||
backendNodeId: 42,
|
||||
frameId: "child-frame",
|
||||
role: "textbox",
|
||||
name: "Email",
|
||||
line: expect.any(Number),
|
||||
});
|
||||
expect(out.text.split("\n")[out.refs[0]?.line ?? -1]).toContain('@e1 textbox "Email"');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5,6 +5,7 @@ export type {
|
||||
CondSurface,
|
||||
LayerKind,
|
||||
Rect,
|
||||
RenderedRef,
|
||||
Viewport,
|
||||
VomNode,
|
||||
VomOptions,
|
||||
|
||||
@@ -4,9 +4,9 @@ import type {
|
||||
BlockingLayer,
|
||||
CondSurface,
|
||||
Rect,
|
||||
RenderedRef,
|
||||
VomNode,
|
||||
VomOptions,
|
||||
VomRef,
|
||||
VomResult,
|
||||
VomScene,
|
||||
} from "./types";
|
||||
@@ -615,6 +615,7 @@ function renderNodeLine(
|
||||
ref: string | undefined,
|
||||
context: string[] = [],
|
||||
surface: CondSurface | undefined = undefined,
|
||||
redactValues = false,
|
||||
): string {
|
||||
let line = `${" ".repeat(depth)}${ref ? `@${ref} ` : ""}${node.role}`;
|
||||
|
||||
@@ -636,13 +637,14 @@ function renderNodeLine(
|
||||
}
|
||||
const placeholder = cleaned(node.placeholder);
|
||||
if (placeholder) line += ` placeholder=${JSON.stringify(placeholder)}`;
|
||||
const sensitiveHasValue =
|
||||
const hasValue =
|
||||
rawValue !== undefined || node.inputState === "filled" || node.inputState === "default";
|
||||
const value = node.sensitive
|
||||
? sensitiveHasValue
|
||||
? SENSITIVE_MASK
|
||||
: undefined
|
||||
: rawValue?.slice(0, MAX_VALUE_LENGTH);
|
||||
const value =
|
||||
node.sensitive || redactValues
|
||||
? hasValue
|
||||
? SENSITIVE_MASK
|
||||
: undefined
|
||||
: rawValue?.slice(0, MAX_VALUE_LENGTH);
|
||||
if (value !== undefined) line += ` =${JSON.stringify(value)}`;
|
||||
|
||||
if (surface && surface.subItems.length > 0) {
|
||||
@@ -695,11 +697,12 @@ function shouldSkipRedundantChildren(node: VomNode, state: RenderState): boolean
|
||||
|
||||
interface RenderState {
|
||||
lines: string[];
|
||||
refs: VomRef[];
|
||||
refs: RenderedRef[];
|
||||
nextRef: number;
|
||||
tokens: number;
|
||||
maxDepth: number;
|
||||
maxTokens: number;
|
||||
redactValues: boolean;
|
||||
truncated: boolean;
|
||||
stopped: boolean;
|
||||
children: Map<number | null, VomNode[]>;
|
||||
@@ -764,7 +767,7 @@ function renderTree(
|
||||
const ref = isVomReferenceNode(node) ? `e${state.nextRef}` : undefined;
|
||||
const context = ref ? handleContext(node, state) : [];
|
||||
const surface = state.surfaceMap.get(node.id);
|
||||
const line = renderNodeLine(node, depth, ref, context, surface);
|
||||
const line = renderNodeLine(node, depth, ref, context, surface, state.redactValues);
|
||||
const nextTokens = state.tokens + estimateTokens(line);
|
||||
if (nextTokens > state.maxTokens) {
|
||||
state.truncated = true;
|
||||
@@ -775,10 +778,15 @@ function renderTree(
|
||||
state.lines.push(line);
|
||||
state.tokens = nextTokens;
|
||||
if (ref) {
|
||||
const name = cleaned(node.name);
|
||||
state.refs.push({
|
||||
ref,
|
||||
backendNodeId: node.backendNodeId ?? node.id,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
...(node.role ? { role: node.role } : {}),
|
||||
...(name ? { name } : {}),
|
||||
...(context.length > 0 ? { ctx: context.join(" > ") } : {}),
|
||||
line: state.lines.length - 1,
|
||||
});
|
||||
state.nextRef += 1;
|
||||
}
|
||||
@@ -808,6 +816,7 @@ function renderNodes(
|
||||
tokens: estimateTokens(initialLines.join("\n")),
|
||||
maxDepth: options.maxDepth ?? DEFAULT_MAX_DEPTH,
|
||||
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
|
||||
redactValues: options.redactValues ?? false,
|
||||
truncated: false,
|
||||
stopped: false,
|
||||
children,
|
||||
|
||||
@@ -64,6 +64,10 @@ export interface VomScene {
|
||||
export interface VomOptions {
|
||||
maxDepth?: number;
|
||||
maxTokens?: number;
|
||||
/**
|
||||
* When true, form values are rendered as masks instead of literal values.
|
||||
*/
|
||||
redactValues?: boolean;
|
||||
/**
|
||||
* Experimental: filter refs that are geometrically blocked by foreground
|
||||
* fixed/absolute/sticky regions. Disabled by default and does not alter the
|
||||
@@ -90,9 +94,17 @@ export interface VomRef {
|
||||
frameId?: string;
|
||||
}
|
||||
|
||||
export interface RenderedRef extends VomRef {
|
||||
role?: string;
|
||||
name?: string;
|
||||
ctx?: string;
|
||||
/** Zero-based line index in `VomResult.text`. */
|
||||
line: number;
|
||||
}
|
||||
|
||||
export interface VomResult {
|
||||
text: string;
|
||||
refs: VomRef[];
|
||||
refs: RenderedRef[];
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user