Compare commits

..
Author SHA1 Message Date
drakezhang a4b6a61d29 docs(dsh-plugin): clarify package description 2026-08-22 21:29:33 +08:00
drakezhang cad0b5cbc9 feat(dsh-plugin): add Browser tool parity 2026-08-22 21:15:53 +08:00
109 changed files with 4112 additions and 14645 deletions
@@ -1,12 +1,10 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { RecordStepPayload } from "@/lib/record-bridge";
import { startRecordCapture } from "../record-capture";
import { RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
vi.stubGlobal("chrome", {
runtime: {
sendMessage: vi.fn((message: { sequence?: number }) =>
Promise.resolve({ ok: true, sequence: message.sequence }),
),
sendMessage: vi.fn(() => Promise.resolve()),
},
});
@@ -48,6 +46,58 @@ function click(el: Element): void {
el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
}
describe("handleRecordContentMessage stop/cancel", () => {
it("ignores STOP when no recording is active", () => {
const dispose = vi.fn();
const onStop = vi.fn();
const setActiveRequestId = vi.fn();
const setCapture = vi.fn();
const sendResponse = vi.fn();
const needsAsync = handleRecordContentMessage(
{ type: RECORD_STOP, requestId: "rec-stale" },
{
activeRequestId: null,
capture: { dispose },
setActiveRequestId,
setCapture,
onStart: vi.fn(),
onStop,
},
sendResponse,
);
expect(needsAsync).toBe(false);
expect(dispose).not.toHaveBeenCalled();
expect(onStop).not.toHaveBeenCalled();
expect(setActiveRequestId).not.toHaveBeenCalled();
expect(setCapture).not.toHaveBeenCalled();
expect(sendResponse).not.toHaveBeenCalled();
});
it("ignores STOP for a mismatched requestId", () => {
const dispose = vi.fn();
const onStop = vi.fn();
const needsAsync = handleRecordContentMessage(
{ type: RECORD_STOP, requestId: "rec-other" },
{
activeRequestId: "rec-1",
capture: { dispose },
setActiveRequestId: vi.fn(),
setCapture: vi.fn(),
onStart: vi.fn(),
onStop,
},
vi.fn(),
);
expect(needsAsync).toBe(false);
expect(dispose).not.toHaveBeenCalled();
expect(onStop).not.toHaveBeenCalled();
});
});
describe("record-capture semantic", () => {
let steps: RecordStepPayload[];
@@ -124,19 +174,6 @@ describe("record-capture semantic", () => {
]);
});
it("does not emit page navigation from a child Document capture", () => {
const originalUrl = location.href;
const capture = startRecordCapture("rec-child", (step) => steps.push(step), {
captureNavigation: false,
});
history.pushState({}, "", "#inside-frame");
expect(steps).toEqual([]);
capture.dispose();
history.replaceState({}, "", originalUrl);
});
it("does not record clicks on anonymous layout divs", () => {
document.body.innerHTML = `
<div id="chrome">page chrome</div>
@@ -191,7 +228,6 @@ describe("record-capture semantic", () => {
expect(steps[0]).toMatchObject({
op: "hover",
target: { role: "button", name: "Open user navigation menu" },
geometry: { tag: "button", rect: { x: 900, y: 8, w: 32, h: 32 } },
});
expect(steps[1]).toMatchObject({
op: "click",
@@ -1,102 +0,0 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { RecordFramePortMessage } from "@/lib/recording/frame-bridge";
import { RECORD_FRAME_START } from "@/lib/recording/frame-bridge";
import { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity";
import { RecordFrameAgent } from "../recording/frame-agent";
class PortListeners<T extends (...args: never[]) => unknown> {
readonly values = new Set<T>();
addListener = (listener: T) => this.values.add(listener);
removeListener = (listener: T) => this.values.delete(listener);
}
function portHarness() {
const onMessage = new PortListeners<(message: unknown) => void>();
const onDisconnect = new PortListeners<() => void>();
const outbound: RecordFramePortMessage[] = [];
const port = {
name: "bsk-record-frame",
onMessage,
onDisconnect,
postMessage(message: RecordFramePortMessage) {
outbound.push(message);
if (message.type === "ready") {
queueMicrotask(() => {
for (const listener of onMessage.values) {
listener({
type: "ready_ack",
requestId: message.requestId,
producerId: message.producerId,
});
}
});
}
},
disconnect: vi.fn(),
} as unknown as chrome.runtime.Port;
return {
port,
outbound,
receive(message: RecordFramePortMessage) {
for (const listener of onMessage.values) listener(message);
},
};
}
describe("RecordFrameAgent", () => {
afterEach(() => {
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
document.body.replaceChildren();
});
it("keeps a failed stop retryable and flushes the final dirty fill", async () => {
const harness = portHarness();
const sendMessage = vi
.fn<(message: { sequence: number }) => Promise<unknown>>()
.mockRejectedValueOnce(new Error("offline"))
.mockRejectedValueOnce(new Error("still offline"))
.mockImplementation(async (message) => ({ ok: true, sequence: message.sequence }));
vi.stubGlobal("chrome", {
runtime: {
connect: vi.fn(() => harness.port),
sendMessage,
},
});
document.body.innerHTML = `<label for="draft">Draft</label><input id="draft" />`;
const agent = new RecordFrameAgent();
await expect(
agent.start({ type: RECORD_FRAME_START, requestId: "rec-1", startedAtMs: 10 }),
).resolves.toEqual({ ok: true });
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true);
const input = document.querySelector("input")!;
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
input.value = "final value";
input.dispatchEvent(new Event("input", { bubbles: true }));
harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-1" });
await vi.waitFor(() =>
expect(harness.outbound).toContainEqual({
type: "stopped",
requestId: "rec-1",
commandId: "stop-1",
ok: false,
error: "failed to deliver one or more recorded steps",
}),
);
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true);
harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-2" });
await vi.waitFor(() =>
expect(harness.outbound).toContainEqual({
type: "stopped",
requestId: "rec-1",
commandId: "stop-2",
ok: true,
}),
);
expect(sendMessage).toHaveBeenCalledTimes(3);
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(false);
});
});
@@ -1,36 +0,0 @@
import { describe, expect, it, vi } from "vitest";
import type { RecordStepMessage } from "@/lib/record-bridge";
import { RecordStepDelivery } from "../record-step-delivery";
describe("RecordStepDelivery", () => {
it("retries from the first unacknowledged sequence before sending later steps", async () => {
const sent: RecordStepMessage[] = [];
const send = vi.fn(async (message: RecordStepMessage) => {
sent.push(message);
if (sent.length === 1) throw new Error("service worker unavailable");
return { ok: true, sequence: message.sequence };
});
const delivery = new RecordStepDelivery("rec-ordered", send, "document-1");
delivery.enqueue({ op: "click", target: { tag: "button", name: "First" } });
delivery.enqueue({ op: "click", target: { tag: "button", name: "Second" } });
await expect(delivery.flush()).resolves.toBe(true);
expect(sent.map((message) => message.sequence)).toEqual([1, 1, 2]);
expect(sent.map((message) => message.step.target?.name)).toEqual(["First", "First", "Second"]);
});
it("keeps unacknowledged steps pending for a later flush", async () => {
const send = vi
.fn<(message: RecordStepMessage) => Promise<unknown>>()
.mockRejectedValueOnce(new Error("offline"))
.mockRejectedValueOnce(new Error("still offline"))
.mockImplementation(async (message) => ({ ok: true, sequence: message.sequence }));
const delivery = new RecordStepDelivery("rec-retry", send, "document-1");
delivery.enqueue({ op: "click", target: { tag: "button", name: "Save" } });
await expect(delivery.flush()).resolves.toBe(false);
await expect(delivery.flush()).resolves.toBe(true);
expect(send).toHaveBeenCalledTimes(3);
});
});
+150 -76
View File
@@ -1,9 +1,9 @@
import {
type CaptureTargetDescriptor,
describeEventTarget,
describeTarget,
resolveClickableElement,
resolveHoverElement,
type TargetDescriptor,
} from "@/lib/describe-target";
import {
evaluateHoverTrigger,
@@ -12,8 +12,22 @@ import {
hasDirectHoverInteractiveSignal,
hasStrongHoverExpansionSignal,
} from "@/lib/hover-trigger-policy";
import type { RecordStepPayload } from "@/lib/record-bridge";
import { shouldRecordPress } from "@/lib/recording/draft-policy";
import {
isRecordCancelMessage,
isRecordStartMessage,
isRecordStopMessage,
RECORD_CANCEL,
RECORD_START,
RECORD_STEP,
RECORD_STOP,
type RecordCancelMessage,
type RecordStartAck,
type RecordStartMessage,
type RecordStepPayload,
type RecordStopAck,
type RecordStopMessage,
} from "@/lib/record-bridge";
import { shouldRecordPress } from "@/lib/trace-reducer";
import {
closestHoverSurfaceCandidate,
collectHoverSurfaceStates,
@@ -23,21 +37,42 @@ import {
isLikelyHoverSurfaceOwner,
} from "./record-hover-surface";
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
const failedStepDeliveries = new Set<string>();
const knownRecordRequests = new Set<string>();
function sendRecordStep(requestId: string, step: RecordStepPayload): void {
const payload = { type: RECORD_STEP, requestId, step };
const pending = Promise.resolve(chrome.runtime.sendMessage(payload)).then(
() => true,
() => {
failedStepDeliveries.add(requestId);
return false;
},
);
const sends = pendingStepSends.get(requestId) ?? new Set<Promise<boolean>>();
sends.add(pending);
pendingStepSends.set(requestId, sends);
void pending.then(() => {
sends.delete(pending);
if (sends.size === 0) pendingStepSends.delete(requestId);
});
}
export interface RecordCaptureController {
dispose(): void;
}
interface FillSession {
element: FillableElement;
target: CaptureTargetDescriptor;
target: TargetDescriptor;
baselineValue: string;
lastValue: string;
pendingCommit?: "enter" | "suggestion" | "blur";
}
interface HoverCandidate {
element: Element;
target: CaptureTargetDescriptor;
target: TargetDescriptor;
recordedAt: number;
score: number;
eligible: boolean;
@@ -122,24 +157,6 @@ function nearbyFillableFromSearchChrome(target: Element): FillableElement | null
return null;
}
function captureGeometry(el: Element): RecordStepPayload["geometry"] {
const rect = el.getBoundingClientRect();
return {
rect: { x: rect.x, y: rect.y, w: rect.width, h: rect.height },
tag: el.tagName.toLowerCase(),
};
}
function geometryForEventTarget(
target: EventTarget | null,
): RecordStepPayload["geometry"] | undefined {
if (!(target instanceof Element)) return undefined;
const clickable = target.closest(
'a,button,input,select,textarea,[role="button"],[role="link"],[role="menuitem"],[contenteditable="true"]',
);
return captureGeometry(clickable ?? target);
}
function fillableValue(el: FillableElement): string {
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
return el.value;
@@ -215,10 +232,7 @@ function collectHoverTriggerLabelText(root: Element): string {
return normalizeLabelText(text);
}
function compactHoverTargetName(
el: Element,
desc: CaptureTargetDescriptor,
): CaptureTargetDescriptor {
function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor {
if (!desc.name) return desc;
const fullText = normalizeLabelText(el.textContent ?? "");
const compactName = collectHoverTriggerLabelText(el);
@@ -236,7 +250,7 @@ function compactHoverTargetName(
return desc;
}
function isWeakHoverTarget(target: CaptureTargetDescriptor): boolean {
function isWeakHoverTarget(target: TargetDescriptor): boolean {
return !target.role && !target.name && target.tag === "div";
}
@@ -247,9 +261,9 @@ function looksLikeAvatarElement(el: Element): boolean {
function normalizeHoverTarget(
el: Element,
desc: CaptureTargetDescriptor,
desc: TargetDescriptor,
decision: HoverTriggerDecision,
): CaptureTargetDescriptor {
): TargetDescriptor {
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
return { ...desc, name: "image" };
}
@@ -263,7 +277,7 @@ function normalizeHoverTarget(
return desc;
}
function hoverTriggerSignals(el: Element, desc: CaptureTargetDescriptor) {
function hoverTriggerSignals(el: Element, desc: TargetDescriptor) {
if (!(el instanceof HTMLElement)) return null;
const style = hoverTriggerStyle(el);
return {
@@ -359,7 +373,6 @@ function scheduleInputCompletionCommit(
export function startRecordCapture(
_requestId: string,
sendStep: (step: RecordStepPayload) => void,
options: { captureNavigation?: boolean } = {},
): RecordCaptureController {
const emitStep = (step: RecordStepPayload) => {
sendStep({ page_url: location.href, ...step });
@@ -398,17 +411,14 @@ export function startRecordCapture(
emitStep({
op: "fill",
target: session.target,
geometry: captureGeometry(session.element),
value,
commit: session.pendingCommit ?? "blur",
...(isPassword ? { redacted: true } : {}),
});
};
const commitFillSession = (commit: "enter" | "suggestion" | "blur" = "blur") => {
const commitFillSession = () => {
if (!fillSession || composing) return;
const session = fillSession;
session.pendingCommit = commit;
fillSession = null;
emitFill(session);
committedValues.set(session.element, session.lastValue);
@@ -614,7 +624,6 @@ export function startRecordCapture(
emitStep({
op: "hover",
target: hover.target,
geometry: captureGeometry(hover.element),
});
emittedHoverElements.add(hover.element);
};
@@ -669,7 +678,6 @@ export function startRecordCapture(
emitStep({
op: "click",
target,
geometry: geometryForEventTarget(eventTarget(event)),
expects_navigation: true,
});
};
@@ -771,7 +779,7 @@ export function startRecordCapture(
scheduleInputCompletionCommit(
sessionElement,
syncFillSessionValue,
() => commitFillSession("suggestion"),
commitFillSession,
(el) => fillSession?.element === el,
);
return;
@@ -831,7 +839,6 @@ export function startRecordCapture(
emitStep({
op: "select",
target: desc,
geometry: captureGeometry(target),
values,
labels,
expects_navigation: true,
@@ -872,7 +879,7 @@ export function startRecordCapture(
};
}
if (fillable) {
commitFillSession(event.key === "Enter" ? "enter" : "blur");
commitFillSession();
}
const desc = describeEventTarget(target);
if (!desc && !event.key) return;
@@ -880,7 +887,7 @@ export function startRecordCapture(
emitStep({
op: "press",
key: event.key,
...(desc ? { target: desc, geometry: geometryForEventTarget(target) } : {}),
...(desc ? { target: desc } : {}),
...(modifiers.length ? { modifiers } : {}),
expects_navigation: event.key === "Enter",
});
@@ -896,33 +903,23 @@ export function startRecordCapture(
document.addEventListener("change", onChange, true);
document.addEventListener("keydown", onKeyDown, true);
const captureNavigation = options.captureNavigation ?? true;
const urlObserver = captureNavigation
? new MutationObserver(() => emitNavigateIfChanged())
: null;
urlObserver?.observe(document, { subtree: true, childList: true });
const urlObserver = new MutationObserver(() => emitNavigateIfChanged());
urlObserver.observe(document, { subtree: true, childList: true });
const onUrlEvent = () => emitNavigateIfChanged();
// Wrap: passing commitFillSession directly would forward the DOM event as
// the `commit` argument and stamp it onto the recorded fill step.
const onPageHide = () => commitFillSession();
if (captureNavigation) {
window.addEventListener("hashchange", onUrlEvent);
window.addEventListener("popstate", onUrlEvent);
}
window.addEventListener("pagehide", onPageHide);
window.addEventListener("hashchange", onUrlEvent);
window.addEventListener("popstate", onUrlEvent);
window.addEventListener("pagehide", commitFillSession);
const originalPushState = history.pushState;
const originalReplaceState = history.replaceState;
if (captureNavigation) {
history.pushState = function (...args: Parameters<History["pushState"]>) {
originalPushState.apply(this, args);
emitNavigateIfChanged(navigationActionPending ? true : undefined);
};
history.replaceState = function (...args: Parameters<History["replaceState"]>) {
originalReplaceState.apply(this, args);
emitNavigateIfChanged(navigationActionPending ? true : undefined);
};
}
history.pushState = function (...args: Parameters<History["pushState"]>) {
originalPushState.apply(this, args);
emitNavigateIfChanged(navigationActionPending ? true : undefined);
};
history.replaceState = function (...args: Parameters<History["replaceState"]>) {
originalReplaceState.apply(this, args);
emitNavigateIfChanged(navigationActionPending ? true : undefined);
};
return {
dispose() {
@@ -936,16 +933,93 @@ export function startRecordCapture(
document.removeEventListener("compositionend", onCompositionEnd, true);
document.removeEventListener("change", onChange, true);
document.removeEventListener("keydown", onKeyDown, true);
urlObserver?.disconnect();
if (captureNavigation) {
window.removeEventListener("hashchange", onUrlEvent);
window.removeEventListener("popstate", onUrlEvent);
}
window.removeEventListener("pagehide", onPageHide);
if (captureNavigation) {
history.pushState = originalPushState;
history.replaceState = originalReplaceState;
}
urlObserver.disconnect();
window.removeEventListener("hashchange", onUrlEvent);
window.removeEventListener("popstate", onUrlEvent);
window.removeEventListener("pagehide", commitFillSession);
history.pushState = originalPushState;
history.replaceState = originalReplaceState;
},
};
}
export type RecordContentMessage = RecordStartMessage | RecordStopMessage | RecordCancelMessage;
export function isRecordContentMessage(msg: unknown): msg is RecordContentMessage {
return isRecordStartMessage(msg) || isRecordStopMessage(msg) || isRecordCancelMessage(msg);
}
export function handleRecordContentMessage(
message: RecordContentMessage,
state: {
activeRequestId: string | null;
capture: RecordCaptureController | null;
setActiveRequestId(id: string | null): void;
setCapture(capture: RecordCaptureController | null): void;
onStart(requestId: string, startedAtMs?: number): void;
onStop(): void;
},
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
): boolean {
if (isRecordStartMessage(message)) {
if (!knownRecordRequests.has(message.requestId)) {
knownRecordRequests.add(message.requestId);
failedStepDeliveries.delete(message.requestId);
}
state.capture?.dispose();
state.setCapture(
startRecordCapture(message.requestId, (step) => {
sendRecordStep(message.requestId, step);
}),
);
state.setActiveRequestId(message.requestId);
state.onStart(message.requestId, message.startedAtMs);
sendResponse?.({ ok: true });
return sendResponse !== undefined;
}
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
// Require an active recording that matches this requestId — otherwise a
// stray STOP/CANCEL (e.g. after teardown) would still run finishStop and
// clear overlay state even though nothing was capturing.
if (!state.activeRequestId || state.activeRequestId !== message.requestId) {
return false;
}
state.capture?.dispose();
state.setCapture(null);
const finishStop = () => {
state.onStop();
state.setActiveRequestId(null);
};
if (isRecordStopMessage(message) && sendResponse) {
const pending = [...(pendingStepSends.get(message.requestId) ?? [])];
void Promise.all(pending).then((delivered) => {
finishStop();
const succeeded = delivered.every(Boolean) && !failedStepDeliveries.has(message.requestId);
if (succeeded) {
failedStepDeliveries.delete(message.requestId);
knownRecordRequests.delete(message.requestId);
}
sendResponse(
succeeded
? { ok: true }
: {
ok: false,
error: "failed to deliver one or more recorded steps",
},
);
});
return true;
}
if (isRecordCancelMessage(message)) {
failedStepDeliveries.delete(message.requestId);
knownRecordRequests.delete(message.requestId);
}
finishStop();
return false;
}
return false;
}
export { RECORD_CANCEL, RECORD_START, RECORD_STEP, RECORD_STOP };
@@ -1,65 +0,0 @@
import {
isAcceptedRecordStepAck,
RECORD_STEP,
type RecordStepMessage,
type RecordStepPayload,
} from "@/lib/record-bridge";
export type SendRecordStepMessage = (message: RecordStepMessage) => Promise<unknown>;
export class RecordStepDelivery {
readonly #requestId: string;
readonly #producerId: string;
readonly #send: SendRecordStepMessage;
readonly #pending = new Map<number, RecordStepPayload>();
#nextSequence = 1;
#tail = Promise.resolve();
constructor(
requestId: string,
send: SendRecordStepMessage = (message) => chrome.runtime.sendMessage(message),
producerId: string = crypto.randomUUID(),
) {
this.#requestId = requestId;
this.#producerId = producerId;
this.#send = send;
}
enqueue(step: RecordStepPayload): void {
const sequence = this.#nextSequence;
this.#nextSequence += 1;
this.#pending.set(sequence, step);
void this.#schedule();
}
async flush(): Promise<boolean> {
await this.#tail;
await this.#schedule();
return this.#pending.size === 0;
}
#schedule(): Promise<void> {
const drain = () => this.#drain();
this.#tail = this.#tail.then(drain, drain);
return this.#tail;
}
async #drain(): Promise<void> {
for (const [sequence, step] of this.#pending) {
const message: RecordStepMessage = {
type: RECORD_STEP,
requestId: this.#requestId,
producerId: this.#producerId,
sequence,
step,
};
try {
const response = await this.#send(message);
if (!isAcceptedRecordStepAck(response, sequence)) return;
this.#pending.delete(sequence);
} catch {
return;
}
}
}
}
@@ -1,209 +0,0 @@
import {
markRecordingDocument,
type RecordingDocumentMarker,
waitForDocumentElement,
} from "@/lib/recording/document-marker";
import {
isRecordFrameStartMessage,
RECORD_FRAME_PORT,
RECORD_FRAME_QUERY,
RECORD_FRAME_START,
type RecordFramePortMessage,
type RecordFrameQueryResponse,
type RecordFrameStartMessage,
} from "@/lib/recording/frame-bridge";
import { type RecordCaptureController, startRecordCapture } from "../record-capture";
import { RecordStepDelivery } from "../record-step-delivery";
interface ActiveFrameRecording {
requestId: string;
producerId: string;
port: chrome.runtime.Port;
delivery: RecordStepDelivery;
capture: RecordCaptureController;
marker: RecordingDocumentMarker;
}
const RECORD_FRAME_REGISTER_TIMEOUT_MS = 5_000;
export class RecordFrameAgent {
#active: ActiveFrameRecording | null = null;
#startPromise: Promise<{ ok: true } | { ok: false; error: string }> | null = null;
#disposed = false;
async start(
message: RecordFrameStartMessage,
): Promise<{ ok: true } | { ok: false; error: string }> {
if (this.#active?.requestId === message.requestId) return { ok: true };
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
if (this.#startPromise) {
await this.#startPromise;
return this.start(message);
}
if (this.#active) this.cancel(this.#active.requestId);
const starting = this.#start(message);
this.#startPromise = starting;
try {
return await starting;
} finally {
if (this.#startPromise === starting) this.#startPromise = null;
}
}
cancel(requestId: string): void {
const active = this.#active;
if (!active || active.requestId !== requestId) return;
active.capture.dispose();
active.marker.restore();
active.port.disconnect();
this.#active = null;
}
dispose(): void {
this.#disposed = true;
if (this.#active) this.cancel(this.#active.requestId);
}
async #start(
message: RecordFrameStartMessage,
): Promise<{ ok: true } | { ok: false; error: string }> {
const producerId = crypto.randomUUID();
const root = await waitForDocumentElement();
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
const marker = markRecordingDocument(producerId, root);
const port = chrome.runtime.connect({ name: RECORD_FRAME_PORT });
try {
await registerPort(port, message.requestId, producerId);
} catch {
marker.restore();
port.disconnect();
return { ok: false, error: "failed to register recording document" };
}
if (this.#disposed) {
marker.restore();
port.disconnect();
return { ok: false, error: "recording document is disposed" };
}
const delivery = new RecordStepDelivery(message.requestId, undefined, producerId);
const capture = startRecordCapture(
message.requestId,
(step) => {
marker.ensure();
delivery.enqueue(step);
},
{ captureNavigation: window.top === window },
);
const active: ActiveFrameRecording = {
requestId: message.requestId,
producerId,
port,
delivery,
capture,
marker,
};
this.#active = active;
port.onMessage.addListener((raw: unknown) => {
const command = raw as Partial<RecordFramePortMessage>;
if (command.type === "stop" && command.requestId === active.requestId) {
void this.#stop(active, command.commandId);
} else if (command.type === "cancel" && command.requestId === active.requestId) {
this.cancel(active.requestId);
}
});
port.onDisconnect.addListener(() => {
if (this.#active !== active) return;
active.capture.dispose();
active.marker.restore();
this.#active = null;
});
return { ok: true };
}
async #stop(active: ActiveFrameRecording, commandId: string | undefined): Promise<void> {
if (!commandId || this.#active !== active) return;
active.capture.dispose();
const succeeded = await active.delivery.flush();
try {
active.port.postMessage({
type: "stopped",
requestId: active.requestId,
commandId,
ok: succeeded,
...(succeeded ? {} : { error: "failed to deliver one or more recorded steps" }),
} satisfies RecordFramePortMessage);
} catch {
return;
}
if (!succeeded || this.#active !== active) return;
active.marker.restore();
this.#active = null;
active.port.disconnect();
}
}
function registerPort(
port: chrome.runtime.Port,
requestId: string,
producerId: string,
): Promise<void> {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => {
cleanup();
reject(new Error("recording document registration timed out"));
}, RECORD_FRAME_REGISTER_TIMEOUT_MS);
const cleanup = () => {
clearTimeout(timer);
port.onMessage.removeListener(onMessage);
port.onDisconnect.removeListener(onDisconnect);
};
const onMessage = (raw: unknown) => {
const message = raw as Partial<RecordFramePortMessage>;
if (
message.type !== "ready_ack" ||
message.requestId !== requestId ||
message.producerId !== producerId
) {
return;
}
cleanup();
resolve();
};
const onDisconnect = () => {
cleanup();
reject(new Error("recording document port disconnected"));
};
port.onMessage.addListener(onMessage);
port.onDisconnect.addListener(onDisconnect);
port.postMessage({ type: "ready", requestId, producerId } satisfies RecordFramePortMessage);
});
}
export function attachRecordFrameAgent(): () => void {
const agent = new RecordFrameAgent();
const onMessage = (
message: unknown,
_sender: chrome.runtime.MessageSender,
sendResponse: (response: { ok: true } | { ok: false; error: string }) => void,
) => {
if (!isRecordFrameStartMessage(message)) return false;
void agent.start(message).then(sendResponse);
return true;
};
chrome.runtime.onMessage.addListener(onMessage);
void chrome.runtime
.sendMessage({ type: RECORD_FRAME_QUERY })
.then((response: RecordFrameQueryResponse | undefined) => {
if (!response?.active || !response.requestId || response.startedAtMs === undefined) return;
return agent.start({
type: RECORD_FRAME_START,
requestId: response.requestId,
startedAtMs: response.startedAtMs,
});
})
.catch(() => {});
return () => {
chrome.runtime.onMessage.removeListener(onMessage);
agent.dispose();
};
}
+21 -25
View File
@@ -21,7 +21,6 @@ import {
type OverlayMode,
} from "@/lib/overlay-bridge";
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
import { recordFrameCoordinator } from "@/lib/recording/frame-coordinator";
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
import { attachSessionEventHandler } from "@/session-manager/event-handler";
@@ -39,7 +38,6 @@ import {
attachRecordFinishListener,
attachRecordQueryListener,
attachRecordStepListener,
type RecordRuntimeDeps,
} from "@/tools/record";
import { detectBrowserMeta } from "@/transport/handshake";
import type { Transport } from "@/transport/transport";
@@ -147,33 +145,10 @@ export default defineBackground(() => {
void sessionsLive.syncFromManager();
},
});
const recordDeps = {
tabsApi: chrome.tabs,
cdp,
frameCoordinator: recordFrameCoordinator,
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
chrome.tabs.sendMessage(tabId, msg),
bypassOverlay: async (tabId: number, enabled: boolean) => {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
} satisfies RecordRuntimeDeps;
recordFrameCoordinator.attach();
attachRecordStepListener(recordDeps);
attachRecordFinishListener(recordDeps);
attachRecordQueryListener(recordDeps);
const dispatcher = new ToolDispatcher({
transport,
sessions,
cdp,
recording: recordDeps,
onSessionsChanged: onOverlaySessionStateChanged,
onBrowserControlResumed,
approveBorrow: (ctx) =>
@@ -195,6 +170,27 @@ export default defineBackground(() => {
}),
});
dispatcher.start();
const recordDeps = {
tabsApi: chrome.tabs,
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
chrome.tabs.sendMessage(tabId, msg),
bypassOverlay: async (tabId: number, enabled: boolean) => {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
};
// Message listeners stay up (cheap; fire only for record message types).
// Tab / webNavigation observation attaches lazily while a recording is
// active — see ensureBrowserObservationListeners in tools/record.ts.
attachRecordStepListener();
attachRecordFinishListener(recordDeps);
attachRecordQueryListener(recordDeps);
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
attachBorrowNotificationClickHandler({
onClicked: chrome.notifications.onClicked,
+45 -34
View File
@@ -11,6 +11,11 @@ import { createHelpRequestData } from "@/content/help-request";
import overlayCss from "@/content/overlay.css?inline";
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
import { RecordOverlay } from "@/content/RecordOverlay";
import {
handleRecordContentMessage,
isRecordContentMessage,
type RecordCaptureController,
} from "@/content/record-capture";
import {
type CaptureSuppressAck,
type CaptureSuppressMessage,
@@ -40,17 +45,11 @@ import {
} from "@/lib/overlay-bridge";
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
import {
isRecordCancelMessage,
isRecordStartMessage,
isRecordStopMessage,
RECORD_FINISH,
RECORD_QUERY,
type RecordCancelMessage,
type RecordQueryResponse,
type RecordStartAck,
type RecordStartMessage,
type RecordStopAck,
type RecordStopMessage,
} from "@/lib/record-bridge";
import type {
BorrowCancelMessage,
@@ -70,6 +69,7 @@ export default defineContentScript({
if (window.top !== window) return;
const overlays = new OverlayController();
let recordCapture: RecordCaptureController | null = null;
let activeRecordRequestId: string | null = null;
let reactRoot: ReactDOM.Root | null = null;
let overlayHost: HTMLElement | null = null;
@@ -228,6 +228,8 @@ export default defineContentScript({
if (previousHelp) {
void sendHelpFinish(previousHelp.id, "cancelled");
}
recordCapture?.dispose();
recordCapture = null;
activeRecordRequestId = null;
renderAll();
}
@@ -257,9 +259,6 @@ export default defineContentScript({
| HelpRequestMessage
| HelpCancelMessage
| CaptureSuppressMessage
| RecordStartMessage
| RecordStopMessage
| RecordCancelMessage
| OverlayAgentOverlayResetMessage
| OverlayAgentStateMessage
| OverlayAutomationBypassMessage,
@@ -270,32 +269,41 @@ export default defineContentScript({
return captureSuppress.handleMessage(message, sendResponse);
}
if (isRecordStartMessage(message)) {
activeRecordRequestId = message.requestId;
overlays.setAgentRecordRequest({
id: message.requestId,
...(typeof message.startedAtMs === "number" ? { startedAtMs: message.startedAtMs } : {}),
onFinish: () => {
void chrome.runtime.sendMessage({
type: RECORD_FINISH,
requestId: message.requestId,
});
if (isRecordContentMessage(message)) {
const needsAsync = handleRecordContentMessage(
message,
{
activeRequestId: activeRecordRequestId,
capture: recordCapture,
setActiveRequestId: (id) => {
activeRecordRequestId = id;
},
setCapture: (capture) => {
recordCapture = capture;
},
onStart: (requestId, startedAtMs) => {
overlays.setAgentRecordRequest({
id: requestId,
...(typeof startedAtMs === "number" ? { startedAtMs } : {}),
onFinish: () => {
void chrome.runtime.sendMessage({
type: RECORD_FINISH,
requestId,
});
},
});
renderAll();
},
onStop: () => {
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
renderAll();
},
},
});
renderAll();
(sendResponse as unknown as (response: RecordStartAck) => void)({ ok: true });
return false;
}
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
if (activeRecordRequestId !== message.requestId) return false;
overlays.clearAgentRecordRequest(message.requestId);
activeRecordRequestId = null;
renderAll();
if (isRecordStopMessage(message)) {
(sendResponse as unknown as (response: RecordStopAck) => void)({ ok: true });
}
return false;
sendResponse as unknown as
| ((response: RecordStartAck | RecordStopAck) => void)
| undefined,
);
return needsAsync;
}
if (
@@ -518,6 +526,9 @@ export default defineContentScript({
chrome.runtime.onMessage.removeListener(onMessage);
chrome.storage.onChanged.removeListener(onStorageChange);
window.removeEventListener("pageshow", onPageShow);
// Restore history hooks / remove capture listeners before the CS unloads.
recordCapture?.dispose();
recordCapture = null;
activeRecordRequestId = null;
});
},
@@ -200,9 +200,6 @@ describe("App", () => {
expect(copied).toContain("--browser 03c3e47f");
expect(copied).toContain('--purpose "发布 wiki 文档"');
expect(copied).not.toMatch(/bsk record start[^\n]*--url/);
expect(copied).toContain("./trace");
expect(copied).toContain("trace.json");
expect(copied).toContain("states/");
// Button label stays static; a transient toast confirms the copy.
expect(copyButton.textContent).toContain("复制录制指令");
await waitFor(() => expect(screen.getByRole("status").textContent).toContain("已复制"));
@@ -1,14 +0,0 @@
import { attachRecordFrameAgent } from "@/content/recording/frame-agent";
export default defineContentScript({
matches: ["<all_urls>"],
runAt: "document_start",
allFrames: true,
matchAboutBlank: true,
matchOriginAsFallback: true,
main(ctx) {
const dispose = attachRecordFrameAgent();
ctx.onInvalidated(dispose);
},
});
@@ -26,20 +26,20 @@ function handshake(
}
describe("computeConnectedState (protocol-based compat)", () => {
it("returns connected when daemon protocol equals extension protocol", () => {
expect(computeConnectedState(handshake("1.1", "1.0"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
it("returns connected when protocol strings match", () => {
expect(computeConnectedState(handshake("1.0", "1.0"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
kind: "connected",
});
});
it("returns version_skew when daemon protocol minor is newer", () => {
expect(computeConnectedState(handshake("1.2", "1.0"))).toEqual({
expect(computeConnectedState(handshake("1.1", "1.0"))).toEqual({
kind: "version_skew",
});
});
it("returns version_skew when daemon protocol string differs but floor is satisfied", () => {
expect(computeConnectedState(handshake("1.1.0", "1.0"))).toEqual({
expect(computeConnectedState(handshake("1", "1.0"))).toEqual({
kind: "version_skew",
});
});
@@ -53,7 +53,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
});
it("rejects when extension is below daemon min_compatible_protocol", () => {
const result = computeConnectedState(handshake("1.1", "1.5"));
const result = computeConnectedState(handshake("1.0", "1.5"));
expect(result.kind).toBe("rejected");
if (result.kind === "rejected") {
expect(result.reason).toContain("min_compatible_protocol");
@@ -65,7 +65,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
const result = computeConnectedState({
server: "browser-skill-daemon",
version: "0.1.0",
protocol_version: "1.1",
protocol_version: "1.0",
min_compatible_peer: "0.1.0",
});
expect(result).toEqual({ kind: "connected" });
@@ -80,7 +80,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
});
it("rejects malformed daemon min_compatible_protocol with a daemon-floor reason", () => {
const result = computeConnectedState(handshake("1.1", "not-a-protocol"));
const result = computeConnectedState(handshake("1.0", "not-a-protocol"));
expect(result.kind).toBe("rejected");
if (result.kind === "rejected") {
expect(result.reason).toContain("daemon min_compatible_protocol");
@@ -242,11 +242,11 @@ describe("ConnectionController connectionEnabled", () => {
const second = transport.send.mock.calls[1]?.[0] as { id: string };
expect(second.id).not.toBe(first.id);
transport.emitMessage({ id: first.id, result: handshake("1.1", "1.0") });
transport.emitMessage({ id: first.id, result: handshake("1.0", "1.0") });
await Promise.resolve();
expect(controller.snapshot().state).not.toBe("connected");
transport.emitMessage({ id: second.id, result: handshake("1.1", "1.0") });
transport.emitMessage({ id: second.id, result: handshake("1.0", "1.0") });
await vi.waitFor(() => expect(controller.snapshot().state).toBe("connected"));
});
});
@@ -1,101 +0,0 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpRunner } from "@/tools/shared";
import { waitForDocumentSettled } from "../recording/document-settle";
function quietCdp(): { cdp: CdpRunner; send: ReturnType<typeof vi.fn> } {
const send = vi.fn(async () => ({
result: { value: { idleMs: 1_000, readyState: "complete" } },
}));
return {
cdp: { send: send as unknown as CdpRunner["send"] },
send,
};
}
describe("waitForDocumentSettled", () => {
afterEach(() => {
vi.useRealTimers();
});
it("waits through the minimum observation floor before accepting a quiet page", async () => {
vi.useFakeTimers();
const { cdp, send } = quietCdp();
const settled = waitForDocumentSettled(cdp, { target: { tabId: 7 } });
await vi.advanceTimersByTimeAsync(180);
await expect(settled).resolves.toBe("quiet");
expect(send).toHaveBeenCalledTimes(3);
});
it("cancels before probing a superseded observation", async () => {
vi.useFakeTimers();
const { cdp, send } = quietCdp();
const controller = new AbortController();
const settled = waitForDocumentSettled(
cdp,
{ target: { tabId: 7 } },
{
signal: controller.signal,
},
);
controller.abort();
await vi.advanceTimersByTimeAsync(60);
await expect(settled).resolves.toBe("cancelled");
expect(send).not.toHaveBeenCalled();
});
it("probes a same-process iframe through its isolated execution context", async () => {
vi.useFakeTimers();
const send = vi.fn(async (_tabId: number, method: string) => {
if (method === "Page.createIsolatedWorld") return { executionContextId: 91 };
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
});
const cdp: CdpRunner = { send: send as unknown as CdpRunner["send"] };
const settled = waitForDocumentSettled(cdp, {
target: { tabId: 7 },
frameId: "child-frame",
});
await vi.advanceTimersByTimeAsync(180);
await expect(settled).resolves.toBe("quiet");
expect(send).toHaveBeenCalledWith(
7,
"Page.createIsolatedWorld",
expect.objectContaining({ frameId: "child-frame" }),
);
expect(send).toHaveBeenCalledWith(
7,
"Runtime.evaluate",
expect.objectContaining({ contextId: 91 }),
);
});
it("routes an OOPIF probe through its CDP target session", async () => {
vi.useFakeTimers();
const send = vi.fn();
const sendToTarget = vi.fn(async (_target, method: string) => {
if (method === "Page.createIsolatedWorld") return { executionContextId: 27 };
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
});
const cdp: CdpRunner = {
send: send as unknown as CdpRunner["send"],
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
};
const target = { tabId: 7, sessionId: "oopif-session" };
const settled = waitForDocumentSettled(cdp, { target, frameId: "oopif-frame" });
await vi.advanceTimersByTimeAsync(180);
await expect(settled).resolves.toBe("quiet");
expect(send).not.toHaveBeenCalled();
expect(sendToTarget).toHaveBeenCalledWith(
target,
"Runtime.evaluate",
expect.objectContaining({ contextId: 27 }),
);
});
});
@@ -1,32 +0,0 @@
import { describe, expect, it } from "vitest";
import {
isAcceptedRecordStepAck,
isRecordStepMessage,
RECORD_STEP,
type RecordStepMessage,
} from "../record-bridge";
const message: RecordStepMessage = {
type: RECORD_STEP,
requestId: "rec-1",
producerId: "document-1",
sequence: 1,
step: { op: "click", target: { tag: "button", name: "Submit" } },
};
describe("record step delivery protocol", () => {
it("requires a producer and a positive integer sequence", () => {
expect(isRecordStepMessage(message)).toBe(true);
expect(isRecordStepMessage({ ...message, producerId: "" })).toBe(false);
expect(isRecordStepMessage({ ...message, sequence: 0 })).toBe(false);
expect(isRecordStepMessage({ ...message, sequence: 1.5 })).toBe(false);
});
it("accepts only the acknowledgement for the sent sequence", () => {
expect(isAcceptedRecordStepAck({ ok: true, sequence: 2 }, 2)).toBe(true);
expect(isAcceptedRecordStepAck({ ok: true, sequence: 1 }, 2)).toBe(false);
expect(isAcceptedRecordStepAck({ ok: false, expectedSequence: 2, error: "gap" }, 2)).toBe(
false,
);
});
});
@@ -1,213 +0,0 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { RECORD_FRAME_PORT, RECORD_FRAME_QUERY } from "../recording/frame-bridge";
import { RecordFrameCoordinator } from "../recording/frame-coordinator";
class ListenerSet<T extends (...args: never[]) => unknown> {
readonly listeners = new Set<T>();
addListener = (listener: T) => this.listeners.add(listener);
removeListener = (listener: T) => this.listeners.delete(listener);
}
function fakePort(sender: chrome.runtime.MessageSender) {
const messages: unknown[] = [];
const onMessage = new ListenerSet<(message: unknown) => void>();
const onDisconnect = new ListenerSet<() => void>();
return {
port: {
name: RECORD_FRAME_PORT,
sender,
onMessage,
onDisconnect,
postMessage: vi.fn((message: unknown) => messages.push(message)),
disconnect: vi.fn(),
} as unknown as chrome.runtime.Port,
messages,
receive(message: unknown) {
for (const listener of onMessage.listeners) listener(message);
},
};
}
describe("RecordFrameCoordinator", () => {
const onConnect = new ListenerSet<(port: chrome.runtime.Port) => void>();
const onMessage = new ListenerSet<
(
message: unknown,
sender: chrome.runtime.MessageSender,
sendResponse: (response: unknown) => void,
) => boolean
>();
beforeEach(() => {
onConnect.listeners.clear();
onMessage.listeners.clear();
vi.stubGlobal("chrome", {
runtime: { onConnect, onMessage },
});
});
it("answers child-document queries for the initial tab before it is armed", () => {
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [],
sendToDocument: async () => ({ ok: true }),
});
coordinator.attach();
coordinator.begin("rec-1", 10, 3);
const query = (tabId: number) => {
let response: unknown;
for (const listener of onMessage.listeners) {
listener(
{ type: RECORD_FRAME_QUERY },
{
tab: { id: tabId },
frameId: 7,
documentId: "child-document",
} as chrome.runtime.MessageSender,
(value) => {
response = value;
},
);
}
return response;
};
expect(query(3)).toEqual({ active: true, requestId: "rec-1", startedAtMs: 10 });
expect(query(4)).toEqual({ active: false });
});
it("starts a child Document that appears after the initial frame snapshot", async () => {
let notifyFrameNavigation:
| ((frame: {
tabId: number;
frameId: number;
documentId?: string;
lifecycle: "committed" | "completed";
}) => void)
| undefined;
const sendToDocument = vi.fn(async () => ({ ok: true }));
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }],
sendToDocument,
subscribeFrameNavigation(listener) {
notifyFrameNavigation = listener;
return () => {};
},
});
coordinator.attach();
const onDocumentReady = vi.fn();
coordinator.begin("rec-1", 10, 3, onDocumentReady);
await coordinator.armTab("rec-1", 3);
sendToDocument.mockClear();
notifyFrameNavigation?.({
tabId: 3,
frameId: 7,
documentId: "late-child",
lifecycle: "completed",
});
await vi.waitFor(() => {
expect(sendToDocument).toHaveBeenCalledWith(
3,
{ type: "bsk-record-frame-start", requestId: "rec-1", startedAtMs: 10 },
{ documentId: "late-child" },
);
});
expect(onDocumentReady).not.toHaveBeenCalled();
const child = fakePort({
tab: { id: 3 },
frameId: 7,
documentId: "late-child",
} as chrome.runtime.MessageSender);
for (const listener of onConnect.listeners) listener(child.port);
child.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
notifyFrameNavigation?.({
tabId: 3,
frameId: 7,
documentId: "late-child",
lifecycle: "completed",
});
await vi.waitFor(() => {
expect(onDocumentReady).toHaveBeenCalledWith({
tabId: 3,
documentId: "late-child",
browserFrameId: 7,
producerId: "producer-1",
});
});
});
it("binds a producer to its sender Document and keeps final steps valid while stopping", async () => {
const sendToDocument = vi.fn(async () => ({ ok: true }));
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [
{ frameId: 0, documentId: "top-document" },
{ frameId: 7, documentId: "child-document" },
],
sendToDocument,
});
coordinator.attach();
coordinator.begin("rec-1", 10, 3);
await expect(coordinator.armTab("rec-1", 3)).resolves.toBe(true);
const sender = {
tab: { id: 3 },
frameId: 7,
documentId: "child-document",
} as chrome.runtime.MessageSender;
const frame = fakePort(sender);
for (const listener of onConnect.listeners) listener(frame.port);
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toEqual({
tabId: 3,
documentId: "child-document",
browserFrameId: 7,
producerId: "producer-1",
});
const stopping = coordinator.stop("rec-1");
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).not.toBeNull();
const stop = frame.messages.find(
(message): message is { type: "stop"; commandId: string } =>
typeof message === "object" &&
message !== null &&
(message as { type?: string }).type === "stop",
);
expect(stop).toBeDefined();
frame.receive({
type: "stopped",
requestId: "rec-1",
commandId: stop!.commandId,
ok: true,
});
await expect(stopping).resolves.toBe(true);
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toBeNull();
});
it("rejects a producer used from another Document", async () => {
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }],
sendToDocument: async () => ({ ok: true }),
});
coordinator.attach();
coordinator.begin("rec-1", 10, 3);
await coordinator.armTab("rec-1", 3);
const sender = {
tab: { id: 3 },
frameId: 0,
documentId: "top-document",
} as chrome.runtime.MessageSender;
const frame = fakePort(sender);
for (const listener of onConnect.listeners) listener(frame.port);
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
expect(
coordinator.sourceFor("rec-1", "producer-1", {
...sender,
documentId: "other-document",
}),
).toBeNull();
});
});
@@ -1,70 +0,0 @@
import { describe, expect, it } from "vitest";
import {
RECORD_DOCUMENT_ATTRIBUTE,
recordingDocumentMarkerValue,
} from "@/shared/recording-document-identity";
import { markRecordingDocument } from "../recording/document-marker";
import { ObservationNodeIndex } from "../recording/observation-capture";
describe("recording document marker", () => {
it("rejects a non-random Document identity", () => {
expect(() => markRecordingDocument("user@example.com")).toThrow(
"recording Document identity must be a random UUID",
);
});
it("restores the Document attribute exactly", () => {
document.documentElement.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, "page-value");
const producerId = "123e4567-e89b-42d3-a456-426614174000";
const marker = markRecordingDocument(producerId);
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(
recordingDocumentMarkerValue(producerId),
);
marker.restore();
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe("page-value");
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
});
it("indexes the CDP scope from the safe frame identity", () => {
const index = new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "child",
target: { tabId: 3, sessionId: "oopif-session" },
recordingDocumentId: "producer-1",
},
],
matchNodes: [],
refs: [],
});
expect(index.documentScope("producer-1")).toEqual({
frameId: "child",
target: { tabId: 3, sessionId: "oopif-session" },
});
});
it("fails closed when a recording identity maps to multiple Documents", () => {
const index = new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "left",
target: { tabId: 3, sessionId: "left-session" },
recordingDocumentId: "duplicate",
},
{
frameId: "right",
target: { tabId: 3, sessionId: "right-session" },
recordingDocumentId: "duplicate",
},
],
matchNodes: [],
refs: [],
});
expect(index.documentScope("duplicate")).toBeUndefined();
});
});
@@ -1,146 +0,0 @@
import { describe, expect, it } from "vitest";
import { ObservationNodeIndex } from "../recording/observation-capture";
import { RecordingObservationSession } from "../recording/observation-session";
import { RecordingStateRegistry } from "../recording/state-registry";
import { buildTraceV3 } from "../recording/trace-builder-v3";
import type { RecordingDraftStep } from "../recording/types";
const URL = "https://example.com/login";
function sessionWithInput(redactValues = false): RecordingObservationSession {
const session = new RecordingObservationSession({ redactValues });
const state = session.registry.register({
url: URL,
vomText: '@vom 1\ntextbox "Password" value="••••••" [ref=e1]',
});
session.cursor.lastSettled = {
stateId: state.id,
rootFrameId: "root",
index: new ObservationNodeIndex({
rootFrameId: "root",
matchNodes: [
{
frameId: "root",
backendNodeId: 42,
tag: "input",
rect: { x: 20, y: 40, w: 200, h: 30 },
localRect: { x: 20, y: 40, w: 200, h: 30 },
},
],
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
}),
url: URL,
};
return session;
}
function finalizedFillTrace(value: string, redactValues: boolean) {
const session = sessionWithInput(redactValues);
const draft: RecordingDraftStep = {
op: "fill",
captureTarget: { tag: "input", role: "textbox", name: "Password" },
value,
targetHint: {
geometry: { rect: { x: 20, y: 40, w: 200, h: 30 }, tag: "input" },
},
};
session.bindDraft(draft, 1);
draft.postStateId = draft.preStateId;
return buildTraceV3({
registry: session.registry,
drafts: [draft],
annotations: session.annotations,
startedAt: "2026-08-12T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "test",
redactValues,
});
}
function finalizedFillBody(value: string, redactValues: boolean): string {
return finalizedFillTrace(value, redactValues).states[0]!.body;
}
describe("record observation annotations", () => {
it("omits a fill literal when values are redacted", () => {
const secret = "hunter2-private";
const dumped = JSON.stringify(finalizedFillTrace(secret, true));
expect(dumped).toContain("step 1: fill");
expect(dumped).not.toContain(secret);
});
it("keeps ordinary fill details", () => {
expect(finalizedFillBody("ordinary text", false)).toContain('step 1: fill: "ordinary text"');
});
it("encodes title and URL so line breaks cannot corrupt state metadata", () => {
const registry = new RecordingStateRegistry();
const state = registry.register({
url: "https://example.com/a\nb",
title: "hello\nworld",
vomText: "@vom 1",
});
const trace = buildTraceV3({
registry,
drafts: [{ op: "scroll", preStateId: state.id, postStateId: state.id }],
startedAt: "2026-08-12T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "test",
});
expect(trace.states[0]?.body).toContain('url: "https://example.com/a\\nb"');
expect(trace.states[0]?.body).toContain('title: "hello\\nworld"');
});
});
describe("recording state ownership", () => {
it("deduplicates within one recording and isolates ids between recordings", () => {
const first = new RecordingStateRegistry();
const second = new RecordingStateRegistry();
expect(first.register({ url: URL, vomText: "same" }).id).toBe("s1");
expect(first.register({ url: URL, vomText: "same" }).id).toBe("s1");
expect(second.register({ url: URL, vomText: "other" }).id).toBe("s1");
});
it("enriches metadata when a deduplicated observation becomes more complete", () => {
const registry = new RecordingStateRegistry();
registry.register({ url: URL, vomText: "same" });
const state = registry.register({
url: URL,
title: "Login",
vomText: "same",
truncated: true,
});
expect(state).toMatchObject({ id: "s1", title: "Login", truncated: true });
});
});
describe("draft binding", () => {
it("keeps capture semantics when no observation exists", () => {
const session = new RecordingObservationSession();
const draft: RecordingDraftStep = {
op: "hover",
captureTarget: { tag: "button", role: "button", name: "新建" },
};
session.bindDraft(draft, 1);
expect(draft.matchedTarget).toEqual({ role: "button", name: "新建", unmatched: true });
expect(draft.preStateId).toBeUndefined();
});
it("does not bind an unmatched new action to a stale observation", () => {
const session = sessionWithInput();
const draft: RecordingDraftStep = {
op: "click",
captureTarget: { tag: "button", role: "button", name: "Confirm" },
targetHint: {
geometry: { rect: { x: 400, y: 300, w: 80, h: 30 }, tag: "button" },
},
};
session.bindDraft(draft, 2, true);
expect(draft.preStateId).toBeUndefined();
expect("matchedTarget" in draft ? draft.matchedTarget : undefined).toEqual({
role: "button",
name: "Confirm",
unmatched: true,
});
});
});
@@ -1,289 +0,0 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
const captureRecordingObservation = vi.hoisted(() => vi.fn());
const waitForDocumentSettled = vi.hoisted(() => vi.fn(async () => "quiet" as const));
vi.mock("../recording/observation-capture", async (importOriginal) => {
const actual = await importOriginal<typeof import("../recording/observation-capture")>();
return { ...actual, captureRecordingObservation };
});
vi.mock("../recording/document-settle", async (importOriginal) => {
const actual = await importOriginal<typeof import("../recording/document-settle")>();
return { ...actual, waitForDocumentSettled };
});
import { ObservationNodeIndex } from "../recording/observation-capture";
import { RecordingObservationRuntime } from "../recording/recording-runtime";
import type { RecordingDraftStep } from "../recording/types";
function observation(url = "https://example.com/") {
return {
rootFrameId: "root",
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
url,
title: "Example",
vomText: '@vom 1\nRootWebArea "Example"',
truncated: false,
};
}
function runtime(): RecordingObservationRuntime {
return new RecordingObservationRuntime({
cdp: { send: vi.fn() as unknown as CdpRunner["send"] },
tabsApi: { get: vi.fn(), query: vi.fn() } as unknown as ChromeTabsApi,
});
}
describe("RecordingObservationRuntime", () => {
afterEach(() => {
captureRecordingObservation.mockReset();
waitForDocumentSettled.mockClear();
});
it("shares one initial capture and includes it in flush", async () => {
let release!: (value: ReturnType<typeof observation>) => void;
captureRecordingObservation.mockReturnValueOnce(
new Promise((resolve) => {
release = resolve;
}),
);
const recording = runtime();
const first = recording.captureInitial(7);
const second = recording.captureInitial(7);
const flushed = recording.flush();
let flushCompleted = false;
void flushed.then(() => {
flushCompleted = true;
});
await Promise.resolve();
expect(captureRecordingObservation).toHaveBeenCalledTimes(1);
expect(flushCompleted).toBe(false);
release(observation());
await Promise.all([first, second, flushed]);
expect(flushCompleted).toBe(true);
});
it("cancels an in-flight initial capture", async () => {
captureRecordingObservation.mockImplementationOnce(
(input: { signal?: AbortSignal }) =>
new Promise((_, reject) => {
input.signal?.addEventListener(
"abort",
() => reject(new DOMException("observation aborted", "AbortError")),
{ once: true },
);
}),
);
const recording = runtime();
const pending = recording.captureInitial(7);
recording.cancel();
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
});
it("retries a failed initial capture when the recording settles at stop", async () => {
captureRecordingObservation
.mockRejectedValueOnce(new Error("document swapped"))
.mockResolvedValueOnce(observation());
const recording = runtime();
await expect(recording.captureInitial(7)).rejects.toThrow("document swapped");
await recording.settleTrailing(7, []);
const trace = recording.buildTrace({
drafts: [],
startedAt: "2026-08-19T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "0.1.6",
});
expect(captureRecordingObservation).toHaveBeenCalledTimes(2);
expect(trace.states).toHaveLength(1);
});
it("keeps runtime matching data out of the serialized trace", async () => {
const captured = observation();
captured.index = new ObservationNodeIndex({
rootFrameId: "root",
matchNodes: [
{
frameId: "root",
backendNodeId: 42,
tag: "button",
rect: { x: 10, y: 20, w: 100, h: 30 },
localRect: { x: 10, y: 20, w: 100, h: 30 },
},
],
refs: [{ ref: "e1", backendNodeId: 42, role: "button", name: "Submit", line: 1 }],
});
captureRecordingObservation.mockResolvedValueOnce(captured);
const recording = runtime();
await recording.captureInitial(7);
const dumped = JSON.stringify(
recording.buildTrace({
drafts: [],
startedAt: "2026-08-19T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "0.1.6",
}),
);
expect(dumped).toContain("RootWebArea");
expect(dumped).not.toContain("matchNodes");
expect(dumped).not.toContain("localRect");
expect(dumped).not.toContain("backendNodeId");
expect(dumped).not.toContain("ObservationNodeIndex");
});
it("captures a tab transition without sharing observation cursors", async () => {
captureRecordingObservation
.mockResolvedValueOnce(observation("https://example.com/first"))
.mockResolvedValueOnce(observation("https://example.com/second"));
const recording = runtime();
await recording.captureInitial(4);
const transition = await recording.captureTabTransition(4, 5);
await recording.captureInitial(4);
await recording.captureInitial(5);
expect(transition).toEqual({
preStateId: "s1",
postStateId: "s2",
targetUrl: "https://example.com/second",
});
expect(captureRecordingObservation.mock.calls.map(([input]) => input.tabId)).toEqual([4, 5]);
});
it("binds an iframe draft to its marked CDP Document scope", async () => {
captureRecordingObservation.mockResolvedValueOnce({
...observation(),
index: new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "child",
target: { tabId: 7, sessionId: "oopif-session" },
recordingDocumentId: "producer-1",
},
],
matchNodes: [
{
backendNodeId: 42,
frameId: "child",
tag: "button",
rect: { x: 410, y: 20, w: 100, h: 30 },
localRect: { x: 10, y: 20, w: 100, h: 30 },
},
],
refs: [{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
}),
});
const recording = runtime();
const drafts: RecordingDraftStep[] = [
{
op: "click" as const,
captureTarget: { tag: "button", role: "button", name: "Save" },
targetHint: {
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
},
];
await recording.captureInitial(7);
await recording.processDraft(7, drafts, 0, "producer-1");
recording.cancel();
const draft = drafts[0];
expect(draft?.op).toBe("click");
if (!draft || draft.op !== "click") throw new Error("expected click draft");
expect(draft.targetHint).toMatchObject({
frameId: "child",
geometrySpace: "local",
});
expect(draft.matchedTarget?.ref).toBe("e1");
});
it("refreshes a late iframe before matching its first action", async () => {
const iframeObservation = {
...observation(),
index: new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "child",
target: { tabId: 7, sessionId: "oopif-session" },
recordingDocumentId: "producer-1",
},
],
matchNodes: [
{
backendNodeId: 42,
frameId: "child",
tag: "button",
rect: { x: 410, y: 20, w: 100, h: 30 },
localRect: { x: 10, y: 20, w: 100, h: 30 },
},
],
refs: [
{
ref: "e1",
backendNodeId: 42,
frameId: "child",
role: "button",
name: "表格视图",
line: 1,
},
],
}),
};
captureRecordingObservation
.mockResolvedValueOnce(observation())
.mockResolvedValueOnce(iframeObservation)
.mockResolvedValueOnce(iframeObservation);
const recording = runtime();
await recording.captureInitial(7);
await recording.refreshDocument(7, "producer-1");
const drafts: RecordingDraftStep[] = [
{
op: "click",
captureTarget: { tag: "button", role: "button", name: "表格视图" },
targetHint: {
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
},
];
await recording.processDraft(7, drafts, 0, "producer-1");
recording.cancel();
expect(captureRecordingObservation).toHaveBeenCalledTimes(3);
expect(waitForDocumentSettled).toHaveBeenCalledWith(
expect.anything(),
{ frameId: "child", target: { tabId: 7, sessionId: "oopif-session" } },
{ signal: expect.any(AbortSignal) },
);
const draft = drafts[0];
expect(draft?.op).toBe("click");
if (!draft || draft.op !== "click") throw new Error("expected click draft");
expect(draft.matchedTarget).toMatchObject({ ref: "e1", name: "表格视图" });
});
it("does not poison final flush when a frame readiness refresh fails", async () => {
captureRecordingObservation
.mockResolvedValueOnce(observation())
.mockRejectedValueOnce(new Error("child Document replaced"));
const recording = runtime();
await recording.captureInitial(7);
await expect(recording.refreshDocument(7, "producer-1")).rejects.toThrow(
"child Document replaced",
);
await expect(recording.flush()).resolves.toBeUndefined();
});
});
@@ -0,0 +1,80 @@
import { describe, expect, it } from "vitest";
import { appendRecordedPayload, observeRecordedNavigation } from "../recording-step-buffer";
describe("recording-step-buffer", () => {
it("stores semantic click without summary", () => {
const buffer = { steps: [], pendingNavigation: false };
appendRecordedPayload(buffer, {
op: "click",
target: { tag: "button", role: "button", name: "发布" },
expects_navigation: true,
});
expect(buffer.steps).toEqual([
{
op: "click",
target: { tag: "button", role: "button", name: "发布" },
},
]);
expect(buffer.pendingNavigation).toBe(true);
});
it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => {
const buffer = {
steps: [
{
op: "click" as const,
target: { tag: "button", role: "button", name: "发布" },
},
],
currentUrl: "https://example.com/a",
pendingNavigation: true,
pendingNavigationDeadline: Date.now() + 5_000,
};
observeRecordedNavigation(buffer, "https://example.com/b", true);
expect(buffer.steps).toEqual([
{
op: "click",
target: { tag: "button", role: "button", name: "发布" },
navigated_to: "https://example.com/b",
},
]);
expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
});
it("annotates navigated_to onto a select that auto-submits a navigation", () => {
const buffer = {
steps: [
{
op: "select" as const,
target: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
},
],
currentUrl: "https://example.com/list",
pendingNavigation: true,
pendingNavigationDeadline: Date.now() + 5_000,
};
observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true);
expect(buffer.steps).toEqual([
{
op: "select",
target: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
navigated_to: "https://example.com/list?cat=tech",
},
]);
});
it("emits navigate for uncaused URL changes", () => {
const buffer = {
steps: [],
currentUrl: "https://example.com/a",
pendingNavigation: false,
};
observeRecordedNavigation(buffer, "https://example.com/b", false);
expect(buffer.steps[0]).toMatchObject({
op: "navigate",
url: "https://example.com/b",
});
});
});
@@ -1,136 +0,0 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
import { RecordingObservationSession } from "../recording/observation-session";
import { SettleController } from "../recording/settle-controller";
import type { RecordingDraftStep } from "../recording/types";
const OBSERVATION: RegisteredObservation = {
stateId: "s-next",
rootFrameId: "root",
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
url: "https://example.com/next",
};
describe("SettleController", () => {
afterEach(() => vi.useRealTimers());
it("aborts superseded capture work and lands the prior action on the next origin", async () => {
vi.useFakeTimers();
const cdp: CdpRunner = {
send: vi.fn(async () => ({
result: { value: { idleMs: 1_000, readyState: "complete" } },
})) as unknown as CdpRunner["send"],
};
const tabsApi: ChromeTabsApi = {
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
query: vi.fn(async () => []),
};
const session = new RecordingObservationSession();
let firstSignal: AbortSignal | undefined;
vi.spyOn(session, "capture")
.mockImplementationOnce(async (_cdp, _tabs, _tabId, signal) => {
firstSignal = signal;
return new Promise<RegisteredObservation>((_resolve, reject) => {
signal?.addEventListener(
"abort",
() => reject(new DOMException("observation aborted", "AbortError")),
{ once: true },
);
});
})
.mockResolvedValueOnce(OBSERVATION);
const drafts: RecordingDraftStep[] = [
{ op: "click", captureTarget: { tag: "button", name: "First" }, preStateId: "s1" },
];
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
controller.schedule(drafts, 0);
await vi.advanceTimersByTimeAsync(180);
drafts.push({
op: "click",
captureTarget: { tag: "button", name: "Second" },
preStateId: "s-next",
});
controller.schedule(drafts, 1);
expect(firstSignal?.aborted).toBe(true);
expect(drafts[0]?.postStateId).toBe("s-next");
await vi.advanceTimersByTimeAsync(500);
await controller.flush();
expect(drafts[1]?.postStateId).toBe("s-next");
});
it("does not consume a newer redirect while reading the prior landing URL", async () => {
vi.useFakeTimers();
const cdp: CdpRunner = {
send: vi.fn(async () => ({
result: { value: { idleMs: 1_000, readyState: "complete" } },
})) as unknown as CdpRunner["send"],
};
let resolveFirstTab!: (tab: chrome.tabs.Tab) => void;
const firstTab = new Promise<chrome.tabs.Tab>((resolve) => {
resolveFirstTab = resolve;
});
const tabsApi: ChromeTabsApi = {
get: vi
.fn()
.mockImplementationOnce(() => firstTab)
.mockResolvedValue({ id: 7, url: OBSERVATION.url }),
query: vi.fn(async () => []),
};
const session = new RecordingObservationSession();
session.cursor.lastSettled = OBSERVATION;
const drafts: RecordingDraftStep[] = [];
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
controller.scheduleRedirect(drafts, "https://example.com/intermediate");
await vi.advanceTimersByTimeAsync(180);
expect(tabsApi.get).toHaveBeenCalledTimes(1);
controller.scheduleRedirect(drafts, OBSERVATION.url);
resolveFirstTab({ id: 7, url: "https://example.com/intermediate" } as chrome.tabs.Tab);
await vi.advanceTimersByTimeAsync(500);
await controller.flushRedirects();
expect(drafts).toEqual([]);
expect(controller.hasPending).toBe(false);
});
it("settles an action in its own OOPIF document scope", async () => {
vi.useFakeTimers();
const send = vi.fn();
const sendToTarget = vi.fn(async (_target, method: string) => {
if (method === "Page.createIsolatedWorld") return { executionContextId: 39 };
return { result: { value: { idleMs: 1_000, readyState: "complete" } } };
});
const cdp: CdpRunner = {
send: send as unknown as CdpRunner["send"],
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
};
const tabsApi: ChromeTabsApi = {
get: vi.fn(async () => ({ id: 7, url: OBSERVATION.url }) as chrome.tabs.Tab),
query: vi.fn(async () => []),
};
const session = new RecordingObservationSession();
vi.spyOn(session, "capture").mockResolvedValue(OBSERVATION);
const drafts: RecordingDraftStep[] = [
{ op: "click", captureTarget: { tag: "button", name: "Inside frame" } },
];
const controller = new SettleController({ session, cdp, tabsApi, tabId: 7 });
const target = { tabId: 7, sessionId: "oopif-session" };
controller.schedule(drafts, 0, { target, frameId: "oopif-frame" });
await vi.advanceTimersByTimeAsync(180);
await controller.flush();
expect(send).not.toHaveBeenCalled();
expect(sendToTarget).toHaveBeenCalledWith(
target,
"Runtime.evaluate",
expect.objectContaining({ contextId: 39 }),
);
expect(drafts[0]?.postStateId).toBe("s-next");
});
});
@@ -1,164 +0,0 @@
import { describe, expect, it } from "vitest";
import { appendRecordedPayload, observeRecordedNavigation } from "../recording/step-buffer";
describe("recording-step-buffer", () => {
it("stores semantic click without summary", () => {
const buffer = { steps: [], navigation: { pendingNavigation: false } };
appendRecordedPayload(buffer, {
op: "click",
target: { tag: "button", role: "button", name: "发布" },
expects_navigation: true,
});
expect(buffer.steps).toEqual([
{
op: "click",
captureTarget: { tag: "button", role: "button", name: "发布" },
},
]);
expect(buffer.navigation.pendingNavigation).toBe(true);
});
it("keeps the hovered element description as a capture fallback", () => {
const buffer = { steps: [], navigation: { pendingNavigation: false } };
appendRecordedPayload(
buffer,
{
op: "hover",
target: { tag: "button", role: "button", name: "新建" },
geometry: {
rect: { x: 900, y: 8, w: 60, h: 32 },
tag: "button",
},
},
{
geometry: {
rect: { x: 900, y: 8, w: 60, h: 32 },
tag: "button",
},
},
);
expect(buffer.steps).toEqual([
{
op: "hover",
captureTarget: { tag: "button", role: "button", name: "新建" },
targetHint: {
geometry: {
rect: { x: 900, y: 8, w: 60, h: 32 },
tag: "button",
},
},
},
]);
});
it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => {
const buffer = {
steps: [
{
op: "click" as const,
captureTarget: { tag: "button", role: "button", name: "发布" },
},
],
navigation: {
currentUrl: "https://example.com/a",
pendingNavigation: true,
pendingNavigationDeadline: Date.now() + 5_000,
},
};
observeRecordedNavigation(buffer, "https://example.com/b", true);
expect(buffer.steps).toEqual([
{
op: "click",
captureTarget: { tag: "button", role: "button", name: "发布" },
navigatedTo: "https://example.com/b",
},
]);
expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
});
it("annotates navigated_to onto a select that auto-submits a navigation", () => {
const buffer = {
steps: [
{
op: "select" as const,
captureTarget: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
},
],
navigation: {
currentUrl: "https://example.com/list",
pendingNavigation: true,
pendingNavigationDeadline: Date.now() + 5_000,
},
};
observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true);
expect(buffer.steps).toEqual([
{
op: "select",
captureTarget: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
navigatedTo: "https://example.com/list?cat=tech",
},
]);
});
it("emits navigate for uncaused URL changes", () => {
const buffer = {
steps: [],
navigation: { currentUrl: "https://example.com/a", pendingNavigation: false },
};
const result = observeRecordedNavigation(buffer, "https://example.com/b", false);
expect(result).toEqual({ kind: "appended", index: 0 });
expect(buffer.steps[0]).toMatchObject({
op: "navigate",
url: "https://example.com/b",
});
});
it("asks the recorder to coalesce redirect hops instead of emitting each one", () => {
const buffer = {
steps: [],
navigation: {
currentUrl: "https://passport.example/login",
pendingNavigation: false,
},
};
const hop1 = observeRecordedNavigation(
buffer,
"https://passport.example/callback",
false,
"link",
["server_redirect"],
);
expect(hop1).toEqual({
kind: "coalesce_redirect",
url: "https://passport.example/callback",
});
expect(buffer.steps).toEqual([]);
const hop2 = observeRecordedNavigation(buffer, "https://app.example/dashboard", false, "link", [
"client_redirect",
]);
expect(hop2).toEqual({
kind: "coalesce_redirect",
url: "https://app.example/dashboard",
});
expect(buffer.steps).toEqual([]);
expect(buffer.navigation.currentUrl).toBe("https://app.example/dashboard");
});
it("does not reuse redirect metadata for a later content-observed URL change", () => {
const buffer = {
steps: [],
navigation: {
currentUrl: "https://example.com/redirect",
pendingNavigation: false,
},
};
const result = observeRecordedNavigation(buffer, "https://example.com/spa");
expect(result).toEqual({ kind: "appended", index: 0 });
expect(buffer.steps[0]).toMatchObject({ op: "navigate", url: "https://example.com/spa" });
});
});
@@ -1,39 +0,0 @@
import { describe, expect, it } from "vitest";
import { RecordingTabCoordinator } from "../recording/tab-coordinator";
describe("RecordingTabCoordinator", () => {
it("keeps navigation state isolated per tab", () => {
const tabs = new RecordingTabCoordinator(4, "https://example.com/first");
const first = tabs.navigation(4);
first.pendingNavigation = true;
const second = tabs.navigation(5, "https://example.com/second");
expect(second).toEqual({
currentUrl: "https://example.com/second",
pendingNavigation: false,
});
expect(tabs.navigation(4)).toBe(first);
expect(tabs.navigation(4).pendingNavigation).toBe(true);
});
it("invalidates an earlier activation when a newer tab becomes active", () => {
const tabs = new RecordingTabCoordinator(4);
const second = tabs.noteActivation(5);
const third = tabs.noteActivation(6);
expect(tabs.isLatest(second)).toBe(false);
expect(tabs.isLatest(third)).toBe(true);
tabs.commit(6);
expect(tabs.currentTabId).toBe(6);
});
it("commits the freshly observed URL when returning to an existing tab", () => {
const tabs = new RecordingTabCoordinator(4);
tabs.navigation(5, "https://example.com/old");
tabs.commit(5, "https://example.com/new");
expect(tabs.currentTabId).toBe(5);
expect(tabs.navigation(5).currentUrl).toBe("https://example.com/new");
});
});
@@ -1,159 +0,0 @@
import type { RenderedRef } from "@browser-skill/vom";
import { describe, expect, it } from "vitest";
import type { CaptureVomMatchNode } from "@/tools/capture-vom-observation";
import { ObservationNodeIndex, type RegisteredObservation } from "../recording/observation-capture";
import { matchObservationTarget } from "../recording/target-matcher";
function node(backendNodeId: number, frameId: string, x = 10): CaptureVomMatchNode {
return {
backendNodeId,
frameId,
tag: "button",
rect: { x, y: 20, w: 100, h: 30 },
localRect: { x, y: 20, w: 100, h: 30 },
};
}
function observation(
matchNodes: CaptureVomMatchNode[],
refs: RenderedRef[],
): RegisteredObservation {
return {
stateId: "s1",
rootFrameId: "root",
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes, refs }),
url: "https://example.com",
};
}
describe("matchObservationTarget", () => {
it("indexes the safe geometry DTO without discarding frame-local geometry", () => {
const geometry = node(42, "child");
geometry.localRect = { x: 5, y: 6, w: 100, h: 30 };
const ref: RenderedRef = {
ref: "e1",
backendNodeId: 42,
frameId: "child",
line: 1,
};
const candidate = observation([geometry], [ref]).index.candidates("child", "button")[0];
expect(candidate).toEqual({ frameId: "child", geometry, ref });
});
it("matches a unique node using canonical top-level viewport geometry", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "root")],
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
});
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
});
it("uses frame id with backend node id so sibling frames cannot collide", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "left"), node(42, "right")],
[
{ ref: "e1", backendNodeId: 42, frameId: "left", line: 1 },
{ ref: "e2", backendNodeId: 42, frameId: "right", line: 2 },
],
),
hint: {
frameId: "right",
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
});
expect(target.ref).toBe("e2");
});
it("matches iframe capture geometry in the child viewport coordinate space", () => {
const child = node(42, "child", 410);
child.localRect = { x: 10, y: 20, w: 100, h: 30 };
const target = matchObservationTarget({
observation: observation(
[child],
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
),
hint: {
frameId: "child",
geometrySpace: "local",
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
});
expect(target.ref).toBe("e1");
});
it("restricts a missing frame hint to the root frame", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "child")],
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
fallback: { tag: "button", name: "发布" },
});
expect(target).toEqual({ name: "发布", unmatched: true });
});
it("fails closed when the source Document has no VOM frame mapping", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "root")],
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
),
hint: {
frameId: null,
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
fallback: { tag: "button", role: "button", name: "发布" },
});
expect(target).toEqual({ role: "button", name: "发布", unmatched: true });
});
it("returns unmatched for ambiguous geometry", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "root"), node(43, "root")],
[
{ ref: "e1", backendNodeId: 42, line: 1 },
{ ref: "e2", backendNodeId: 43, line: 2 },
],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
});
expect(target.unmatched).toBe(true);
});
it("uses semantics when geometry is unavailable without crossing frame boundaries", () => {
const target = matchObservationTarget({
observation: observation(
[node(41, "root"), node(42, "child")],
[
{ ref: "e1", backendNodeId: 41, role: "button", name: "保存", line: 1 },
{ ref: "e2", backendNodeId: 42, frameId: "child", role: "button", name: "保存", line: 2 },
],
),
hint: { frameId: "child" },
fallback: { tag: "button", role: "button", name: "保存" },
});
expect(target.ref).toBe("e2");
});
it("uses semantics to disambiguate equal geometry in one frame", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "root"), node(43, "root")],
[
{ ref: "e1", backendNodeId: 42, role: "button", name: "保存", line: 1 },
{ ref: "e2", backendNodeId: 43, role: "button", name: "取消", line: 2 },
],
),
hint: { geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" } },
fallback: { tag: "button", role: "button", name: "取消" },
});
expect(target.ref).toBe("e2");
});
});
@@ -1,132 +0,0 @@
import { describe, expect, it } from "vitest";
import { TRACE_VERSION_V3, VOM_FORMAT_VERSION } from "@/transport/types";
import { RecordingStateRegistry } from "../recording/state-registry";
import { buildTraceV3 } from "../recording/trace-builder-v3";
import { reduceTraceStepsV3 } from "../recording/trace-reducer-v3";
import type { RecordingDraftStep } from "../recording/types";
describe("trace reducer v3", () => {
it("removes form literals when value redaction is requested", () => {
const reduced = reduceTraceStepsV3(
[
{
op: "fill",
value: "private@example.com",
preStateId: "s1",
postStateId: "s1",
},
{
op: "select",
values: ["private-account-id"],
labels: ["Private account"],
preStateId: "s1",
postStateId: "s1",
},
],
{ redactValues: true },
);
expect(reduced.steps[0]).toMatchObject({ op: "fill", value: "***", redacted: true });
expect(reduced.steps[1]).toMatchObject({ op: "select" });
expect(reduced.steps[1]).not.toHaveProperty("selection");
expect(JSON.stringify(reduced.steps)).not.toContain("private@example.com");
expect(JSON.stringify(reduced.steps)).not.toContain("private-account-id");
});
it("emits tab transitions only for callers that advertised support", () => {
const drafts: RecordingDraftStep[] = [
{
op: "switch_tab",
preStateId: "s1",
postStateId: "s2",
},
];
expect(reduceTraceStepsV3(drafts).steps).toEqual([]);
expect(reduceTraceStepsV3(drafts, { includeTabSwitches: true }).steps).toEqual([
{ op: "switch_tab", id: 1, state: "s1", result: { state: "s2" } },
]);
});
it("keeps hover steps before menu clicks", () => {
const drafts: RecordingDraftStep[] = [
{
op: "hover",
captureTarget: { tag: "span", role: "button", name: "Account" },
preStateId: "s1",
postStateId: "s1",
},
{
op: "click",
captureTarget: { tag: "a", role: "link", name: "Profile" },
preStateId: "s1",
postStateId: "s1",
},
];
expect(reduceTraceStepsV3(drafts).steps.map((s) => s.op)).toEqual(["hover", "click"]);
});
it("does not export a tab transition without both observation endpoints", () => {
const sourceOnly: RecordingDraftStep = { op: "switch_tab", preStateId: "s1" };
const targetOnly: RecordingDraftStep = { op: "switch_tab", postStateId: "s2" };
expect(
reduceTraceStepsV3([sourceOnly, targetOnly], { includeTabSwitches: true }).steps,
).toEqual([]);
});
it("collapses redirect hops while retaining draft-to-step identity", () => {
const drafts: RecordingDraftStep[] = [
{ op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" },
{
op: "navigate",
url: "https://example.com/final",
transitionQualifiers: ["server_redirect"],
preStateId: "s2",
postStateId: "s3",
},
];
const before = structuredClone(drafts);
const reduced = reduceTraceStepsV3(drafts);
expect(drafts).toEqual(before);
expect(reduced.steps).toEqual([
expect.objectContaining({
id: 1,
op: "navigate",
state: "s1",
to: "https://example.com/final",
result: { state: "s3" },
}),
]);
expect(reduced.stepIdByDraftId.get(1)).toBe(1);
expect(reduced.stepIdByDraftId.get(2)).toBe(1);
});
it("builds the wire model from protocol constants", () => {
const registry = new RecordingStateRegistry();
const state = registry.register({ url: "https://example.com", vomText: "@vom 1" });
const trace = buildTraceV3({
registry,
drafts: [
{
op: "click",
captureTarget: { tag: "button", role: "button", name: "Save" },
preStateId: state.id,
postStateId: state.id,
},
],
startedAt: "2026-08-12T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "test",
});
expect(trace.version).toBe(TRACE_VERSION_V3);
expect(trace.recorder.vom).toBe(VOM_FORMAT_VERSION);
expect(trace.steps[0]).toMatchObject({
op: "click",
target: { role: "button", name: "Save", unmatched: true },
});
});
});
@@ -1,15 +1,6 @@
import { describe, expect, it } from "vitest";
import type { RecordingDraftStep } from "@/lib/recording/types";
import { shouldRecordPress } from "../recording/draft-policy";
import { buildTraceV2 } from "../recording/trace-reducer-v2";
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string) {
return buildTraceV2({
steps,
startedAt: "2026-01-01T00:00:00.000Z",
...(startUrl ? { startUrl } : {}),
});
}
import type { DraftTraceStep } from "@/transport/types";
import { reduceTraceSteps, resolveTraceStartUrl, shouldRecordPress } from "../trace-reducer";
describe("shouldRecordPress", () => {
it("keeps Enter and Escape", () => {
@@ -27,47 +18,36 @@ describe("shouldRecordPress", () => {
});
describe("reduceTraceSteps", () => {
it("does not expose internal tab transitions in trace v2", () => {
const trace = reduceTraceSteps([
{
op: "switch_tab",
preStateId: "s1",
postStateId: "s2",
},
]);
expect(trace.steps).toEqual([]);
});
it("builds steps with pages dictionary and page id references", () => {
const drafts: RecordingDraftStep[] = [
const drafts: DraftTraceStep[] = [
{
op: "navigate",
url: "https://example.com/search?q=hello&utm_source=x",
pageUrl: "https://example.com/search?q=hello&utm_source=x",
page_url: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "fill",
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
value: "browser skill",
pageUrl: "https://example.com/search?q=hello&utm_source=x",
page_url: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "press",
key: "Enter",
captureTarget: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
navigatedTo: "https://example.com/results/42",
pageUrl: "https://example.com/search?q=hello&utm_source=x",
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
navigated_to: "https://example.com/results/42",
page_url: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "click",
captureTarget: { tag: "button", role: "button", name: "发布" },
navigatedTo: "https://example.com/p/99",
pageUrl: "https://example.com/results/42",
target: { tag: "button", role: "button", name: "发布" },
navigated_to: "https://example.com/p/99",
page_url: "https://example.com/results/42",
},
{
op: "press",
key: "a",
pageUrl: "https://example.com/p/99",
page_url: "https://example.com/p/99",
},
];
@@ -129,9 +109,9 @@ describe("reduceTraceSteps", () => {
[
{
op: "fill",
captureTarget: { tag: "input", role: "textbox", name: "Search query" },
target: { tag: "input", role: "textbox", name: "Search query" },
value: "",
pageUrl: "https://example.com/search",
page_url: "https://example.com/search",
},
],
"https://example.com/search",
@@ -145,16 +125,16 @@ describe("reduceTraceSteps", () => {
]);
});
it("maps select navigatedTo onto effect.navigatedTo (page id)", () => {
it("maps select navigated_to onto effect.navigated_to (page id)", () => {
const { pages, steps } = reduceTraceSteps(
[
{
op: "select",
captureTarget: { tag: "select", role: "combobox", name: "分类" },
target: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
labels: ["技术"],
navigatedTo: "https://example.com/list?cat=tech",
pageUrl: "https://example.com/list",
navigated_to: "https://example.com/list?cat=tech",
page_url: "https://example.com/list",
},
],
"https://example.com/list",
@@ -171,32 +151,36 @@ describe("reduceTraceSteps", () => {
});
});
it("drops hover steps, which only exist in trace v3", () => {
it("keeps hover steps before menu clicks", () => {
const { steps } = reduceTraceSteps(
[
{
op: "hover",
captureTarget: { tag: "span", role: "button", name: "Account" },
pageUrl: "https://example.com/app",
target: { tag: "span", role: "button", name: "Account" },
page_url: "https://example.com/app",
},
{
op: "click",
captureTarget: { tag: "a", role: "link", name: "Profile" },
pageUrl: "https://example.com/app",
target: { tag: "a", role: "link", name: "Profile" },
page_url: "https://example.com/app",
},
],
"https://example.com/app",
);
expect(steps.map((s) => s.op)).toEqual(["click"]);
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
page: "p1",
target: { name: "Account" },
});
});
it("resolveTraceStartUrl prefers explicit start URL", () => {
expect(
buildTraceV2({
steps: [{ op: "navigate", url: "https://example.com/other" }],
startedAt: "2026-01-01T00:00:00.000Z",
startUrl: "https://example.com/start",
}).entry.start_url,
resolveTraceStartUrl(
[{ op: "navigate", url: "https://example.com/other" }],
"https://example.com/start",
),
).toBe("https://example.com/start");
});
});
+5 -6
View File
@@ -1,13 +1,12 @@
/**
* Build a semantic capture descriptor for an interacted element.
* Build a semantic TargetDescriptor for an interacted element.
*
* Trace steps are an LLM *textbook*: each click must say what to look for
* on screen (usually a short visible name). Tag-only noise like
* `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
*/
/** Content-script capture descriptor before VOM geometric matching. */
export interface CaptureTargetDescriptor {
export interface TargetDescriptor {
role?: string;
name?: string;
tag: string;
@@ -248,7 +247,7 @@ export function resolveHoverElement(target: Element): Element | null {
* (visible name), or at least a form `name_attr` for checkbox/radio.
* Recording “点击div” with no name fails this bar.
*/
export function isMeaningfulClickTarget(target: CaptureTargetDescriptor): boolean {
export function isMeaningfulClickTarget(target: TargetDescriptor): boolean {
const name = target.name?.trim();
if (name && isActionableLabel(name)) return true;
if (
@@ -324,7 +323,7 @@ function nearbyLabelText(el: Element): string | undefined {
return undefined;
}
export function describeTarget(el: Element): CaptureTargetDescriptor {
export function describeTarget(el: Element): TargetDescriptor {
const tag = el.tagName.toLowerCase();
const role = inferRole(el);
const name = accessibleName(el);
@@ -351,7 +350,7 @@ export function describeTarget(el: Element): CaptureTargetDescriptor {
};
}
export function describeEventTarget(target: EventTarget | null): CaptureTargetDescriptor | null {
export function describeEventTarget(target: EventTarget | null): TargetDescriptor | null {
if (!(target instanceof Element)) return null;
const clickable = resolveClickableElement(target);
if (!clickable) return null;
+3 -39
View File
@@ -3,7 +3,7 @@
* service worker and a tab's content script.
*/
import type { CaptureTargetDescriptor } from "./describe-target";
import type { TargetDescriptor } from "./describe-target";
export const RECORD_START = "bsk-record-start";
export const RECORD_STEP = "bsk-record-step";
@@ -16,10 +16,6 @@ export interface RecordStartAck {
ok: true;
}
export type RecordStepAck =
| { ok: true; sequence: number }
| { ok: false; expectedSequence: number; error: string };
export type RecordStopAck = { ok: true } | { ok: false; error: string };
export interface RecordQueryMessage {
@@ -46,7 +42,7 @@ export interface RecordStartMessage {
export interface RecordStepPayload {
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
target?: CaptureTargetDescriptor;
target?: TargetDescriptor;
value?: string;
key?: string;
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
@@ -54,30 +50,17 @@ export interface RecordStepPayload {
labels?: string[];
url?: string;
redacted?: boolean;
commit?: "enter" | "suggestion" | "blur";
/** Page URL when the step was captured. */
page_url?: string;
/** Event-target bounds in the source frame's viewport coordinate space. */
geometry?: {
rect: { x: number; y: number; w: number; h: number };
tag: string;
};
/** Capture-only hint; never persisted unless converted to navigated_to. */
expects_navigation?: boolean;
/** Whether an observed URL change was synchronously caused by the action. */
navigation_caused_by_action?: boolean;
/** Raw webNavigation transition metadata for cause mapping. */
transitionType?: string;
transitionQualifiers?: string[];
}
export interface RecordStepMessage {
type: typeof RECORD_STEP;
requestId: string;
/** Stable for the lifetime of one content-script document. */
producerId: string;
/** Monotonic within one content-script producer and recording request. */
sequence: number;
step: RecordStepPayload;
}
@@ -129,27 +112,8 @@ export function isRecordQueryMessage(msg: unknown): msg is RecordQueryMessage {
export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage {
if (typeof msg !== "object" || msg === null) return false;
const m = msg as Record<string, unknown>;
if (
m.type !== RECORD_STEP ||
typeof m.requestId !== "string" ||
typeof m.producerId !== "string" ||
m.producerId.length === 0 ||
typeof m.sequence !== "number" ||
!Number.isSafeInteger(m.sequence) ||
m.sequence < 1
) {
return false;
}
if (m.type !== RECORD_STEP || typeof m.requestId !== "string") return false;
const step = m.step;
if (typeof step !== "object" || step === null) return false;
return typeof (step as RecordStepPayload).op === "string";
}
export function isAcceptedRecordStepAck(
value: unknown,
sequence: number,
): value is Extract<RecordStepAck, { ok: true }> {
if (typeof value !== "object" || value === null) return false;
const ack = value as Partial<RecordStepAck>;
return ack.ok === true && ack.sequence === sequence;
}
@@ -0,0 +1,136 @@
import type { DraftTraceStep } from "@/transport/types";
import type { RecordStepPayload } from "./record-bridge";
export interface RecordingStepBuffer {
steps: DraftTraceStep[];
currentUrl?: string;
pendingNavigation: boolean;
pendingNavigationDeadline?: number;
}
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
const pageUrl = payload.page_url;
switch (payload.op) {
case "click":
return payload.target
? {
op: "click",
target: payload.target,
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
case "hover":
return payload.target
? {
op: "hover",
target: payload.target,
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
case "fill":
return payload.target
? {
op: "fill",
target: payload.target,
value: payload.value ?? "",
...(payload.redacted ? { redacted: true } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
case "press":
return payload.key
? {
op: "press",
key: payload.key,
...(payload.target ? { target: payload.target } : {}),
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
case "select":
return payload.target && payload.values
? {
op: "select",
target: payload.target,
values: payload.values,
...(payload.labels?.length ? { labels: payload.labels } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
case "navigate":
return null;
}
}
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): boolean {
for (let i = buffer.steps.length - 1; i >= 0; i -= 1) {
const step = buffer.steps[i];
if (!step) continue;
if (step.op === "click" || step.op === "press" || step.op === "select") {
buffer.steps[i] = { ...step, navigated_to: url };
return true;
}
break;
}
return false;
}
export function observeRecordedNavigation(
buffer: RecordingStepBuffer,
url: string,
causedByAction?: boolean,
): void {
if (!url || url === buffer.currentUrl) return;
buffer.currentUrl = url;
const pendingIsCurrent =
buffer.pendingNavigation &&
(buffer.pendingNavigationDeadline === undefined ||
buffer.pendingNavigationDeadline >= Date.now());
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
buffer.pendingNavigation = false;
buffer.pendingNavigationDeadline = undefined;
if (!annotateLastStepNavigation(buffer, url)) {
buffer.steps.push({
op: "navigate",
url,
page_url: url,
});
}
return;
}
buffer.pendingNavigation = false;
buffer.pendingNavigationDeadline = undefined;
buffer.steps.push({
op: "navigate",
url,
page_url: url,
});
}
export function appendRecordedPayload(
buffer: RecordingStepBuffer,
payload: RecordStepPayload,
): void {
if (payload.op === "navigate") {
if (payload.url) {
observeRecordedNavigation(buffer, payload.url, payload.navigation_caused_by_action);
}
return;
}
const step = toDraftStep({
...payload,
page_url: payload.page_url ?? buffer.currentUrl,
});
if (!step) return;
buffer.steps.push(step);
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
buffer.pendingNavigation = payload.expects_navigation === true;
buffer.pendingNavigationDeadline = buffer.pendingNavigation
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
: undefined;
}
}
@@ -1,39 +0,0 @@
import {
RECORD_DOCUMENT_ATTRIBUTE,
recordingDocumentMarkerValue,
} from "@/shared/recording-document-identity";
export interface RecordingDocumentMarker {
restore(): void;
ensure(): void;
}
export function waitForDocumentElement(): Promise<HTMLElement> {
if (document.documentElement) return Promise.resolve(document.documentElement);
return new Promise((resolve) => {
const observer = new MutationObserver(() => {
if (!document.documentElement) return;
observer.disconnect();
resolve(document.documentElement);
});
observer.observe(document, { childList: true });
});
}
export function markRecordingDocument(
producerId: string,
root: HTMLElement = document.documentElement,
): RecordingDocumentMarker {
const previous = root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE);
const markerValue = recordingDocumentMarkerValue(producerId);
const ensure = () => root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, markerValue);
ensure();
return {
ensure,
restore() {
if (root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE) !== markerValue) return;
if (previous === null) root.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
else root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, previous);
},
};
}
@@ -1,134 +0,0 @@
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { type CdpRunner, sendToCdpTarget } from "@/tools/shared";
const SETTLE_MIN_MS = 150;
const SETTLE_QUIET_MS = 250;
const SETTLE_MAX_MS = 2_000;
const SETTLE_POLL_MS = 60;
const SETTLE_WORLD_NAME = "__bsk_record_settle__";
const QUIET_PROBE = `(() => {
const scope = window;
let probe = scope.__bskRecordQuiet;
if (!probe) {
probe = { changedAt: Date.now() };
const observer = new MutationObserver(() => {
probe.changedAt = Date.now();
});
observer.observe(document.documentElement, {
subtree: true,
childList: true,
attributes: true,
characterData: true,
});
scope.__bskRecordQuiet = probe;
}
return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState };
})()`;
export interface DocumentSettleScope {
target: CdpTarget;
/** Omit only for the target's root document. */
frameId?: string;
}
type SettleOutcome = "quiet" | "timeout" | "cancelled";
interface QuietProbe {
idleMs: number;
readyState: string;
}
interface ProbeContext {
executionContextId?: number;
}
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve) => {
if (signal?.aborted) return resolve();
const timer = setTimeout(resolve, ms);
signal?.addEventListener(
"abort",
() => {
clearTimeout(timer);
resolve();
},
{ once: true },
);
});
}
async function executionContextForScope(
cdp: CdpRunner,
scope: DocumentSettleScope,
): Promise<number | undefined> {
if (!scope.frameId) return undefined;
const result = await sendToCdpTarget<{ executionContextId?: number }>(
cdp,
scope.target,
"Page.createIsolatedWorld",
{
frameId: scope.frameId,
worldName: SETTLE_WORLD_NAME,
grantUniveralAccess: false,
},
);
return result.executionContextId;
}
async function readQuietProbe(
cdp: CdpRunner,
scope: DocumentSettleScope,
context: ProbeContext,
): Promise<QuietProbe | null> {
try {
if (scope.frameId && context.executionContextId === undefined) {
context.executionContextId = await executionContextForScope(cdp, scope);
if (context.executionContextId === undefined) return null;
}
const reply = await sendToCdpTarget<{ result?: { value?: unknown } }>(
cdp,
scope.target,
"Runtime.evaluate",
{
expression: QUIET_PROBE,
returnByValue: true,
...(context.executionContextId !== undefined
? { contextId: context.executionContextId }
: {}),
},
);
const value = reply.result?.value;
if (!value || typeof value !== "object") return null;
const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown };
if (typeof idleMs !== "number" || typeof readyState !== "string") return null;
return { idleMs, readyState };
} catch {
context.executionContextId = undefined;
return null;
}
}
export async function waitForDocumentSettled(
cdp: CdpRunner,
scope: DocumentSettleScope,
options: { signal?: AbortSignal } = {},
): Promise<SettleOutcome> {
const startedAt = Date.now();
const floor = startedAt + SETTLE_MIN_MS;
const deadline = startedAt + SETTLE_MAX_MS;
const context: ProbeContext = {};
for (;;) {
if (options.signal?.aborted) return "cancelled";
await sleep(SETTLE_POLL_MS, options.signal);
if (options.signal?.aborted) return "cancelled";
const probe = await readQuietProbe(cdp, scope, context);
const now = Date.now();
if (now < floor) continue;
if (now >= deadline) return "timeout";
if (!probe || probe.readyState === "loading") continue;
if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet";
}
}
@@ -1,34 +0,0 @@
import type { RecordingDraftStep } from "./types";
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
export function shouldRecordPress(
key: string,
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
): boolean {
if (MODIFIER_ONLY_KEYS.has(key)) return false;
const mods = modifiers ?? [];
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
if (key === "Enter" || key === "Escape") return true;
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
return false;
}
export function shouldIncludeDraft(step: RecordingDraftStep): boolean {
return step.op !== "press" || shouldRecordPress(step.key, step.modifiers);
}
export function resolveDraftStartUrl(
drafts: RecordingDraftStep[],
explicitStartUrl?: string,
fallbackUrl?: string,
): string {
if (explicitStartUrl) return explicitStartUrl;
const navigation = drafts.find(
(step): step is Extract<RecordingDraftStep, { op: "navigate" }> => step.op === "navigate",
);
if (navigation) return navigation.url;
return drafts.find((step) => step.pageUrl)?.pageUrl ?? fallbackUrl ?? "about:blank";
}
@@ -1,65 +0,0 @@
export const RECORD_FRAME_PORT = "bsk-record-frame";
export const RECORD_FRAME_QUERY = "bsk-record-frame-query";
export const RECORD_FRAME_START = "bsk-record-frame-start";
export interface RecordFrameQueryMessage {
type: typeof RECORD_FRAME_QUERY;
}
export interface RecordFrameQueryResponse {
active: boolean;
requestId?: string;
startedAtMs?: number;
}
export interface RecordFrameStartMessage {
type: typeof RECORD_FRAME_START;
requestId: string;
startedAtMs: number;
}
export type RecordFramePortMessage =
| {
type: "ready";
requestId: string;
producerId: string;
}
| {
type: "ready_ack";
requestId: string;
producerId: string;
}
| {
type: "stop";
requestId: string;
commandId: string;
}
| {
type: "cancel";
requestId: string;
}
| {
type: "stopped";
requestId: string;
commandId: string;
ok: boolean;
error?: string;
};
export function isRecordFrameQueryMessage(value: unknown): value is RecordFrameQueryMessage {
return (
typeof value === "object" &&
value !== null &&
(value as { type?: unknown }).type === RECORD_FRAME_QUERY
);
}
export function isRecordFrameStartMessage(value: unknown): value is RecordFrameStartMessage {
if (typeof value !== "object" || value === null) return false;
const message = value as Partial<RecordFrameStartMessage>;
return (
message.type === RECORD_FRAME_START &&
typeof message.requestId === "string" &&
typeof message.startedAtMs === "number"
);
}
@@ -1,399 +0,0 @@
import {
isRecordFrameQueryMessage,
RECORD_FRAME_PORT,
RECORD_FRAME_START,
type RecordFramePortMessage,
type RecordFrameQueryResponse,
type RecordFrameStartMessage,
} from "./frame-bridge";
export interface RecordingCaptureScope {
tabId: number;
documentId: string;
browserFrameId: number;
producerId: string;
}
interface ArmedRecording {
requestId: string;
startedAtMs: number;
tabIds: Set<number>;
agents: Map<string, FrameAgent>;
onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise<void>;
finishing: boolean;
}
interface FrameAgent extends RecordingCaptureScope {
requestId: string;
port: chrome.runtime.Port;
stopWaiters: Map<string, (ok: boolean) => void>;
}
interface BrowserFrame {
frameId: number;
documentId?: string;
}
interface BrowserFrameNavigation extends BrowserFrame {
tabId: number;
lifecycle: "committed" | "completed";
}
const RECORD_FRAME_STOP_TIMEOUT_MS = 5_000;
export interface RecordFrameCoordinatorDeps {
getAllFrames(tabId: number): Promise<BrowserFrame[]>;
sendToDocument(
tabId: number,
message: RecordFrameStartMessage,
target: { documentId?: string; frameId?: number },
): Promise<unknown>;
subscribeFrameNavigation?(listener: (frame: BrowserFrameNavigation) => void): () => void;
}
function documentKey(tabId: number, documentId: string): string {
return `${tabId}:${documentId}`;
}
function senderAddress(
sender: chrome.runtime.MessageSender,
): { tabId: number; documentId: string; browserFrameId: number } | null {
const tabId = sender.tab?.id;
const documentId = sender.documentId;
const browserFrameId = sender.frameId;
if (
typeof tabId !== "number" ||
typeof documentId !== "string" ||
typeof browserFrameId !== "number"
) {
return null;
}
return { tabId, documentId, browserFrameId };
}
function defaultDeps(): RecordFrameCoordinatorDeps {
return {
async getAllFrames(tabId) {
return (await chrome.webNavigation.getAllFrames({ tabId })) ?? [];
},
sendToDocument(tabId, message, target) {
return chrome.tabs.sendMessage(tabId, message, target);
},
subscribeFrameNavigation(listener) {
const onCommitted = (details: chrome.webNavigation.WebNavigationTransitionCallbackDetails) =>
listener({ ...details, lifecycle: "committed" });
const onCompleted = (details: chrome.webNavigation.WebNavigationFramedCallbackDetails) =>
listener({ ...details, lifecycle: "completed" });
chrome.webNavigation.onCommitted.addListener(onCommitted);
chrome.webNavigation.onCompleted.addListener(onCompleted);
return () => {
chrome.webNavigation.onCommitted.removeListener(onCommitted);
chrome.webNavigation.onCompleted.removeListener(onCompleted);
};
},
};
}
export class RecordFrameCoordinator {
readonly #deps: RecordFrameCoordinatorDeps;
readonly #recordings = new Map<string, ArmedRecording>();
#attached = false;
#detachFrameNavigation: (() => void) | null = null;
constructor(deps: RecordFrameCoordinatorDeps = defaultDeps()) {
this.#deps = deps;
}
attach(): () => void {
if (this.#attached) return () => {};
this.#attached = true;
chrome.runtime.onConnect.addListener(this.#onConnect);
chrome.runtime.onMessage.addListener(this.#onMessage);
this.#detachFrameNavigation =
this.#deps.subscribeFrameNavigation?.(this.#onFrameNavigation) ?? null;
return () => {
if (!this.#attached) return;
this.#attached = false;
chrome.runtime.onConnect.removeListener(this.#onConnect);
chrome.runtime.onMessage.removeListener(this.#onMessage);
this.#detachFrameNavigation?.();
this.#detachFrameNavigation = null;
for (const recording of this.#recordings.values()) this.#cancelAgents(recording);
this.#recordings.clear();
};
}
begin(
requestId: string,
startedAtMs: number,
initialTabId: number,
onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise<void>,
): void {
const previous = this.#recordings.get(requestId);
if (previous) this.#cancelAgents(previous);
this.#recordings.set(requestId, {
requestId,
startedAtMs,
// Register the initial tab before navigation begins. A newly-created
// child document can query at document_start before armTab's frame
// snapshot sees it, especially when Chromium promotes it to an OOPIF.
tabIds: new Set([initialTabId]),
agents: new Map(),
...(onDocumentReady ? { onDocumentReady } : {}),
finishing: false,
});
}
async armTab(requestId: string, tabId: number): Promise<boolean> {
const recording = this.#recordings.get(requestId);
if (!recording || recording.finishing) return false;
recording.tabIds.add(tabId);
let frames: BrowserFrame[];
try {
frames = await this.#deps.getAllFrames(tabId);
} catch {
frames = [{ frameId: 0 }];
}
if (!frames.some((frame) => frame.frameId === 0)) frames.unshift({ frameId: 0 });
const results = await Promise.all(
frames.map(async (frame) => {
const started = await this.#startFrame(recording, tabId, frame);
return { frameId: frame.frameId, started };
}),
);
return results.some((result) => result.frameId === 0 && result.started);
}
sourceFor(
requestId: string,
producerId: string,
sender: chrome.runtime.MessageSender,
): RecordingCaptureScope | null {
const address = senderAddress(sender);
const recording = this.#recordings.get(requestId);
if (!address || !recording) return null;
const agent = recording.agents.get(documentKey(address.tabId, address.documentId));
if (!agent || agent.producerId !== producerId) return null;
return {
tabId: agent.tabId,
documentId: agent.documentId,
browserFrameId: agent.browserFrameId,
producerId: agent.producerId,
};
}
async stop(requestId: string): Promise<boolean> {
const recording = this.#recordings.get(requestId);
if (!recording) return true;
recording.finishing = true;
const commandId = crypto.randomUUID();
const results = await Promise.all(
[...recording.agents.values()].map((agent) => this.#stopAgent(agent, requestId, commandId)),
);
const succeeded = results.every(Boolean);
if (succeeded) this.#recordings.delete(requestId);
else recording.finishing = false;
return succeeded;
}
cancel(requestId: string): void {
const recording = this.#recordings.get(requestId);
if (!recording) return;
this.#cancelAgents(recording);
this.#recordings.delete(requestId);
}
readonly #onMessage = (
message: unknown,
sender: chrome.runtime.MessageSender,
sendResponse: (response: RecordFrameQueryResponse) => void,
): boolean => {
if (!isRecordFrameQueryMessage(message)) return false;
const tabId = sender.tab?.id;
const recording =
typeof tabId === "number"
? [...this.#recordings.values()].find(
(candidate) => !candidate.finishing && candidate.tabIds.has(tabId),
)
: undefined;
sendResponse(
recording
? {
active: true,
requestId: recording.requestId,
startedAtMs: recording.startedAtMs,
}
: { active: false },
);
return false;
};
readonly #onFrameNavigation = (frame: BrowserFrameNavigation): void => {
if (frame.frameId === 0) return;
for (const recording of this.#recordings.values()) {
if (recording.finishing || !recording.tabIds.has(frame.tabId)) continue;
// onCommitted can race document_start, while onCompleted runs after the
// content script is available. Starting twice is safe and closes both
// the early-query and late-SPA-frame gaps.
void (async () => {
const started = await this.#startFrame(recording, frame.tabId, frame);
if (!started || frame.lifecycle !== "completed" || recording.finishing) return;
const agent = frame.documentId
? recording.agents.get(documentKey(frame.tabId, frame.documentId))
: [...recording.agents.values()].find(
(candidate) =>
candidate.tabId === frame.tabId && candidate.browserFrameId === frame.frameId,
);
if (!agent) return;
await recording.onDocumentReady?.({
tabId: agent.tabId,
documentId: agent.documentId,
browserFrameId: agent.browserFrameId,
producerId: agent.producerId,
});
})();
}
};
readonly #onConnect = (port: chrome.runtime.Port): void => {
if (port.name !== RECORD_FRAME_PORT) return;
const address = senderAddress(port.sender ?? {});
if (!address) {
port.disconnect();
return;
}
let registered: FrameAgent | null = null;
port.onMessage.addListener((raw: unknown) => {
const message = raw as Partial<RecordFramePortMessage>;
if (message.type === "ready") {
if (typeof message.requestId !== "string" || typeof message.producerId !== "string") {
port.disconnect();
return;
}
const requestId = message.requestId;
const producerId = message.producerId;
if (registered) {
if (registered.requestId === requestId && registered.producerId === producerId) {
port.postMessage({
type: "ready_ack",
requestId,
producerId,
} satisfies RecordFramePortMessage);
} else {
port.disconnect();
}
return;
}
const recording = this.#recordings.get(requestId);
if (
!recording ||
recording.finishing ||
!recording.tabIds.has(address.tabId) ||
producerId.length === 0
) {
port.disconnect();
return;
}
const agent: FrameAgent = {
...address,
requestId,
producerId,
port,
stopWaiters: new Map(),
};
registered = agent;
const key = documentKey(address.tabId, address.documentId);
const previous = recording.agents.get(key);
if (previous && previous !== agent) previous.port.disconnect();
recording.agents.set(key, agent);
port.postMessage({
type: "ready_ack",
requestId,
producerId,
} satisfies RecordFramePortMessage);
return;
}
if (
message.type === "stopped" &&
registered &&
message.requestId === registered.requestId &&
typeof message.commandId === "string"
) {
registered.stopWaiters.get(message.commandId)?.(message.ok === true);
registered.stopWaiters.delete(message.commandId);
}
});
port.onDisconnect.addListener(() => {
if (!registered) return;
for (const resolve of registered.stopWaiters.values()) resolve(true);
const recording = this.#recordings.get(registered.requestId);
const key = documentKey(registered.tabId, registered.documentId);
if (recording?.agents.get(key) === registered) recording.agents.delete(key);
});
};
#stopAgent(agent: FrameAgent, requestId: string, commandId: string): Promise<boolean> {
return new Promise((resolve) => {
const timer = setTimeout(() => {
agent.stopWaiters.delete(commandId);
resolve(false);
}, RECORD_FRAME_STOP_TIMEOUT_MS);
agent.stopWaiters.set(commandId, (ok) => {
clearTimeout(timer);
resolve(ok);
});
try {
agent.port.postMessage({
type: "stop",
requestId,
commandId,
} satisfies RecordFramePortMessage);
} catch {
clearTimeout(timer);
agent.stopWaiters.delete(commandId);
resolve(true);
}
});
}
async #startFrame(
recording: ArmedRecording,
tabId: number,
frame: BrowserFrame,
): Promise<boolean> {
const message: RecordFrameStartMessage = {
type: RECORD_FRAME_START,
requestId: recording.requestId,
startedAtMs: recording.startedAtMs,
};
try {
const response = await this.#deps.sendToDocument(tabId, message, {
...(frame.documentId ? { documentId: frame.documentId } : { frameId: frame.frameId }),
});
return isStarted(response);
} catch {
return false;
}
}
#cancelAgents(recording: ArmedRecording): void {
for (const agent of recording.agents.values()) {
try {
agent.port.postMessage({
type: "cancel",
requestId: recording.requestId,
} satisfies RecordFramePortMessage);
} catch {
// A destroyed Document has no remaining capture state to cancel.
}
}
}
}
function isStarted(value: unknown): boolean {
return typeof value === "object" && value !== null && (value as { ok?: unknown }).ok === true;
}
export const recordFrameCoordinator = new RecordFrameCoordinator();
@@ -1,5 +0,0 @@
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
export function hasRedirectQualifier(qualifiers?: readonly string[]): boolean {
return (qualifiers ?? []).some((qualifier) => REDIRECT_QUALIFIERS.has(qualifier));
}
@@ -1,140 +0,0 @@
import type { RenderedRef } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import {
type CaptureVomMatchNode,
type CaptureVomObservationResult,
captureVomObservation,
} from "@/tools/capture-vom-observation";
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
export interface IndexedObservationNode {
frameId: string;
geometry: CaptureVomMatchNode;
ref?: RenderedRef;
}
export interface CapturedRecordingObservation {
rootFrameId: string;
index: ObservationNodeIndex;
url: string;
title?: string;
vomText: string;
truncated: boolean;
}
export interface RegisteredObservation {
stateId: string;
rootFrameId: string;
index: ObservationNodeIndex;
url: string;
}
export interface RecordingDocumentScope {
frameId: string;
target: CdpTarget;
}
function nodeKey(frameId: string, backendNodeId: number): string {
return `${frameId}:${backendNodeId}`;
}
function frameTagKey(frameId: string, tag: string): string {
return `${frameId}:${tag.toLowerCase()}`;
}
export class ObservationNodeIndex {
readonly #nodesByFrameTag = new Map<string, IndexedObservationNode[]>();
readonly #refById = new Map<string, RenderedRef>();
readonly #refsByFrame = new Map<string, RenderedRef[]>();
readonly #scopeByProducer = new Map<string, RecordingDocumentScope | null>();
constructor(
input: Pick<CaptureVomObservationResult, "rootFrameId" | "matchNodes" | "refs"> &
Partial<Pick<CaptureVomObservationResult, "frames">>,
) {
const refByNode = new Map<string, RenderedRef>();
for (const ref of input.refs) {
const frameId = ref.frameId ?? input.rootFrameId;
refByNode.set(nodeKey(frameId, ref.backendNodeId), ref);
this.#refById.set(ref.ref, ref);
const frameRefs = this.#refsByFrame.get(frameId) ?? [];
frameRefs.push(ref);
this.#refsByFrame.set(frameId, frameRefs);
}
for (const frame of input.frames ?? []) {
const documentId = frame.recordingDocumentId;
if (!documentId) continue;
this.#scopeByProducer.set(
documentId,
this.#scopeByProducer.has(documentId)
? null
: { frameId: frame.frameId, target: frame.target },
);
}
for (const geometry of input.matchNodes) {
const { frameId } = geometry;
const entry = {
frameId,
geometry,
ref: refByNode.get(nodeKey(frameId, geometry.backendNodeId)),
};
const key = frameTagKey(frameId, geometry.tag);
const bucket = this.#nodesByFrameTag.get(key) ?? [];
bucket.push(entry);
this.#nodesByFrameTag.set(key, bucket);
}
}
candidates(frameId: string, tag: string): readonly IndexedObservationNode[] {
return this.#nodesByFrameTag.get(frameTagKey(frameId, tag)) ?? [];
}
ref(refId: string): RenderedRef | undefined {
return this.#refById.get(refId);
}
refs(frameId: string): readonly RenderedRef[] {
return this.#refsByFrame.get(frameId) ?? [];
}
documentScope(producerId: string): RecordingDocumentScope | undefined {
return this.#scopeByProducer.get(producerId) ?? undefined;
}
}
async function readTabMeta(
tabsApi: ChromeTabsApi,
tabId: number,
): Promise<{ url: string; title?: string }> {
try {
const tab = await tabsApi.get(tabId);
return { url: tab.url ?? "about:blank", title: tab.title };
} catch {
return { url: "about:blank" };
}
}
export async function captureRecordingObservation(input: {
cdp: CdpRunner;
tabsApi: ChromeTabsApi;
tabId: number;
maxTokens: number;
redactValues: boolean;
signal?: AbortSignal;
}): Promise<CapturedRecordingObservation> {
const { url, title } = await readTabMeta(input.tabsApi, input.tabId);
const captured = await captureVomObservation(input.cdp, input.tabId, url, {
maxTokens: input.maxTokens,
redactValues: input.redactValues,
conditionalSurfaceProbe: false,
signal: input.signal,
});
return {
rootFrameId: captured.rootFrameId,
index: new ObservationNodeIndex(captured),
url,
title,
vomText: captured.text,
truncated: captured.truncated,
};
}
@@ -1,128 +0,0 @@
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import { captureRecordingObservation, type RegisteredObservation } from "./observation-capture";
import { RecordingStateRegistry } from "./state-registry";
import { matchObservationTarget, unmatchedTarget } from "./target-matcher";
import type { RecordingDraftStep, StepAnnotation, TargetedRecordingDraft } from "./types";
const DEFAULT_MAX_PAGE_TOKENS = 3_000;
const MIN_CAPTURE_INTERVAL_MS = 200;
export interface TabObservationCursor {
lastSettled: RegisteredObservation | null;
lastCaptureAt: number;
}
function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve) => {
if (signal?.aborted) return resolve();
const timer = setTimeout(resolve, ms);
signal?.addEventListener(
"abort",
() => {
clearTimeout(timer);
resolve();
},
{ once: true },
);
});
}
function isTargeted(draft: RecordingDraftStep): draft is TargetedRecordingDraft {
return (
draft.op === "click" ||
draft.op === "hover" ||
draft.op === "fill" ||
draft.op === "press" ||
draft.op === "select"
);
}
export class RecordingObservationSession {
readonly registry: RecordingStateRegistry;
readonly cursor: TabObservationCursor;
readonly annotations: StepAnnotation[];
readonly #maxTokens: number;
readonly #redactValues: boolean;
constructor(
options: {
registry?: RecordingStateRegistry;
cursor?: TabObservationCursor;
annotations?: StepAnnotation[];
maxTokens?: number;
redactValues?: boolean;
} = {},
) {
this.registry = options.registry ?? new RecordingStateRegistry();
this.cursor = options.cursor ?? { lastSettled: null, lastCaptureAt: 0 };
this.annotations = options.annotations ?? [];
this.#maxTokens = options.maxTokens ?? DEFAULT_MAX_PAGE_TOKENS;
this.#redactValues = options.redactValues ?? false;
}
async capture(
cdp: CdpRunner,
tabsApi: ChromeTabsApi,
tabId: number,
signal?: AbortSignal,
): Promise<RegisteredObservation> {
const waitMs = Math.max(0, MIN_CAPTURE_INTERVAL_MS - (Date.now() - this.cursor.lastCaptureAt));
if (waitMs > 0) await abortableDelay(waitMs, signal);
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
const captured = await captureRecordingObservation({
cdp,
tabsApi,
tabId,
maxTokens: this.#maxTokens,
redactValues: this.#redactValues,
signal,
});
const state = this.registry.register({
url: captured.url,
title: captured.title,
vomText: captured.vomText,
truncated: captured.truncated,
});
const observation: RegisteredObservation = {
stateId: state.id,
rootFrameId: captured.rootFrameId,
index: captured.index,
url: captured.url,
};
this.cursor.lastSettled = observation;
this.cursor.lastCaptureAt = Date.now();
return observation;
}
bindDraft(draft: RecordingDraftStep, draftId: number, previousActionPending = false): void {
const observation = this.cursor.lastSettled;
if (isTargeted(draft)) {
draft.matchedTarget = observation
? matchObservationTarget({
observation,
hint: draft.targetHint,
fallback: draft.captureTarget,
})
: unmatchedTarget(draft.captureTarget);
}
if (!observation) return;
const unmatched = isTargeted(draft) && draft.matchedTarget?.unmatched === true;
if (previousActionPending && unmatched) return;
draft.preStateId = observation.stateId;
this.registry.markStep(observation.stateId, draftId);
if (!isTargeted(draft) || !draft.matchedTarget?.ref) return;
const ref = observation.index.ref(draft.matchedTarget.ref);
if (!ref) return;
this.annotations.push({
draftId,
op: draft.op,
line: ref.line,
stateId: observation.stateId,
...(draft.op === "fill" && !this.#redactValues
? { detail: JSON.stringify(draft.value) }
: {}),
});
}
}
@@ -1,277 +0,0 @@
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import type { StopReason, TraceV3 } from "@/transport/types";
import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
import type { RegisteredObservation } from "./observation-capture";
import { RecordingObservationSession } from "./observation-session";
import { inferMissingPostStates, SettleController } from "./settle-controller";
import { RecordingStateRegistry } from "./state-registry";
import { buildTraceV3 } from "./trace-builder-v3";
import type { RecordingDraftStep, StepAnnotation } from "./types";
interface TabRecordingContext {
session: RecordingObservationSession;
settle: SettleController;
pendingCapture: PendingCapture | null;
frameRefresh: Promise<void>;
frameRefreshAbort: AbortController | null;
}
interface PendingCapture {
promise: Promise<RegisteredObservation>;
abort: AbortController;
}
export class RecordingObservationRuntime {
readonly #cdp: CdpRunner;
readonly #tabsApi: ChromeTabsApi;
readonly #registry = new RecordingStateRegistry();
readonly #annotations: StepAnnotation[] = [];
readonly #contexts = new Map<number, TabRecordingContext>();
readonly #maxTokens?: number;
readonly #redactValues: boolean;
constructor(input: {
cdp: CdpRunner;
tabsApi: ChromeTabsApi;
maxTokens?: number;
redactValues?: boolean;
}) {
this.#cdp = input.cdp;
this.#tabsApi = input.tabsApi;
this.#maxTokens = input.maxTokens;
this.#redactValues = input.redactValues ?? false;
}
#context(tabId: number): TabRecordingContext {
const existing = this.#contexts.get(tabId);
if (existing) return existing;
const session = new RecordingObservationSession({
registry: this.#registry,
annotations: this.#annotations,
maxTokens: this.#maxTokens,
redactValues: this.#redactValues,
});
const context: TabRecordingContext = {
session,
settle: new SettleController({
session,
cdp: this.#cdp,
tabsApi: this.#tabsApi,
tabId,
}),
pendingCapture: null,
frameRefresh: Promise.resolve(),
frameRefreshAbort: null,
};
this.#contexts.set(tabId, context);
return context;
}
async #capture(tabId: number): Promise<RegisteredObservation> {
const context = this.#context(tabId);
if (context.pendingCapture) return context.pendingCapture.promise;
const abort = new AbortController();
let pending!: PendingCapture;
const promise = context.session.capture(this.#cdp, this.#tabsApi, tabId, abort.signal);
pending = {
abort,
promise: promise.finally(() => {
if (context.pendingCapture === pending) context.pendingCapture = null;
}),
};
context.pendingCapture = pending;
return pending.promise;
}
async captureInitial(tabId: number): Promise<void> {
const context = this.#context(tabId);
if (context.session.cursor.lastSettled) return;
await this.#capture(tabId);
}
/** Refresh the safe observation after a late child Document becomes usable. */
async refreshDocument(tabId: number, producerId: string): Promise<void> {
const context = this.#context(tabId);
context.frameRefreshAbort?.abort();
const abort = new AbortController();
context.frameRefreshAbort = abort;
const previous = context.frameRefresh;
const refresh = (async () => {
await previous;
if (abort.signal.aborted) return;
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
if (abort.signal.aborted) return;
// The first capture discovers the safe producer -> CDP Document scope.
await this.#capture(tabId);
if (abort.signal.aborted) return;
const scope = context.session.cursor.lastSettled?.index.documentScope(producerId);
if (scope) {
await waitForDocumentSettled(this.#cdp, scope, { signal: abort.signal });
}
if (abort.signal.aborted) return;
// Capture again after the child SPA is quiet so the next action can be
// matched against refs that actually exist in its pre-action state.
await this.#capture(tabId);
})();
const tracked = refresh.catch(() => {});
context.frameRefresh = tracked;
try {
await refresh;
} finally {
if (context.frameRefresh === tracked) context.frameRefreshAbort = null;
}
}
async captureTabTransition(
fromTabId: number,
toTabId: number,
): Promise<{ preStateId?: string; postStateId?: string; targetUrl?: string }> {
const from = this.#context(fromTabId);
await this.#flushContext(from);
if (!from.session.cursor.lastSettled) {
try {
await this.captureInitial(fromTabId);
} catch {
// A target-tab observation can still preserve the forward transition.
}
}
const to = this.#context(toTabId);
let target = to.session.cursor.lastSettled;
try {
target = await this.#capture(toTabId);
} catch {
// Reuse the last settled target state when a refresh races navigation.
}
const source = from.session.cursor.lastSettled;
return {
...(source ? { preStateId: source.stateId } : {}),
...(target ? { postStateId: target.stateId, targetUrl: target.url } : {}),
};
}
bindTabTransition(
draft: Extract<RecordingDraftStep, { op: "switch_tab" }>,
draftId: number,
): void {
if (draft.preStateId) this.#registry.markStep(draft.preStateId, draftId);
}
async processDraft(
tabId: number,
drafts: RecordingDraftStep[],
draftIndex: number,
producerId?: string,
): Promise<void> {
const draft = drafts[draftIndex];
if (!draft) return;
const context = this.#context(tabId);
if (!context.session.cursor.lastSettled) {
try {
await this.captureInitial(tabId);
} catch {
// Post-action settle can still provide a usable state.
}
}
const scope = producerId
? context.session.cursor.lastSettled?.index.documentScope(producerId)
: undefined;
if (
producerId &&
(draft.op === "click" ||
draft.op === "hover" ||
draft.op === "fill" ||
draft.op === "press" ||
draft.op === "select")
) {
draft.targetHint = {
...(draft.targetHint ?? {}),
frameId: scope?.frameId ?? null,
geometrySpace: "local",
};
}
context.session.bindDraft(draft, draftIndex + 1, context.settle.hasPending);
context.settle.schedule(drafts, draftIndex, scope);
}
scheduleSettle(
tabId: number,
drafts: RecordingDraftStep[],
draftIndex: number,
scope?: DocumentSettleScope,
): void {
this.#context(tabId).settle.schedule(drafts, draftIndex, scope);
}
clearRedirect(tabId: number): void {
this.#contexts.get(tabId)?.settle.clearRedirect();
}
scheduleRedirect(tabId: number, drafts: RecordingDraftStep[], url: string): void {
this.#context(tabId).settle.scheduleRedirect(drafts, url);
}
async #flushContext(context: TabRecordingContext): Promise<void> {
await context.frameRefresh;
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
await context.settle.flushRedirects();
await context.settle.flush();
}
async flushRedirects(tabId: number): Promise<void> {
await this.#contexts.get(tabId)?.settle.flushRedirects();
}
async flush(): Promise<void> {
for (const context of this.#contexts.values()) await this.#flushContext(context);
}
async settleTrailing(tabId: number, drafts: RecordingDraftStep[]): Promise<void> {
const context = this.#context(tabId);
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
if (!context.session.cursor.lastSettled) {
try {
await this.captureInitial(tabId);
} catch {
// The trace may remain observation-free when CDP is unavailable.
}
}
await context.settle.settleTrailing(drafts);
inferMissingPostStates(drafts);
}
cancel(): void {
for (const context of this.#contexts.values()) {
context.frameRefreshAbort?.abort();
context.pendingCapture?.abort.abort();
context.settle.cancel();
}
}
buildTrace(input: {
drafts: RecordingDraftStep[];
startedAt: string;
purpose?: string;
startUrl?: string;
stoppedBy: StopReason;
bskVersion: string;
includeTabSwitches?: boolean;
}): TraceV3 {
return buildTraceV3({
registry: this.#registry,
drafts: input.drafts,
annotations: this.#annotations,
startedAt: input.startedAt,
purpose: input.purpose,
startUrl: input.startUrl,
stoppedBy: input.stoppedBy,
bskVersion: input.bskVersion,
redactValues: this.#redactValues,
includeTabSwitches: input.includeTabSwitches,
});
}
}
@@ -1,233 +0,0 @@
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
import { RecordingObservationSession } from "./observation-session";
import type { RecordingDraftStep } from "./types";
interface PendingSettle {
abort: AbortController;
scope: DocumentSettleScope;
}
interface PendingRedirect {
url: string;
abort: AbortController;
}
const CAPTURE_RETRY_DELAY_MS = 250;
function delay(ms: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve) => {
if (signal?.aborted) return resolve();
const timer = setTimeout(resolve, ms);
signal?.addEventListener(
"abort",
() => {
clearTimeout(timer);
resolve();
},
{ once: true },
);
});
}
function isAbortError(error: unknown): boolean {
return (
typeof error === "object" &&
error !== null &&
(error as { name?: string }).name === "AbortError"
);
}
export function inferMissingPostStates(drafts: RecordingDraftStep[]): void {
for (let index = 0; index < drafts.length - 1; index += 1) {
const draft = drafts[index];
const next = drafts[index + 1];
if (draft && next && !draft.postStateId && next.preStateId) {
draft.postStateId = next.preStateId;
}
}
}
export class SettleController {
readonly #session: RecordingObservationSession;
readonly #cdp: CdpRunner;
readonly #tabsApi: ChromeTabsApi;
readonly #tabId: number;
readonly #rootScope: DocumentSettleScope;
readonly #pending = new Map<number, PendingSettle>();
#queue = Promise.resolve();
#pendingRedirect: PendingRedirect | null = null;
#redirectQueue = Promise.resolve();
constructor(input: {
session: RecordingObservationSession;
cdp: CdpRunner;
tabsApi: ChromeTabsApi;
tabId: number;
}) {
this.#session = input.session;
this.#cdp = input.cdp;
this.#tabsApi = input.tabsApi;
this.#tabId = input.tabId;
this.#rootScope = { target: { tabId: input.tabId } };
}
get hasPending(): boolean {
return this.#pending.size > 0 || this.#pendingRedirect !== null;
}
async #captureWithRetry(signal?: AbortSignal) {
try {
return await this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
} catch (error) {
if (signal?.aborted) throw error;
await delay(CAPTURE_RETRY_DELAY_MS, signal);
return this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
}
}
schedule(
drafts: RecordingDraftStep[],
draftIndex: number,
scope: DocumentSettleScope = this.#rootScope,
): void {
if (scope.target.tabId !== this.#tabId) {
throw new Error(
`settle scope tab ${scope.target.tabId} does not belong to tab ${this.#tabId}`,
);
}
const landing = drafts[draftIndex]?.preStateId;
for (const [index, pending] of this.#pending) {
if (index >= draftIndex) continue;
pending.abort.abort();
this.#pending.delete(index);
if (landing && drafts[index] && !drafts[index].postStateId) {
drafts[index].postStateId = landing;
}
}
this.#pending.get(draftIndex)?.abort.abort();
const pending = { abort: new AbortController(), scope };
this.#pending.set(draftIndex, pending);
const task = async () => {
if (pending.abort.signal.aborted) return;
try {
const outcome = await waitForDocumentSettled(this.#cdp, pending.scope, {
signal: pending.abort.signal,
});
if (outcome === "cancelled") return;
const observation = await this.#captureWithRetry(pending.abort.signal);
if (!pending.abort.signal.aborted && drafts[draftIndex]) {
drafts[draftIndex].postStateId = observation.stateId;
}
} catch (error) {
if (!isAbortError(error)) {
console.warn(
`[bsk record] post-action observation failed for step ${draftIndex + 1}`,
error,
);
}
} finally {
if (this.#pending.get(draftIndex) === pending) this.#pending.delete(draftIndex);
}
};
this.#queue = this.#queue.then(task, task).catch(() => {});
}
cancel(): void {
for (const pending of this.#pending.values()) pending.abort.abort();
this.#pending.clear();
this.clearRedirect();
}
async flush(): Promise<void> {
for (let round = 0; round < 10; round += 1) {
const current = this.#queue;
await current;
if (current === this.#queue) return;
}
console.warn("[bsk record] settle queue kept growing; using completed observations");
}
async settleTrailing(drafts: RecordingDraftStep[]): Promise<void> {
const trailing: RecordingDraftStep[] = [];
for (let index = drafts.length - 1; index >= 0; index -= 1) {
const draft = drafts[index];
if (!draft || draft.postStateId) break;
trailing.push(draft);
}
if (trailing.length === 0) return;
try {
const observation = await this.#captureWithRetry();
for (const draft of trailing) draft.postStateId = observation.stateId;
} catch (error) {
console.warn("[bsk record] final observation at stop failed", error);
}
}
clearRedirect(): void {
this.#pendingRedirect?.abort.abort();
this.#pendingRedirect = null;
}
scheduleRedirect(drafts: RecordingDraftStep[], url: string): void {
this.#pendingRedirect?.abort.abort();
const pending: PendingRedirect = { url, abort: new AbortController() };
this.#pendingRedirect = pending;
const task = () => this.#settleRedirect(drafts, pending);
this.#redirectQueue = this.#redirectQueue.then(task, task).catch(() => {});
}
async #settleRedirect(drafts: RecordingDraftStep[], pending: PendingRedirect): Promise<void> {
const outcome = await waitForDocumentSettled(this.#cdp, this.#rootScope, {
signal: pending.abort.signal,
});
if (outcome === "cancelled" || this.#pendingRedirect !== pending) return;
let finalUrl = pending.url;
try {
finalUrl = (await this.#tabsApi.get(this.#tabId)).url || finalUrl;
} catch {
// The navigation event URL remains the best available destination.
}
if (this.#pendingRedirect !== pending) return;
this.#pendingRedirect = null;
if (
!finalUrl ||
finalUrl === "about:blank" ||
this.#session.cursor.lastSettled?.url === finalUrl
) {
return;
}
const last = drafts[drafts.length - 1];
if (last?.op === "navigate" && last.url === finalUrl) {
if (!last.postStateId) this.schedule(drafts, drafts.length - 1);
await this.flush();
return;
}
const draft: RecordingDraftStep = {
op: "navigate",
url: finalUrl,
pageUrl: finalUrl,
cause: "browser",
transitionQualifiers: ["server_redirect"],
preStateId: this.#session.cursor.lastSettled?.stateId,
};
drafts.push(draft);
const draftIndex = drafts.length - 1;
if (draft.preStateId) this.#session.registry.markStep(draft.preStateId, draftIndex + 1);
this.schedule(drafts, draftIndex);
await this.flush();
}
async flushRedirects(): Promise<void> {
for (let round = 0; round < 10; round += 1) {
const current = this.#redirectQueue;
await current;
if (current === this.#redirectQueue) return;
}
console.warn("[bsk record] redirect queue kept growing; using the final completed landing");
}
}
@@ -1,57 +0,0 @@
export interface RecordedStateEntry {
id: string;
url: string;
title?: string;
/** Rendered VOM text; capture indexes and source DOM/AX data never enter the registry. */
vomText: string;
truncated: boolean;
stepsHere: number[];
}
function stateIdentity(url: string, body: string): string {
return `${url}\0${body}`;
}
export class RecordingStateRegistry {
readonly #entriesById = new Map<string, RecordedStateEntry>();
readonly #idByIdentity = new Map<string, string>();
#nextId = 1;
register(input: {
url: string;
title?: string;
vomText: string;
truncated?: boolean;
}): RecordedStateEntry {
const identity = stateIdentity(input.url, input.vomText);
const existingId = this.#idByIdentity.get(identity);
if (existingId) {
const existing = this.#entriesById.get(existingId)!;
if (!existing.title && input.title) existing.title = input.title;
if (input.truncated) existing.truncated = true;
return existing;
}
const entry: RecordedStateEntry = {
id: `s${this.#nextId}`,
url: input.url,
...(input.title ? { title: input.title } : {}),
vomText: input.vomText,
truncated: input.truncated ?? false,
stepsHere: [],
};
this.#nextId += 1;
this.#entriesById.set(entry.id, entry);
this.#idByIdentity.set(identity, entry.id);
return entry;
}
values(): RecordedStateEntry[] {
return [...this.#entriesById.values()];
}
markStep(stateId: string, draftId: number): void {
const entry = this.#entriesById.get(stateId);
if (entry && !entry.stepsHere.includes(draftId)) entry.stepsHere.push(draftId);
}
}
@@ -1,155 +0,0 @@
import type { RecordStepPayload } from "../record-bridge";
import { hasRedirectQualifier } from "./navigation-policy";
import type { RecordingDraftStep, TargetMatchHint } from "./types";
export interface RecordingStepBuffer {
steps: RecordingDraftStep[];
navigation: RecordingNavigationCursor;
}
export interface RecordingNavigationCursor {
currentUrl?: string;
pendingNavigation: boolean;
pendingNavigationDeadline?: number;
}
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
function toDraftStep(
payload: RecordStepPayload,
targetHint?: TargetMatchHint,
): RecordingDraftStep | null {
const pageUrl = payload.page_url;
const common = {
...(pageUrl ? { pageUrl } : {}),
...(targetHint ? { targetHint } : {}),
};
switch (payload.op) {
case "click":
return payload.target ? { op: "click", captureTarget: payload.target, ...common } : null;
case "hover":
return payload.target ? { op: "hover", captureTarget: payload.target, ...common } : null;
case "fill":
return payload.target
? {
op: "fill",
captureTarget: payload.target,
value: payload.value ?? "",
...(payload.commit ? { commit: payload.commit } : {}),
...(payload.redacted ? { redacted: true } : {}),
...common,
}
: null;
case "press":
return payload.key
? {
op: "press",
key: payload.key,
...(payload.target ? { captureTarget: payload.target } : {}),
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
...common,
}
: null;
case "select":
return payload.target && payload.values
? {
op: "select",
captureTarget: payload.target,
values: payload.values,
...(payload.labels?.length ? { labels: payload.labels } : {}),
...common,
}
: null;
case "navigate":
return null;
}
}
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): number {
for (let index = buffer.steps.length - 1; index >= 0; index -= 1) {
const step = buffer.steps[index];
if (!step) continue;
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
step.navigatedTo = url;
return index;
}
break;
}
return -1;
}
export type NavigationObserveResult =
| { kind: "noop" }
| { kind: "annotated"; index: number }
| { kind: "appended"; index: number }
| { kind: "coalesce_redirect"; url: string };
export function observeRecordedNavigation(
buffer: RecordingStepBuffer,
url: string,
causedByAction?: boolean,
transitionType?: string,
transitionQualifiers?: string[],
): NavigationObserveResult {
const navigation = buffer.navigation;
if (!url || url === navigation.currentUrl) return { kind: "noop" };
navigation.currentUrl = url;
const pendingIsCurrent =
navigation.pendingNavigation &&
(navigation.pendingNavigationDeadline === undefined ||
navigation.pendingNavigationDeadline >= Date.now());
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
navigation.pendingNavigation = false;
navigation.pendingNavigationDeadline = undefined;
const annotatedIndex = annotateLastStepNavigation(buffer, url);
if (annotatedIndex >= 0) return { kind: "annotated", index: annotatedIndex };
} else {
navigation.pendingNavigation = false;
navigation.pendingNavigationDeadline = undefined;
}
if (hasRedirectQualifier(transitionQualifiers)) {
return { kind: "coalesce_redirect", url };
}
buffer.steps.push({
op: "navigate",
url,
pageUrl: url,
transitionType,
transitionQualifiers,
});
return { kind: "appended", index: buffer.steps.length - 1 };
}
export function appendRecordedPayload(
buffer: RecordingStepBuffer,
payload: RecordStepPayload,
targetHint?: TargetMatchHint,
): number | null {
if (payload.op === "navigate") {
if (!payload.url) return null;
const result = observeRecordedNavigation(
buffer,
payload.url,
payload.navigation_caused_by_action,
payload.transitionType,
payload.transitionQualifiers,
);
return result.kind === "appended" ? result.index : null;
}
const step = toDraftStep(
{ ...payload, page_url: payload.page_url ?? buffer.navigation.currentUrl },
targetHint,
);
if (!step) return null;
buffer.steps.push(step);
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
buffer.navigation.pendingNavigation = payload.expects_navigation === true;
buffer.navigation.pendingNavigationDeadline = buffer.navigation.pendingNavigation
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
: undefined;
}
return buffer.steps.length - 1;
}
@@ -1,61 +0,0 @@
import type { RecordingNavigationCursor } from "./step-buffer";
export interface TabActivation {
tabId: number;
revision: number;
}
export class RecordingTabCoordinator {
readonly #navigationByTab = new Map<number, RecordingNavigationCursor>();
#activeTabId: number;
#currentTabId: number;
#activationRevision = 0;
constructor(initialTabId: number, initialUrl?: string) {
this.#activeTabId = initialTabId;
this.#currentTabId = initialTabId;
this.#navigationByTab.set(initialTabId, {
currentUrl: initialUrl,
pendingNavigation: false,
});
}
get activeTabId(): number {
return this.#activeTabId;
}
get currentTabId(): number {
return this.#currentTabId;
}
noteActivation(tabId: number): TabActivation {
this.#activeTabId = tabId;
this.#activationRevision += 1;
return { tabId, revision: this.#activationRevision };
}
isLatest(activation: TabActivation): boolean {
return (
activation.tabId === this.#activeTabId && activation.revision === this.#activationRevision
);
}
commit(tabId: number, currentUrl?: string): void {
if (currentUrl !== undefined) this.navigation(tabId).currentUrl = currentUrl;
this.#currentTabId = tabId;
}
navigation(tabId: number, fallbackUrl?: string): RecordingNavigationCursor {
const existing = this.#navigationByTab.get(tabId);
if (existing) {
if (!existing.currentUrl && fallbackUrl) existing.currentUrl = fallbackUrl;
return existing;
}
const created: RecordingNavigationCursor = {
currentUrl: fallbackUrl,
pendingNavigation: false,
};
this.#navigationByTab.set(tabId, created);
return created;
}
}
@@ -1,86 +0,0 @@
import type { RenderedRef } from "@browser-skill/vom";
import type { TargetDescriptorV3 } from "@/transport/types";
import type { CaptureTargetDescriptor } from "../describe-target";
import type { IndexedObservationNode, RegisteredObservation } from "./observation-capture";
import type { TargetGeometry, TargetMatchHint } from "./types";
const MATCH_TOLERANCE_PX = 2;
function close(a: number, b: number): boolean {
return Math.abs(a - b) <= MATCH_TOLERANCE_PX;
}
function rectMatches(a: TargetGeometry["rect"], b: TargetGeometry["rect"]): boolean {
return close(a.x, b.x) && close(a.y, b.y) && close(a.w, b.w) && close(a.h, b.h);
}
function candidateMatches(
candidate: IndexedObservationNode,
geometry: TargetGeometry,
geometrySpace: "top" | "local",
): boolean {
const rect = geometrySpace === "local" ? candidate.geometry.localRect : candidate.geometry.rect;
return rect != null && rectMatches(geometry.rect, rect);
}
export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 {
return {
...(fallback?.role ? { role: fallback.role } : {}),
...(fallback?.name ? { name: fallback.name } : {}),
unmatched: true,
};
}
function normalized(value: string | undefined): string {
return value?.replace(/\s+/g, " ").trim().toLowerCase() ?? "";
}
function matchesSemantics(ref: RenderedRef, fallback?: CaptureTargetDescriptor): boolean {
const role = normalized(fallback?.role);
const name = normalized(fallback?.name);
if (!role && !name) return false;
if (role && normalized(ref.role) !== role) return false;
if (name && normalized(ref.name) !== name) return false;
return true;
}
function descriptor(ref: RenderedRef): TargetDescriptorV3 {
return {
ref: ref.ref,
...(ref.role ? { role: ref.role } : {}),
...(ref.name ? { name: ref.name } : {}),
...(ref.ctx ? { ctx: ref.ctx } : {}),
};
}
export function matchObservationTarget(input: {
observation: RegisteredObservation;
hint?: TargetMatchHint;
fallback?: CaptureTargetDescriptor;
}): TargetDescriptorV3 {
if (input.hint?.frameId === null) return unmatchedTarget(input.fallback);
const frameId = input.hint?.frameId ?? input.observation.rootFrameId;
const geometry = input.hint?.geometry;
if (geometry) {
const matches = input.observation.index
.candidates(frameId, geometry.tag)
.filter(
(candidate) =>
candidate.ref &&
candidateMatches(candidate, geometry, input.hint?.geometrySpace ?? "top"),
);
if (matches.length === 1) return descriptor(matches[0]!.ref!);
const semanticMatches = matches.filter(
(candidate) => candidate.ref && matchesSemantics(candidate.ref, input.fallback),
);
if (semanticMatches.length === 1) return descriptor(semanticMatches[0]!.ref!);
return unmatchedTarget(input.fallback);
}
const semanticMatches = input.observation.index
.refs(frameId)
.filter((ref) => matchesSemantics(ref, input.fallback));
return semanticMatches.length === 1
? descriptor(semanticMatches[0]!)
: unmatchedTarget(input.fallback);
}
@@ -1,92 +0,0 @@
import {
type StepV3,
type StopReason,
TRACE_VERSION_V3,
type TraceStateV3,
type TraceV3,
VOM_FORMAT_VERSION,
} from "@/transport/types";
import { resolveDraftStartUrl } from "./draft-policy";
import type { RecordedStateEntry, RecordingStateRegistry } from "./state-registry";
import { reduceTraceStepsV3 } from "./trace-reducer-v3";
import { formatTraceStateBody } from "./trace-state-body";
import type { RecordingDraftStep, StepAnnotation } from "./types";
function publishedEntries(registry: RecordingStateRegistry, steps: StepV3[]): RecordedStateEntry[] {
const entries = registry.values();
if (steps.length === 0) return entries.slice(0, 1);
const referenced = new Set(steps.flatMap((step) => [step.state, step.result.state]));
return entries.filter((entry) => referenced.has(entry.id));
}
function remapDraftIds(draftIds: number[], stepIdByDraftId: Map<number, number>): number[] {
return [
...new Set(
draftIds.flatMap((id) => {
const stepId = stepIdByDraftId.get(id);
return stepId === undefined ? [] : [stepId];
}),
),
].sort((a, b) => a - b);
}
export function buildTraceV3(input: {
registry: RecordingStateRegistry;
drafts: RecordingDraftStep[];
annotations?: StepAnnotation[];
startedAt: string;
purpose?: string;
startUrl?: string;
stoppedBy: StopReason;
bskVersion: string;
redactValues?: boolean;
includeTabSwitches?: boolean;
}): TraceV3 {
const reduced = reduceTraceStepsV3(input.drafts, {
includeTabSwitches: input.includeTabSwitches,
redactValues: input.redactValues,
});
const entries = publishedEntries(input.registry, reduced.steps);
const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`]));
const annotationsByState = new Map<string, StepAnnotation[]>();
for (const annotation of input.annotations ?? []) {
const bucket = annotationsByState.get(annotation.stateId) ?? [];
bucket.push(annotation);
annotationsByState.set(annotation.stateId, bucket);
}
const steps = reduced.steps.map((step) => ({
...step,
state: publishedId.get(step.state) ?? step.state,
result: { state: publishedId.get(step.result.state) ?? step.result.state },
}));
const states: TraceStateV3[] = entries.map((entry) => {
const id = publishedId.get(entry.id) ?? entry.id;
return {
id,
url: entry.url,
...(entry.title ? { title: entry.title } : {}),
body: formatTraceStateBody({
stateId: id,
url: entry.url,
title: entry.title,
stepIds: remapDraftIds(entry.stepsHere, reduced.stepIdByDraftId),
vomText: entry.vomText,
annotations: annotationsByState.get(entry.id) ?? [],
stepIdByDraftId: reduced.stepIdByDraftId,
}),
...(entry.truncated ? { truncated: true } : {}),
};
});
return {
version: TRACE_VERSION_V3,
...(input.purpose ? { purpose: input.purpose } : {}),
recorded_at: new Date().toISOString(),
started_at: input.startedAt,
stopped_by: input.stoppedBy,
entry: { start_url: resolveDraftStartUrl(input.drafts, input.startUrl, states[0]?.url) },
recorder: { bsk: input.bskVersion, vom: VOM_FORMAT_VERSION },
states,
steps,
};
}
@@ -1,154 +0,0 @@
import type { NavigationCause, StepV3 } from "@/transport/types";
import { shouldIncludeDraft } from "./draft-policy";
import { hasRedirectQualifier } from "./navigation-policy";
import { unmatchedTarget } from "./target-matcher";
import type { RecordingDraftStep } from "./types";
interface CollapsedDraft {
draft: RecordingDraftStep;
draftIds: number[];
}
const TRANSITION_CAUSES: Record<string, NavigationCause> = {
typed: "user_typed",
generated: "user_typed",
keyword: "user_typed",
keyword_generated: "user_typed",
link: "link",
form_submit: "form_submit",
reload: "reload",
auto_bookmark: "browser",
start_page: "browser",
};
function isRedirect(step: Extract<RecordingDraftStep, { op: "navigate" }>): boolean {
return hasRedirectQualifier(step.transitionQualifiers);
}
function collapseRedirects(steps: RecordingDraftStep[]): CollapsedDraft[] {
const output: CollapsedDraft[] = [];
steps.forEach((step, index) => {
const previous = output[output.length - 1];
if (step.op === "navigate" && previous?.draft.op === "navigate" && isRedirect(step)) {
previous.draft = {
...previous.draft,
url: step.url,
postStateId: step.postStateId ?? previous.draft.postStateId,
};
previous.draftIds.push(index + 1);
return;
}
output.push({ draft: { ...step }, draftIds: [index + 1] });
});
return output;
}
function navigationCause(step: Extract<RecordingDraftStep, { op: "navigate" }>): NavigationCause {
if (step.cause) return step.cause;
const qualifiers = step.transitionQualifiers ?? [];
if (qualifiers.includes("forward_back")) return "history";
if (qualifiers.includes("from_address_bar")) return "user_typed";
return TRANSITION_CAUSES[step.transitionType ?? ""] ?? "browser";
}
function selection(values: string[], labels?: string[]): Array<{ value: string; label?: string }> {
return values.map((value, index) => ({
value,
...(labels?.[index] ? { label: labels[index] } : {}),
}));
}
interface ReduceDraftOptions {
includeTabSwitches: boolean;
redactValues: boolean;
}
function reduceDraft(
draft: RecordingDraftStep,
id: number,
options: ReduceDraftOptions,
): StepV3 | null {
if (!shouldIncludeDraft(draft)) return null;
if (draft.op === "switch_tab") {
if (!options.includeTabSwitches || !draft.preStateId || !draft.postStateId) return null;
return {
op: "switch_tab",
id,
state: draft.preStateId,
result: { state: draft.postStateId },
};
}
const state = draft.preStateId ?? draft.postStateId;
const resultState = draft.postStateId ?? draft.preStateId;
if (!state || !resultState) return null;
const common = { id, state, result: { state: resultState } };
switch (draft.op) {
case "navigate":
return { op: "navigate", ...common, to: draft.url, cause: navigationCause(draft) };
case "click":
return {
op: "click",
...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
};
case "hover":
return {
op: "hover",
...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
};
case "fill":
const fillIsRedacted = options.redactValues || draft.redacted === true;
return {
op: "fill",
...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
value: fillIsRedacted ? "***" : draft.value,
commit: draft.commit ?? "blur",
...(fillIsRedacted ? { redacted: true } : {}),
};
case "press":
return {
op: "press",
...common,
key: draft.key,
...(draft.captureTarget || draft.matchedTarget
? { target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget) }
: {}),
...(draft.modifiers?.length ? { modifiers: draft.modifiers } : {}),
};
case "select":
return {
op: "select",
...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
...(!options.redactValues ? { selection: selection(draft.values, draft.labels) } : {}),
};
case "scroll":
return { op: "scroll", ...common };
}
}
export interface ReducedTraceV3 {
steps: StepV3[];
stepIdByDraftId: Map<number, number>;
}
export function reduceTraceStepsV3(
steps: RecordingDraftStep[],
options: { includeTabSwitches?: boolean; redactValues?: boolean } = {},
): ReducedTraceV3 {
const output: StepV3[] = [];
const stepIdByDraftId = new Map<number, number>();
for (const { draft, draftIds } of collapseRedirects(steps)) {
const step = reduceDraft(draft, output.length + 1, {
includeTabSwitches: options.includeTabSwitches === true,
redactValues: options.redactValues === true,
});
if (!step) continue;
output.push(step);
for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id);
}
return { steps: output, stepIdByDraftId };
}
@@ -1,39 +0,0 @@
import type { StepAnnotation } from "./types";
function annotationText(annotation: StepAnnotation, stepId: number): string {
return ` ⟵ step ${stepId}: ${annotation.op}${annotation.detail ? `: ${annotation.detail}` : ""}`;
}
export function formatTraceStateBody(input: {
stateId: string;
url: string;
title?: string;
stepIds: number[];
vomText: string;
annotations: StepAnnotation[];
stepIdByDraftId: Map<number, number>;
}): string {
const lines = ["# bsk-observation 1", `state: ${JSON.stringify(input.stateId)}`];
lines.push(`url: ${JSON.stringify(input.url)}`);
if (input.title) lines.push(`title: ${JSON.stringify(input.title)}`);
if (input.stepIds.length > 0) lines.push(`steps_here: [${input.stepIds.join(", ")}]`);
lines.push("---");
const byLine = new Map<number, Array<{ annotation: StepAnnotation; stepId: number }>>();
for (const annotation of input.annotations) {
const stepId = input.stepIdByDraftId.get(annotation.draftId);
if (stepId === undefined) continue;
const bucket = byLine.get(annotation.line) ?? [];
bucket.push({ annotation, stepId });
byLine.set(annotation.line, bucket);
}
input.vomText.split("\n").forEach((bodyLine, lineIndex) => {
let line = bodyLine;
for (const item of byLine.get(lineIndex) ?? []) {
line += annotationText(item.annotation, item.stepId);
}
lines.push(line);
});
return `${lines.join("\n")}\n`;
}
-85
View File
@@ -1,85 +0,0 @@
import type { CaptureTargetDescriptor } from "@/lib/describe-target";
import type {
FillCommit,
KeyModifier,
NavigationCause,
StepV3,
TargetDescriptorV3,
} from "@/transport/types";
export interface TargetGeometry {
/** Top-level viewport-relative CSS pixels, as defined by the geometry module. */
rect: { x: number; y: number; w: number; h: number };
tag: string;
}
export interface TargetMatchHint {
geometry?: TargetGeometry;
/** Missing means top frame; null means the source Document could not be resolved. */
frameId?: string | null;
geometrySpace?: "top" | "local";
}
export interface StepAnnotation {
draftId: number;
op: StepV3["op"];
line: number;
stateId: string;
detail?: string;
}
interface DraftStateLink {
pageUrl?: string;
preStateId?: string;
postStateId?: string;
}
interface DraftTarget {
captureTarget?: CaptureTargetDescriptor;
targetHint?: TargetMatchHint;
matchedTarget?: TargetDescriptorV3;
}
interface DraftNavigationEffect {
navigatedTo?: string;
}
export type RecordingDraftStep =
| ({ op: "click" } & DraftStateLink & DraftTarget & DraftNavigationEffect)
| ({ op: "hover" } & DraftStateLink & DraftTarget)
| ({
op: "fill";
value: string;
commit?: FillCommit;
redacted?: boolean;
} & DraftStateLink &
DraftTarget &
DraftNavigationEffect)
| ({
op: "press";
key: string;
modifiers?: KeyModifier[];
} & DraftStateLink &
DraftTarget &
DraftNavigationEffect)
| ({
op: "select";
values: string[];
labels?: string[];
} & DraftStateLink &
DraftTarget &
DraftNavigationEffect)
| ({ op: "scroll" } & DraftStateLink)
| ({ op: "switch_tab" } & DraftStateLink)
| ({
op: "navigate";
url: string;
cause?: NavigationCause;
transitionType?: string;
transitionQualifiers?: string[];
} & DraftStateLink);
export type TargetedRecordingDraft = Extract<
RecordingDraftStep,
{ op: "click" | "hover" | "fill" | "press" | "select" }
>;
@@ -1,10 +1,30 @@
import type { PageRefV2, SelectedOptionV2, StepV2, TraceV2 } from "@/transport/types";
import { resolveDraftStartUrl, shouldIncludeDraft } from "./draft-policy";
import type { RecordingDraftStep } from "./types";
import type { DraftTraceStep, PageRef, SelectedOption, Step } from "@/transport/types";
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
export function shouldRecordPress(
key: string,
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
): boolean {
if (MODIFIER_ONLY_KEYS.has(key)) return false;
const mods = modifiers ?? [];
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
if (key === "Enter" || key === "Escape") return true;
// Drop bare character typing — FillSession already records the value.
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
return false;
}
function shouldIncludeDraft(step: DraftTraceStep): boolean {
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
return true;
}
/** Collapse consecutive navigations to the last hop. */
function collapseNavigations(steps: RecordingDraftStep[]): RecordingDraftStep[] {
const out: RecordingDraftStep[] = [];
function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
const out: DraftTraceStep[] = [];
for (const step of steps) {
const prev = out[out.length - 1];
if (step.op === "navigate" && prev?.op === "navigate") {
@@ -16,7 +36,7 @@ function collapseNavigations(steps: RecordingDraftStep[]): RecordingDraftStep[]
return out;
}
function collectUrls(steps: RecordingDraftStep[], startUrl?: string): string[] {
function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
const urls: string[] = [];
if (startUrl) urls.push(startUrl);
for (const step of steps) {
@@ -24,8 +44,8 @@ function collectUrls(steps: RecordingDraftStep[], startUrl?: string): string[] {
urls.push(step.url);
continue;
}
if (step.pageUrl) urls.push(step.pageUrl);
if ("navigatedTo" in step && step.navigatedTo) urls.push(step.navigatedTo);
if ("page_url" in step && step.page_url) urls.push(step.page_url);
if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to);
}
const seen = new Set<string>();
const unique: string[] = [];
@@ -38,9 +58,9 @@ function collectUrls(steps: RecordingDraftStep[], startUrl?: string): string[] {
}
function buildPageRegistry(
steps: RecordingDraftStep[],
steps: DraftTraceStep[],
startUrl?: string,
): { pages: PageRefV2[]; urlToId: Map<string, string> } {
): { pages: PageRef[]; urlToId: Map<string, string> } {
const urls = collectUrls(steps, startUrl);
const urlToId = new Map<string, string>();
const pages = urls.map((url, index) => {
@@ -61,28 +81,28 @@ function pageIdFor(
return urlToId.values().next().value ?? "p1";
}
function pageUrlForDraft(step: RecordingDraftStep, fallbackUrl?: string): string | undefined {
if (step.op === "navigate") return step.pageUrl ?? step.url;
if (step.pageUrl) return step.pageUrl;
function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined {
if (step.op === "navigate") return step.page_url ?? step.url;
if ("page_url" in step && step.page_url) return step.page_url;
return fallbackUrl;
}
function effectForNavigation(
navigatedTo: string | undefined,
urlToId: Map<string, string>,
): StepV2["effect"] {
): Step["effect"] {
if (!navigatedTo) return undefined;
const pageId = urlToId.get(navigatedTo);
if (!pageId) return undefined;
return { navigated_to: pageId };
}
function withEffect(step: StepV2, effect: StepV2["effect"]): StepV2 {
function withEffect(step: Step, effect: Step["effect"]): Step {
if (!effect) return step;
return { ...step, effect };
}
function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] {
function toSelection(values: string[], labels?: string[]): SelectedOption[] {
return values.map((value, index) => ({
value,
...(labels?.[index] ? { label: labels[index] } : {}),
@@ -90,11 +110,11 @@ function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] {
}
function toV2Step(
step: RecordingDraftStep,
step: DraftTraceStep,
id: number,
urlToId: Map<string, string>,
fallbackUrl?: string,
): StepV2 | null {
): Step | null {
if (!shouldIncludeDraft(step)) return null;
const pageUrl = pageUrlForDraft(step, fallbackUrl);
@@ -109,23 +129,28 @@ function toV2Step(
to: step.url,
};
case "click":
if (!step.captureTarget) return null;
return withEffect(
{
op: "click",
id,
page,
target: step.captureTarget,
target: step.target,
},
effectForNavigation(step.navigatedTo, urlToId),
effectForNavigation(step.navigated_to, urlToId),
);
case "hover":
return {
op: "hover",
id,
page,
target: step.target,
};
case "fill":
if (!step.captureTarget) return null;
return {
op: "fill",
id,
page,
target: step.captureTarget,
target: step.target,
value: step.value,
...(step.redacted ? { redacted: true } : {}),
};
@@ -136,51 +161,44 @@ function toV2Step(
id,
page,
key: step.key,
...(step.captureTarget ? { target: step.captureTarget } : {}),
...(step.target ? { target: step.target } : {}),
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
},
effectForNavigation(step.navigatedTo, urlToId),
effectForNavigation(step.navigated_to, urlToId),
);
case "select":
if (!step.captureTarget) return null;
return withEffect(
{
op: "select",
id,
page,
target: step.captureTarget,
target: step.target,
selection: toSelection(step.values, step.labels),
},
effectForNavigation(step.navigatedTo, urlToId),
effectForNavigation(step.navigated_to, urlToId),
);
// `hover` only exists in Trace v3. Peers that negotiate v2 predate the
// step variant and fail to decode the whole result if we emit it.
case "hover":
case "scroll":
case "switch_tab":
return null;
}
}
interface ReducedTrace {
pages: PageRefV2[];
steps: StepV2[];
export interface ReducedTrace {
pages: PageRef[];
steps: Step[];
}
/**
* Compile capture drafts into record-only trace v2 steps.
* Variable inputs are NOT classified here — executing agents infer that at run time.
*/
function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string): ReducedTrace {
export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): ReducedTrace {
const collapsed = collapseNavigations(steps);
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
const out: StepV2[] = [];
const out: Step[] = [];
let id = 1;
let lastUrl = startUrl;
for (const draft of collapsed) {
if (draft.op === "navigate") lastUrl = draft.url;
else if ("navigatedTo" in draft && draft.navigatedTo) lastUrl = draft.navigatedTo;
else if (draft.pageUrl) lastUrl = draft.pageUrl;
else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to;
else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url;
const step = toV2Step(draft, id, urlToId, lastUrl);
if (!step) continue;
out.push(step);
@@ -189,27 +207,18 @@ function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string): Reduc
return { pages, steps: out };
}
function resolveTraceStartUrl(
drafts: RecordingDraftStep[],
export function resolveTraceStartUrl(
drafts: DraftTraceStep[],
startUrl?: string,
pages?: PageRefV2[],
pages?: PageRef[],
): string {
return resolveDraftStartUrl(drafts, startUrl, pages?.[0]?.url);
}
export function buildTraceV2(input: {
steps: RecordingDraftStep[];
startedAt: string;
startUrl?: string;
purpose?: string;
}): TraceV2 {
const { pages, steps } = reduceTraceSteps(input.steps, input.startUrl);
return {
recorded_at: new Date().toISOString(),
started_at: input.startedAt,
...(input.purpose ? { purpose: input.purpose } : {}),
entry: { start_url: resolveTraceStartUrl(input.steps, input.startUrl, pages) },
pages,
steps,
};
if (startUrl) return startUrl;
const navigate = drafts.find((step): step is Extract<DraftTraceStep, { op: "navigate" }> => {
return step.op === "navigate";
});
if (navigate) return navigate.url;
for (const draft of drafts) {
if ("page_url" in draft && draft.page_url) return draft.page_url;
}
return pages?.[0]?.url ?? "about:blank";
}
@@ -1,22 +0,0 @@
export const RECORD_DOCUMENT_ATTRIBUTE = "data-bsk-record-document";
const RECORDING_DOCUMENT_ID_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
const RECORDING_DOCUMENT_MARKER_PREFIX = "bsk:";
export function recordingDocumentMarkerValue(identity: string): string {
if (!RECORDING_DOCUMENT_ID_PATTERN.test(identity)) {
throw new TypeError("recording Document identity must be a random UUID");
}
return `${RECORDING_DOCUMENT_MARKER_PREFIX}${identity}`;
}
/** Read BrowserSkill's opaque per-Document recording identity from captured attributes. */
export function readRecordingDocumentIdentity(
attrs: Readonly<Record<string, string>>,
): string | undefined {
const marker = attrs[RECORD_DOCUMENT_ATTRIBUTE];
if (!marker?.startsWith(RECORDING_DOCUMENT_MARKER_PREFIX)) return undefined;
const identity = marker.slice(RECORDING_DOCUMENT_MARKER_PREFIX.length);
return identity && RECORDING_DOCUMENT_ID_PATTERN.test(identity) ? identity : undefined;
}
@@ -8,7 +8,6 @@ import type {
RequestFrame,
} from "@/transport/types";
import { ToolDispatcher } from "../dispatcher";
import { resetBrowserObservationForTests, setBrowserObservationAttachForTests } from "../record";
type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>;
@@ -45,72 +44,9 @@ function makeRequest(method: string, params: unknown): RequestFrame {
describe("ToolDispatcher", () => {
afterEach(() => {
resetBrowserObservationForTests();
vi.unstubAllGlobals();
});
it("uses the configured recording runtime for start and stop", async () => {
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
},
});
await sessions.start("aa11");
const tab = {
id: 7,
windowId: 4242,
active: true,
status: "complete",
url: "https://example.com/start",
} as chrome.tabs.Tab;
const frameCoordinator = {
begin: vi.fn(),
armTab: vi.fn(async () => true),
sourceFor: vi.fn(() => null),
stop: vi.fn(async () => true),
cancel: vi.fn(),
};
const sendToTab = vi.fn(async () => ({ ok: true }));
setBrowserObservationAttachForTests(
() => () => {},
() => () => {},
);
const dispatcher = new ToolDispatcher({
transport,
sessions,
recording: {
tabsApi: {
get: vi.fn(async () => tab),
query: vi.fn(async () => [tab]),
},
frameCoordinator,
sendToTab,
},
});
dispatcher.start();
deliver(
makeRequest("tool.record_start", {
session_id: "aa11",
url: "https://example.com/start",
}),
);
await vi.waitFor(() => expect(sent).toHaveLength(1));
expect(sent[0]).toMatchObject({ result: { tab_id: 7, recording: true } });
expect(frameCoordinator.begin).toHaveBeenCalledOnce();
expect(frameCoordinator.armTab).toHaveBeenCalledWith(expect.any(String), 7);
deliver({ id: "r-2", method: "tool.record_stop", params: { session_id: "aa11" } });
await vi.waitFor(() => expect(sent).toHaveLength(2));
expect(frameCoordinator.stop).toHaveBeenCalledOnce();
expect(sent[1]).toMatchObject({ id: "r-2", result: { trace: { steps: [] } } });
});
it("routes tool.session_start to the SessionManager and replies with the window id", async () => {
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
File diff suppressed because it is too large Load Diff
+37 -34
View File
@@ -51,12 +51,7 @@ import {
handleScreenshot,
handleSnapshot,
} from "./observation";
import {
handleRecordAwait,
handleRecordStart,
handleRecordStop,
type RecordRuntimeDeps,
} from "./record";
import { handleRecordAwait, handleRecordStart, handleRecordStop } from "./record";
import {
handleSessionStart,
handleSessionStop,
@@ -104,7 +99,6 @@ export interface DispatcherDeps {
transport: Transport;
sessions: SessionManager;
cdp?: DispatcherCdpRunner;
recording?: RecordRuntimeDeps;
/**
* Invoked whenever a dispatched RPC may have changed the live
* session set (currently `tool.session_start` and
@@ -140,7 +134,6 @@ export class ToolDispatcher {
private readonly transport: Transport;
private readonly sessions: SessionManager;
private readonly cdp?: DispatcherCdpRunner;
private readonly recording?: RecordRuntimeDeps;
private readonly onSessionsChanged?: () => void;
private readonly onBrowserControlResumed?: (sessionId: string) => void;
private readonly approveBorrow?: BorrowConfirmationApprover;
@@ -160,7 +153,6 @@ export class ToolDispatcher {
this.transport = deps.transport;
this.sessions = deps.sessions;
this.cdp = deps.cdp;
this.recording = deps.recording;
this.onSessionsChanged = deps.onSessionsChanged;
this.onBrowserControlResumed = deps.onBrowserControlResumed;
this.approveBorrow = deps.approveBorrow;
@@ -539,26 +531,44 @@ export class ToolDispatcher {
signal,
});
case "tool.record_start":
return this.recording
? handleRecordStart(this.sessions, req.params as RecordStartParams, {
...this.recording,
signal,
})
: recordingRuntimeUnavailable();
return handleRecordStart(this.sessions, req.params as RecordStartParams, {
tabsApi: chromeTabsApi,
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
bypassOverlay: async (tabId, enabled) => {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
...(this.cdp ? { cdp: this.cdp } : {}),
signal,
});
case "tool.record_stop":
return this.recording
? handleRecordStop(this.sessions, req.params as RecordStopParams, {
...this.recording,
signal,
})
: recordingRuntimeUnavailable();
return handleRecordStop(this.sessions, req.params as RecordStopParams, {
tabsApi: chromeTabsApi,
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
bypassOverlay: async (tabId, enabled) => {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
signal,
});
case "tool.record_await":
return this.recording
? handleRecordAwait(this.sessions, req.params as RecordAwaitParams, {
...this.recording,
signal,
})
: recordingRuntimeUnavailable();
return handleRecordAwait(this.sessions, req.params as RecordAwaitParams, {
tabsApi: chromeTabsApi,
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
signal,
});
default:
return {
code: "unknown_method",
@@ -694,13 +704,6 @@ function isRpcError(v: unknown): v is RpcError {
);
}
function recordingRuntimeUnavailable(): RpcError {
return {
code: "protocol_error",
message: "recording runtime is unavailable",
};
}
function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
switch (req.method) {
case "tool.tab_create":
+73 -390
View File
@@ -15,37 +15,22 @@ import {
type RecordQueryResponse,
type RecordStartAck,
type RecordStartMessage,
type RecordStepAck,
type RecordStopMessage,
} from "@/lib/record-bridge";
import {
type RecordFrameCoordinator,
type RecordingCaptureScope,
recordFrameCoordinator,
} from "@/lib/recording/frame-coordinator";
import { RecordingObservationRuntime } from "@/lib/recording/recording-runtime";
import {
appendRecordedPayload,
observeRecordedNavigation,
type RecordingStepBuffer,
} from "@/lib/recording/step-buffer";
import { RecordingTabCoordinator, type TabActivation } from "@/lib/recording/tab-coordinator";
import { buildTraceV2 } from "@/lib/recording/trace-reducer-v2";
import type { RecordingDraftStep } from "@/lib/recording/types";
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer";
import { reduceTraceSteps, resolveTraceStartUrl } from "@/lib/trace-reducer";
import type { SessionManager } from "@/session-manager/manager";
import { EXTENSION_VERSION } from "@/transport/handshake";
import type {
DraftTraceStep,
RecordAwaitParams,
RecordAwaitResult,
RecordedTrace,
RecordStartParams,
RecordStartResult,
RecordStopParams,
RecordStopResult,
RpcError,
StopReason,
Trace,
} from "@/transport/types";
import { TRACE_VERSION_V3 } from "@/transport/types";
import { handleNavigate } from "./navigation";
import {
type CdpRunner,
@@ -58,46 +43,21 @@ import {
interface ActiveRecording {
requestId: string;
tabs: RecordingTabCoordinator;
tabId: number;
agentWindowId: number;
startUrl?: string;
purpose?: string;
steps: RecordingDraftStep[];
steps: DraftTraceStep[];
startedAt: string;
startedAtMs: number;
traceVersion: 2 | 3;
supportsTabSwitchSteps: boolean;
finishPromise: Promise<RecordedTrace>;
resolveFinish: (trace: RecordedTrace) => void;
finishPromise: Promise<Trace>;
resolveFinish: (trace: Trace) => void;
rejectFinish: (err: Error) => void;
settled: boolean;
finishAttempt: Promise<RecordedTrace | null> | null;
observation: RecordingObservationRuntime | null;
stoppedBy: StopReason;
/** Navigation callbacks tracked from event receipt through action enqueue. */
navigationCallbacks: Set<Promise<void>>;
/** Synchronous intake gate closed only after finish drains to stability. */
acceptingNavigation: boolean;
/**
* Serializes step appends with navigation observation so a click is always
* in `steps` before a same-turn `webNavigation` tries to annotate it.
*/
actionQueue: Promise<void>;
/** Last accepted sequence for each content-script document producer. */
lastStepSequenceByProducer: Map<string, number>;
}
function enqueueRecordingAction(
recording: ActiveRecording,
task: () => Promise<void>,
): Promise<void> {
const queued = recording.actionQueue.then(task, task);
recording.actionQueue = queued.catch(() => {});
return queued;
}
function isRecordingFinishing(recording: ActiveRecording): boolean {
return recording.settled || recording.finishAttempt !== null;
finishing: boolean;
currentUrl?: string;
pendingNavigation: boolean;
pendingNavigationDeadline?: number;
}
const recordings = new Map<string, ActiveRecording>();
@@ -118,9 +78,6 @@ function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
/** Recording producer version mirrored into trace.recorder.bsk. */
export const BSK_TRACE_VERSION = EXTENSION_VERSION;
/** Injectable http(s) landing page when `tool.record_start` omits `url`. */
export const RECORD_DEFAULT_START_URL = "https://example.com/";
@@ -180,11 +137,9 @@ async function sendRecordStartWithAck(
tabId: number,
msg: RecordStartMessage,
sendToTab: RecordDeps["sendToTab"],
cancelled?: () => boolean,
): Promise<void> {
let lastError: unknown;
for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) {
if (cancelled?.()) throw new Error("record start cancelled");
try {
const response = await sendToTab(tabId, msg);
if (isRecordStartAck(response)) return;
@@ -192,7 +147,6 @@ async function sendRecordStartWithAck(
} catch (err) {
lastError = err;
}
if (cancelled?.()) throw new Error("record start cancelled");
if (attempt + 1 < RECORD_START_RETRIES) {
await sleep(RECORD_START_RETRY_DELAY_MS);
}
@@ -200,83 +154,19 @@ async function sendRecordStartWithAck(
throw lastError ?? new Error("failed to start recording in content script");
}
function negotiatedTraceVersion(params: RecordStartParams): 2 | 3 | RpcError {
if (params.trace_version === undefined) return 2;
if (params.trace_version === TRACE_VERSION_V3) return 3;
function buildTrace(recording: ActiveRecording): Trace {
const { pages, steps } = reduceTraceSteps(recording.steps, recording.startUrl);
const startUrl = resolveTraceStartUrl(recording.steps, recording.startUrl, pages);
return {
code: "invalid_params",
message: `unsupported trace_version ${params.trace_version}; supported values are omitted (v2) or ${TRACE_VERSION_V3} (v3)`,
};
}
function buildTrace(recording: ActiveRecording): RecordedTrace {
if (recording.traceVersion === 3 && recording.observation) {
return recording.observation.buildTrace({
drafts: recording.steps,
startedAt: recording.startedAt,
purpose: recording.purpose,
startUrl: recording.startUrl,
stoppedBy: recording.stoppedBy,
bskVersion: BSK_TRACE_VERSION,
includeTabSwitches: recording.supportsTabSwitchSteps,
});
}
if (recording.traceVersion === 3) {
throw new Error("trace v3 observation runtime is unavailable");
}
return buildTraceV2({
steps: recording.steps,
startedAt: recording.startedAt,
...(recording.startUrl ? { startUrl: recording.startUrl } : {}),
...(recording.purpose ? { purpose: recording.purpose } : {}),
});
}
function stepBufferFor(
recording: ActiveRecording,
tabId: number,
fallbackUrl?: string,
): RecordingStepBuffer {
return {
steps: recording.steps,
navigation: recording.tabs.navigation(tabId, fallbackUrl),
recorded_at: new Date().toISOString(),
started_at: recording.startedAt,
entry: { start_url: startUrl },
pages,
steps,
};
}
async function activateRecordingTab(
recording: ActiveRecording,
targetTabId: number,
): Promise<void> {
const previousTabId = recording.tabs.currentTabId;
if (targetTabId === previousTabId) return;
const transition = recording.observation
? await recording.observation.captureTabTransition(previousTabId, targetTabId)
: {};
const { targetUrl, ...stateLinks } = transition;
const draft: Extract<RecordingDraftStep, { op: "switch_tab" }> = {
op: "switch_tab",
...stateLinks,
};
recording.steps.push(draft);
recording.observation?.bindTabTransition(draft, recording.steps.length);
recording.tabs.commit(targetTabId, targetUrl);
}
async function processRecordedStep(
recording: ActiveRecording,
draftIndex: number,
tabId: number,
producerId?: string,
): Promise<void> {
if (!recording.observation) return;
try {
await recording.observation.processDraft(tabId, recording.steps, draftIndex, producerId);
} catch (err) {
console.warn(`[bsk record] observation failed for step ${draftIndex + 1}`, err);
}
}
export interface RecordDeps {
tabsApi: ChromeTabsApi;
sendToTab(
@@ -284,25 +174,16 @@ export interface RecordDeps {
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
): Promise<unknown>;
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
frameCoordinator?: Pick<
RecordFrameCoordinator,
"begin" | "armTab" | "sourceFor" | "stop" | "cancel"
>;
cdp?: CdpRunner;
signal?: AbortSignal;
}
export type RecordRuntimeDeps = Omit<RecordDeps, "frameCoordinator" | "signal"> & {
frameCoordinator: NonNullable<RecordDeps["frameCoordinator"]>;
};
let defaultDeps: RecordDeps | null = null;
function getDefaultDeps(): RecordDeps {
if (!defaultDeps) {
defaultDeps = {
tabsApi: chromeTabsApi,
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
frameCoordinator: recordFrameCoordinator,
};
}
return defaultDeps;
@@ -360,58 +241,16 @@ export function releaseBrowserObservationListenersIfIdle(): void {
detachBrowserObservation = null;
}
export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): () => void {
export function attachRecordStepListener(): () => void {
const listener = (
message: unknown,
sender: chrome.runtime.MessageSender,
sendResponse: (response: RecordStepAck) => void,
_sender: chrome.runtime.MessageSender,
_sendResponse: () => void,
) => {
if (!isRecordStepMessage(message)) return false;
for (const recording of recordings.values()) {
if (recording.requestId !== message.requestId) continue;
const source: RecordingCaptureScope | null | undefined = deps.frameCoordinator
? deps.frameCoordinator.sourceFor(message.requestId, message.producerId, sender)
: undefined;
if (deps.frameCoordinator && !source) return false;
const sourceTabId = source?.tabId ?? sender.tab?.id ?? recording.tabs.currentTabId;
const sourceWasActive = sender.tab?.active ?? sourceTabId === recording.tabs.activeTabId;
const producerKey = source
? `${source.tabId}:${source.documentId}:${source.producerId}`
: `${sourceTabId}:${message.producerId}`;
const expectedSequence = (recording.lastStepSequenceByProducer.get(producerKey) ?? 0) + 1;
if (message.sequence < expectedSequence) {
sendResponse({ ok: true, sequence: message.sequence });
return false;
}
if (message.sequence > expectedSequence) {
sendResponse({
ok: false,
expectedSequence,
error: `out-of-order recorded step ${message.sequence}`,
});
return false;
}
recording.lastStepSequenceByProducer.set(producerKey, message.sequence);
enqueueRecordingAction(recording, async () => {
if (sourceTabId !== recording.tabs.currentTabId) {
if (!sourceWasActive) return;
if (sourceTabId !== recording.tabs.activeTabId)
recording.tabs.noteActivation(sourceTabId);
await activateRecordingTab(recording, sourceTabId);
}
await recording.observation?.flushRedirects(sourceTabId);
const targetHint = message.step.geometry ? { geometry: message.step.geometry } : undefined;
const draftIndex = appendRecordedPayload(
stepBufferFor(recording, sourceTabId, message.step.page_url),
message.step,
targetHint,
);
if (draftIndex !== null) {
await processRecordedStep(recording, draftIndex, sourceTabId, source?.producerId);
}
});
sendResponse({ ok: true, sequence: message.sequence });
appendRecordedPayload(recording, message.step);
return false;
}
return false;
@@ -438,7 +277,7 @@ export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()):
function findRecordingByTabId(tabId: number): ActiveRecording | null {
for (const recording of recordings.values()) {
if (recording.tabs.currentTabId === tabId && !recording.settled) return recording;
if (recording.tabId === tabId && !recording.settled) return recording;
}
return null;
}
@@ -475,7 +314,7 @@ async function clearRearmTimersForRecording(
recording: ActiveRecording,
deps: RecordDeps,
): Promise<void> {
clearRearmTimer(recording.tabs.currentTabId);
clearRearmTimer(recording.tabId);
try {
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
for (const tab of tabs) {
@@ -490,16 +329,13 @@ async function stopRecordingOnAllAgentTabs(
recording: ActiveRecording,
deps: RecordDeps,
): Promise<void> {
if (deps.frameCoordinator && !(await deps.frameCoordinator.stop(recording.requestId))) {
throw new Error("failed to flush one or more recording documents");
}
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
let tabIds = [recording.tabs.currentTabId];
let tabIds = [recording.tabId];
try {
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
tabIds = [
...new Set([
recording.tabs.currentTabId,
recording.tabId,
...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])),
]),
];
@@ -510,11 +346,11 @@ async function stopRecordingOnAllAgentTabs(
for (const tabId of tabIds) {
try {
const response = await deps.sendToTab(tabId, stopMsg);
if (tabId === recording.tabs.currentTabId && !isRecordStartAck(response)) {
if (tabId === recording.tabId && !isRecordStartAck(response)) {
throw new Error("content script did not confirm recorded steps");
}
} catch {
if (tabId === recording.tabs.currentTabId) {
if (tabId === recording.tabId) {
throw new Error("failed to flush recorded steps");
}
}
@@ -532,39 +368,22 @@ async function rearmRecording(
recording: ActiveRecording,
targetTabId: number,
deps: RecordDeps,
activation?: TabActivation,
): Promise<boolean> {
// Do NOT toggle automation-bypass here: each retry used to increment the
// content-script counter, and a single stop decrement left the ControlOverlay
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
// already hides the control chrome while activeRecord is set.
const isFinishing = () => isRecordingFinishing(recording);
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
if (isFinishing()) return false;
const startMsg: RecordStartMessage = {
type: RECORD_START,
requestId: recording.requestId,
startedAtMs: recording.startedAtMs,
};
try {
if (deps.frameCoordinator) {
const frameStarted = await deps.frameCoordinator.armTab(recording.requestId, targetTabId);
if (!frameStarted) throw new Error("recording document did not start");
}
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab, isFinishing);
if (isFinishing()) return false;
if (activation) {
if (!recording.tabs.isLatest(activation)) return true;
await enqueueRecordingAction(recording, async () => {
if (isFinishing() || !recording.tabs.isLatest(activation)) {
return;
}
await activateRecordingTab(recording, targetTabId);
});
}
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
recording.tabId = targetTabId;
return true;
} catch {
if (isFinishing()) return false;
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
await sleep(RECORD_REARM_RETRY_DELAY_MS);
}
@@ -573,7 +392,7 @@ async function rearmRecording(
return false;
}
function scheduleRearmForTab(tabId: number, deps: RecordDeps, activation?: TabActivation): void {
function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
const existing = rearmTimers.get(tabId);
if (existing) clearTimeout(existing);
rearmTimers.set(
@@ -582,7 +401,7 @@ function scheduleRearmForTab(tabId: number, deps: RecordDeps, activation?: TabAc
rearmTimers.delete(tabId);
void (async () => {
const current = await findRecordingForTab(tabId, deps);
if (current) await rearmRecording(current, tabId, deps, activation);
if (current) await rearmRecording(current, tabId, deps);
})();
}, RECORD_REARM_DEBOUNCE_MS),
);
@@ -594,20 +413,14 @@ export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): ()
const windowId = tab.windowId;
if (tabId === undefined || windowId === undefined) return;
for (const recording of recordings.values()) {
if (isRecordingFinishing(recording) || recording.agentWindowId !== windowId) continue;
if (recording.settled || recording.agentWindowId !== windowId) continue;
scheduleRearmForTab(tabId, deps);
return;
}
};
const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => {
for (const recording of recordings.values()) {
if (isRecordingFinishing(recording) || recording.agentWindowId !== activeInfo.windowId)
continue;
const activation = recording.tabs.noteActivation(activeInfo.tabId);
scheduleRearmForTab(activeInfo.tabId, deps, activation);
return;
}
scheduleRearmForTab(activeInfo.tabId, deps);
};
chrome.tabs.onCreated.addListener(onCreated);
@@ -619,47 +432,13 @@ export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): ()
}
export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps()): () => void {
const observeMainFrame = (
tabId: number,
url?: string,
causedByAction?: boolean,
transitionType?: string,
transitionQualifiers?: string[],
) => {
if (!url) return;
const candidates = [...recordings.values()].filter(
(recording) =>
!recording.settled && recording.acceptingNavigation && recording.tabs.activeTabId === tabId,
);
for (const recording of candidates) {
const queued = enqueueRecordingAction(recording, async () => {
if (tabId !== recording.tabs.currentTabId) {
await activateRecordingTab(recording, tabId);
}
const result = observeRecordedNavigation(
stepBufferFor(recording, tabId, url),
url,
causedByAction,
transitionType,
transitionQualifiers,
);
if (result.kind === "coalesce_redirect") {
recording.observation?.scheduleRedirect(tabId, recording.steps, result.url);
return;
}
if (result.kind === "noop") return;
recording.observation?.clearRedirect(tabId);
if (result.kind === "appended") {
await processRecordedStep(recording, result.index, tabId);
} else {
recording.observation?.scheduleSettle(tabId, recording.steps, result.index);
}
});
const tracked = queued.catch(() => {});
recording.navigationCallbacks.add(tracked);
void tracked.finally(() => recording.navigationCallbacks.delete(tracked));
}
const observeMainFrame = (tabId: number, url?: string, causedByAction?: boolean) => {
void (async () => {
const recording = await findRecordingForTab(tabId, deps);
if (recording && url) {
observeRecordedNavigation(recording, url, causedByAction);
}
})();
};
const onMainFrameComplete = (tabId: number, url?: string) => {
void (async () => {
@@ -679,13 +458,18 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
details: chrome.webNavigation.WebNavigationTransitionCallbackDetails,
) => {
if (details.frameId !== 0) return;
observeMainFrame(
details.tabId,
details.url,
undefined,
details.transitionType,
details.transitionQualifiers,
);
const causedByAction =
details.transitionType === "link" || details.transitionType === "form_submit"
? true
: details.transitionType === "typed" ||
details.transitionType === "auto_bookmark" ||
details.transitionType === "generated" ||
details.transitionType === "keyword" ||
details.transitionType === "keyword_generated" ||
details.transitionType === "reload"
? false
: undefined;
observeMainFrame(details.tabId, details.url, causedByAction);
};
chrome.webNavigation.onCompleted.addListener(completedListener);
chrome.webNavigation.onCommitted?.addListener(committedListener);
@@ -703,26 +487,6 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
return () => chrome.tabs.onUpdated.removeListener(listener);
}
const MAX_FINISH_DRAIN_ROUNDS = 10;
async function drainRecordingToStability(recording: ActiveRecording): Promise<boolean> {
for (let round = 0; round < MAX_FINISH_DRAIN_ROUNDS; round += 1) {
await Promise.all([...recording.navigationCallbacks]);
const actionTail = recording.actionQueue;
await actionTail;
await recording.observation?.flush();
if (recording.navigationCallbacks.size === 0 && recording.actionQueue === actionTail) {
// No event or promise continuation can interleave with this synchronous
// check-and-close, so work accepted before the cutoff is fully drained.
recording.acceptingNavigation = false;
return true;
}
}
console.warn("[bsk record] navigation/action queues did not stabilize at stop");
return false;
}
export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()): () => void {
const listener = (
message: unknown,
@@ -741,8 +505,7 @@ export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()):
sendResponse({ active: false });
return;
}
const activation = sender.tab?.active ? recording.tabs.noteActivation(tabId) : undefined;
await rearmRecording(recording, tabId, deps, activation);
await rearmRecording(recording, tabId, deps);
sendResponse({
active: true,
requestId: recording.requestId,
@@ -764,50 +527,23 @@ async function finishRecordingByRequest(
if (recording.requestId !== requestId || recording.settled) continue;
const match = await findRecordingForTab(tabId, deps);
if (match !== recording) continue;
await finishRecording(sessionId, deps, "user_finish");
await finishRecording(sessionId, deps);
return;
}
}
async function finishRecording(
sessionId: string,
deps: RecordDeps,
stoppedBy: StopReason,
): Promise<RecordedTrace | null> {
async function finishRecording(sessionId: string, deps: RecordDeps): Promise<Trace | null> {
const recording = recordings.get(sessionId);
if (!recording || recording.settled) return null;
if (recording.finishAttempt) return recording.finishAttempt;
recording.stoppedBy = stoppedBy;
if (!recording || recording.settled || recording.finishing) return null;
recording.finishing = true;
const attempt = finishRecordingAttempt(sessionId, recording, deps);
recording.finishAttempt = attempt;
try {
return await attempt;
} finally {
if (!recording.settled) {
recording.finishAttempt = null;
}
}
}
async function finishRecordingAttempt(
sessionId: string,
recording: ActiveRecording,
deps: RecordDeps,
): Promise<RecordedTrace | null> {
await clearRearmTimersForRecording(recording, deps);
try {
// Disposing content capture may commit a final dirty fill. Flush content
// first so all resulting RECORD_STEP messages enter actionQueue before it
// and the observation queues are drained.
await stopRecordingOnAllAgentTabs(recording, deps);
} catch {
recording.finishing = false;
return null;
}
if (!(await drainRecordingToStability(recording))) {
return null;
}
await recording.observation?.settleTrailing(recording.tabs.currentTabId, recording.steps);
recording.settled = true;
recordings.delete(sessionId);
@@ -822,15 +558,6 @@ export async function handleRecordStart(
params: RecordStartParams,
deps: RecordDeps = getDefaultDeps(),
): Promise<RecordStartResult | RpcError> {
const traceVersionOrErr = negotiatedTraceVersion(params);
if (typeof traceVersionOrErr !== "number") return traceVersionOrErr;
if (traceVersionOrErr === 3 && !deps.cdp) {
return {
code: "protocol_error",
message: "trace v3 recording requires an active CDP connection",
};
}
const ctxOrErr = lookupSession(manager, params, "record_start");
if (isRpcError(ctxOrErr)) return ctxOrErr;
const ctx = ctxOrErr;
@@ -847,64 +574,37 @@ export async function handleRecordStart(
// on the destination page can RECORD_QUERY → rearm → show RecordOverlay
// instead of flashing ControlOverlay ("Agent 正在控制").
const requestId = makeRequestId(target.tabId);
let resolveFinish!: (trace: RecordedTrace) => void;
let resolveFinish!: (trace: Trace) => void;
let rejectFinish!: (err: Error) => void;
const finishPromise = new Promise<RecordedTrace>((resolve, reject) => {
const finishPromise = new Promise<Trace>((resolve, reject) => {
resolveFinish = resolve;
rejectFinish = reject;
});
const navigateUrl = params.url ?? RECORD_DEFAULT_START_URL;
const startedAtMs = Date.now();
const maxPageTokens = params.max_page_tokens;
const redactValues = params.redact_values ?? false;
recordings.set(params.session_id, {
requestId,
tabs: new RecordingTabCoordinator(target.tabId, navigateUrl),
tabId: target.tabId,
agentWindowId: ctx.agentWindowId,
startUrl: navigateUrl,
...(params.purpose ? { purpose: params.purpose } : {}),
steps: [],
startedAt: new Date(startedAtMs).toISOString(),
startedAtMs,
traceVersion: traceVersionOrErr,
supportsTabSwitchSteps: params.supports_tab_switch_steps === true,
finishPromise,
resolveFinish,
rejectFinish,
settled: false,
finishAttempt: null,
observation:
traceVersionOrErr === 3 && deps.cdp
? new RecordingObservationRuntime({
cdp: deps.cdp,
tabsApi: deps.tabsApi,
maxTokens: maxPageTokens,
redactValues,
})
: null,
stoppedBy: "user_finish",
navigationCallbacks: new Set(),
acceptingNavigation: true,
actionQueue: Promise.resolve(),
lastStepSequenceByProducer: new Map(),
});
deps.frameCoordinator?.begin(requestId, startedAtMs, target.tabId, async (scope) => {
const recording = recordings.get(params.session_id);
if (!recording || recording.requestId !== requestId || recording.settled) return;
try {
await recording.observation?.refreshDocument(scope.tabId, scope.producerId);
} catch {
// The normal action-time capture remains available if a child Document
// is replaced while its readiness refresh is in flight.
}
finishing: false,
currentUrl: navigateUrl,
pendingNavigation: false,
pendingNavigationDeadline: undefined,
});
// Observe navigations for the whole recording lifetime; attach before
// optional navigate so the destination load can rearm capture.
ensureBrowserObservationListeners(deps);
const abortPending = async (notifyContent: boolean) => {
recordings.get(params.session_id)?.observation?.cancel();
deps.frameCoordinator?.cancel(requestId);
recordings.delete(params.session_id);
releaseBrowserObservationListenersIfIdle();
if (notifyContent) {
@@ -986,7 +686,7 @@ export async function handleRecordStart(
return cancelledError();
}
active.startUrl = startUrl;
active.tabs.navigation(target.tabId, startUrl).currentUrl = startUrl;
active.currentUrl = startUrl;
if (isContentScriptRestrictedUrl(startUrl)) {
await abortPending(false);
@@ -994,7 +694,7 @@ export async function handleRecordStart(
code: "invalid_params",
message: params.url
? `cannot record on restricted URL (${startUrl}); use an http(s) page`
: `cannot record on restricted URL (${startUrl}); default start page https://example.com/ did not load — pass --url with a page you can open`,
: `cannot record on restricted URL (${startUrl}); default start page must be injectable http(s)`,
};
}
@@ -1021,10 +721,6 @@ export async function handleRecordStart(
const startMsg: RecordStartMessage = { type: RECORD_START, requestId, startedAtMs };
try {
if (deps.frameCoordinator) {
const frameStarted = await deps.frameCoordinator.armTab(requestId, target.tabId);
if (!frameStarted) throw new Error("top recording document did not start");
}
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
} catch {
await abortPending(true);
@@ -1035,17 +731,6 @@ export async function handleRecordStart(
};
}
if (active.observation) {
const activeRecording = recordings.get(params.session_id);
if (activeRecording) {
try {
await activeRecording.observation?.captureInitial(target.tabId);
} catch {
// Proceed without initial observation; steps may be dropped by reducer.
}
}
}
{
const cancelled = await abortIfCancelled(true);
if (cancelled) return cancelled;
@@ -1070,7 +755,7 @@ export async function handleRecordStop(
};
}
const trace = await finishRecording(params.session_id, deps, "cli_stop");
const trace = await finishRecording(params.session_id, deps);
if (!trace) {
return {
code: "protocol_error",
@@ -1100,9 +785,9 @@ export async function handleRecordAwait(
return { code: "cancelled", message: "record_await aborted" };
}
const outcome = await new Promise<{ trace: RecordedTrace } | { error: RpcError }>((resolve) => {
const outcome = await new Promise<{ trace: Trace } | { error: RpcError }>((resolve) => {
let settled = false;
const finish = (result: { trace: RecordedTrace } | { error: RpcError }) => {
const finish = (result: { trace: Trace } | { error: RpcError }) => {
if (settled) return;
settled = true;
if (timer) clearTimeout(timer);
@@ -1144,8 +829,6 @@ export function clearRecordingForSession(sessionId: string): void {
}
void clearRearmTimersForRecording(recording, getDefaultDeps());
if (!recording.settled) {
getDefaultDeps().frameCoordinator?.cancel(recording.requestId);
recording.observation?.cancel();
recording.settled = true;
recording.rejectFinish(new Error("recording cleared"));
}
@@ -1,82 +0,0 @@
import { describe, expect, it } from "vitest";
import {
RECORD_DOCUMENT_ATTRIBUTE,
recordingDocumentMarkerValue,
} from "@/shared/recording-document-identity";
import { projectRecordSafeObservation } from "../record-safe-observation";
describe("projectRecordSafeObservation", () => {
it("projects the opaque recording identity without exposing captured attributes", () => {
const recordingDocumentId = "123e4567-e89b-42d3-a456-426614174000";
const result = projectRecordSafeObservation({
rootFrameId: "child",
frameDocuments: [
{
frameId: "child",
target: { tabId: 4, sessionId: "oopif-session" },
contextScopeId: "scope-1",
axNodes: [],
domNodes: [
{
backendNodeId: 1,
parentBackendNodeId: null,
tag: "html",
attrs: {
[RECORD_DOCUMENT_ATTRIBUTE]: recordingDocumentMarkerValue(recordingDocumentId),
value: "user@example.com",
},
rect: null,
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
],
},
],
rendered: { text: '@vom 1\nRootWebArea "Example"', refs: [], truncated: false },
});
expect(result.frames).toEqual([
{
frameId: "child",
target: { tabId: 4, sessionId: "oopif-session" },
recordingDocumentId,
},
]);
const dumped = JSON.stringify(result);
expect(dumped).not.toContain("user@example.com");
expect(dumped).not.toContain("attrs");
expect(dumped).not.toContain("domNodes");
expect(dumped).not.toContain("axNodes");
});
it("does not project a page-controlled marker value as recording identity", () => {
const result = projectRecordSafeObservation({
rootFrameId: "root",
frameDocuments: [
{
frameId: "root",
target: { tabId: 4 },
contextScopeId: "scope-1",
axNodes: [],
domNodes: [
{
backendNodeId: 1,
parentBackendNodeId: null,
tag: "html",
attrs: { [RECORD_DOCUMENT_ATTRIBUTE]: "bsk:user@example.com" },
rect: null,
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
],
},
],
rendered: { text: '@vom 1\nRootWebArea "Example"', refs: [], truncated: false },
});
expect(result.frames).toEqual([{ frameId: "root", target: { tabId: 4 } }]);
expect(JSON.stringify(result)).not.toContain("user@example.com");
});
});
@@ -1,6 +1,5 @@
import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { readRecordingDocumentIdentity } from "@/shared/recording-document-identity";
import type { CapturedSurfaceProbe } from "./capture";
import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture";
@@ -10,8 +9,6 @@ export interface CaptureVomFrame {
target: CdpTarget;
parentFrameId?: string;
ownerBackendNodeId?: number;
/** BrowserSkill-generated identity for the recording agent in this Document. */
recordingDocumentId?: string;
}
/** Allowlisted geometry used to match a recorded action to a rendered ref. */
@@ -40,22 +37,14 @@ export interface CaptureVomObservationResult {
}
function projectFrames(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomFrame[] {
return documents.map((document) => {
let recordingDocumentId: string | undefined;
for (const node of document.domNodes) {
recordingDocumentId = readRecordingDocumentIdentity(node.attrs);
if (recordingDocumentId) break;
}
return {
frameId: document.frameId,
target: document.target,
...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
...(document.ownerBackendNodeId !== undefined
? { ownerBackendNodeId: document.ownerBackendNodeId }
: {}),
...(recordingDocumentId ? { recordingDocumentId } : {}),
};
});
return documents.map((document) => ({
frameId: document.frameId,
target: document.target,
...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
...(document.ownerBackendNodeId !== undefined
? { ownerBackendNodeId: document.ownerBackendNodeId }
: {}),
}));
}
function projectMatchNodes(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomMatchNode[] {
@@ -65,11 +65,6 @@ function deferredFakeTransport(): { transport: Transport; emit: (frame: Protocol
}
describe("performHandshake", () => {
it("advertises the protocol compatibility boundary", () => {
expect(PROTOCOL_VERSION).toBe("1.1");
expect(MIN_COMPATIBLE_PROTOCOL).toBe("1.0");
});
it("sends system.handshake with identity and both compat fields", async () => {
let sentFrame: ProtocolFrame | null = null;
const transport = fakeTransport((req) => {
@@ -79,7 +74,7 @@ describe("performHandshake", () => {
result: {
server: "browser-skill-daemon",
version: "0.1.0",
protocol_version: "1.1",
protocol_version: "1.0",
min_compatible_peer: "0.0.0",
min_compatible_protocol: "1.0",
},
@@ -134,8 +129,8 @@ describe("performHandshake", () => {
const response = {
server: "browser-skill-daemon",
version: "0.1.0",
protocol_version: "1.1",
min_compatible_protocol: "1.1",
protocol_version: "1.0",
min_compatible_protocol: "1.0",
} satisfies HandshakeResult;
const transport = fakeTransport((req) => ({
id: (req as { id: string }).id,
@@ -150,7 +145,7 @@ describe("performHandshake", () => {
});
expect(outcome.result.min_compatible_peer).toBeUndefined();
expect(outcome.result.min_compatible_protocol).toBe("1.1");
expect(outcome.result.min_compatible_protocol).toBe("1.0");
});
it("rejects when the daemon responds with an error", async () => {
@@ -183,9 +178,9 @@ describe("performHandshake", () => {
result: {
server: "browser-skill-daemon",
version: "0.1.0",
protocol_version: "1.1",
protocol_version: "1.0",
min_compatible_peer: "0.0.0",
min_compatible_protocol: "1.1",
min_compatible_protocol: "1.0",
},
});
emit({
@@ -193,14 +188,14 @@ describe("performHandshake", () => {
result: {
server: "browser-skill-daemon",
version: "0.1.0",
protocol_version: "1.1",
protocol_version: "1.0",
min_compatible_peer: "0.0.0",
min_compatible_protocol: "1.1",
min_compatible_protocol: "1.0",
},
});
await expect(pending).resolves.toMatchObject({
result: { server: "browser-skill-daemon", protocol_version: "1.1" },
result: { server: "browser-skill-daemon", protocol_version: "1.0" },
});
});
+1 -1
View File
@@ -7,7 +7,7 @@ import type {
ResponseFrame,
} from "./types";
export const PROTOCOL_VERSION = "1.1";
export const PROTOCOL_VERSION = "1.0";
/**
* Extension semver, injected at build time from `package.json` via
* Vite's `define` (see `wxt.config.ts` and `vitest.config.ts`).
+65 -121
View File
@@ -657,60 +657,10 @@ export interface EmulateResult {
}
// --------------------------------------------------------------------------
// Semantic record payloads mirror the versioned Rust protocol models.
// Semantic record payloads — mirror bsk-protocol record.rs (Trace v2)
// --------------------------------------------------------------------------
export const TRACE_VERSION_V3 = 3;
/** Logical v2 identifier. Not a wire field — v2 envelopes omit `version`. */
export const TRACE_VERSION_V2 = 2;
export const VOM_FORMAT_VERSION = 1;
export interface TargetDescriptorV3 {
ref?: string;
role?: string;
name?: string;
ctx?: string;
unmatched?: boolean;
}
export interface RecorderInfo {
bsk: string;
vom: number;
}
export type StopReason = "user_finish" | "cli_stop";
export interface TraceStateV3 {
id: string;
url: string;
title?: string;
body: string;
truncated?: boolean;
}
export interface StepResultV3 {
state: string;
}
export interface StepCommonV3 {
id: number;
state: string;
result: StepResultV3;
}
export type NavigationCause =
| "user_typed"
| "link"
| "form_submit"
| "reload"
| "history"
| "script"
| "browser";
export type FillCommit = "enter" | "suggestion" | "blur";
/** Legacy v2 target shape retained for existing record producers. */
export interface TargetDescriptorV2 {
export interface TargetDescriptor {
role?: string;
name?: string;
tag: string;
@@ -723,109 +673,103 @@ export interface TraceEntry {
start_url: string;
}
export interface PageRefV2 {
export interface PageRef {
id: string;
url: string;
title?: string;
}
export interface SelectedOptionV2 {
export interface SelectedOption {
value: string;
label?: string;
}
export interface StepEffectV2 {
export interface StepEffect {
navigated_to: string;
}
export interface StepCommonV2 {
export interface StepCommon {
id: number;
page: string;
effect?: StepEffectV2;
effect?: StepEffect;
}
/** Capture/buffer draft before v2 reduction. */
export type DraftTraceStep =
| {
op: "click";
target: TargetDescriptor;
navigated_to?: string;
page_url?: string;
}
| {
op: "hover";
target: TargetDescriptor;
page_url?: string;
}
| {
op: "fill";
target: TargetDescriptor;
value: string;
redacted?: boolean;
page_url?: string;
}
| {
op: "press";
key: string;
target?: TargetDescriptor;
modifiers?: KeyModifier[];
navigated_to?: string;
page_url?: string;
}
| {
op: "select";
target: TargetDescriptor;
values: string[];
labels?: string[];
navigated_to?: string;
page_url?: string;
}
| {
op: "navigate";
url: string;
page_url?: string;
};
/** Exported record-only step (trace v2). */
export type StepV2 =
| ({ op: "navigate" } & StepCommonV2 & { to: string })
| ({ op: "click" } & StepCommonV2 & { target: TargetDescriptorV2 })
| ({ op: "hover" } & StepCommonV2 & { target: TargetDescriptorV2 })
| ({ op: "fill" } & StepCommonV2 & {
target: TargetDescriptorV2;
export type Step =
| ({ op: "navigate" } & StepCommon & { to: string })
| ({ op: "click" } & StepCommon & { target: TargetDescriptor })
| ({ op: "hover" } & StepCommon & { target: TargetDescriptor })
| ({ op: "fill" } & StepCommon & {
target: TargetDescriptor;
value: string;
redacted?: boolean;
})
| ({ op: "select" } & StepCommonV2 & {
target: TargetDescriptorV2;
selection: SelectedOptionV2[];
| ({ op: "select" } & StepCommon & {
target: TargetDescriptor;
selection: SelectedOption[];
})
| ({ op: "press" } & StepCommonV2 & {
| ({ op: "press" } & StepCommon & {
key: string;
modifiers?: KeyModifier[];
target?: TargetDescriptorV2;
target?: TargetDescriptor;
});
export interface TraceV2 {
export interface Trace {
recorded_at: string;
started_at?: string;
purpose?: string;
entry: TraceEntry;
pages: PageRefV2[];
steps: StepV2[];
pages: PageRef[];
steps: Step[];
}
export interface SelectedOptionV3 {
value: string;
label?: string;
}
export type StepV3 =
| ({ op: "navigate" } & StepCommonV3 & { to: string; cause: NavigationCause })
| ({ op: "switch_tab" } & StepCommonV3)
| ({ op: "click" } & StepCommonV3 & { target: TargetDescriptorV3 })
| ({ op: "hover" } & StepCommonV3 & { target: TargetDescriptorV3 })
| ({ op: "fill" } & StepCommonV3 & {
target: TargetDescriptorV3;
value: string;
commit: FillCommit;
redacted?: boolean;
})
| ({ op: "select" } & StepCommonV3 & {
target: TargetDescriptorV3;
selection?: SelectedOptionV3[];
})
| ({ op: "press" } & StepCommonV3 & {
key: string;
modifiers?: KeyModifier[];
target?: TargetDescriptorV3;
})
| ({ op: "scroll" } & StepCommonV3);
export interface TraceV3 {
version: typeof TRACE_VERSION_V3;
recorded_at: string;
started_at?: string;
purpose?: string;
stopped_by: StopReason;
entry: TraceEntry;
recorder: RecorderInfo;
states: TraceStateV3[];
steps: StepV3[];
}
export type RecordedTrace = TraceV2 | TraceV3;
export type RecordedStep = StepV2 | StepV3;
export interface RecordStartParams {
session_id: string;
tab_id?: number;
url?: string;
purpose?: string;
max_page_tokens?: number;
redact_values?: boolean;
/** Omitted means v2; `3` requests a state-linked v3 trace. */
trace_version?: number;
/** Client can decode the v3 `switch_tab` step variant. */
supports_tab_switch_steps?: boolean;
}
export interface RecordStartResult {
@@ -838,7 +782,7 @@ export interface RecordStopParams {
}
export interface RecordStopResult {
trace: RecordedTrace;
trace: Trace;
}
export interface RecordAwaitParams {
@@ -847,5 +791,5 @@ export interface RecordAwaitParams {
}
export interface RecordAwaitResult {
trace: RecordedTrace;
trace: Trace;
}
+6 -14
View File
@@ -265,27 +265,19 @@ retry — complete the task autonomously or stop gracefully.
### Recording — `bsk record`
Capture the user's own actions in the Agent Window to a **trace bundle**, for later LLM-driven automation. New CLI builds request **trace v3** (page observations + action chain); older extensions may still return **trace v2** (actions only), which the CLI exports as a single `trace.json` without `states/`.
Capture the user's own actions in the Agent Window to a `trace.json`, for later LLM-driven automation:
```bash
bsk record start --browser <instance-id-or-label> \
[--url https://…] [--purpose "publish a wiki doc"] \
[--max-page-tokens 3000] [--redact-values] \
[--output trace]
bsk record start --browser <instance-id-or-label> [--url https://…] [--purpose "publish a wiki doc"] [--output trace.json]
# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
# Blocks until the user clicks Finish in the recording panel, then writes:
# trace/trace.json — action chain (+ state index when v3)
# trace/states/ — v3 only: one `sN.txt` observe snapshot per settled page state
# Blocks until the user clicks Finish in the recording panel, then writes ./trace.json and closes the window.
bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable
bsk record stop [--output trace.json] # terminal fallback if the browser panel is unavailable
```
- **v3 bundle (preferred):** `--output` is a directory (default `./trace`) containing `trace.json` and `states/`. `trace.json` lists `states[]` (page observation ids) and `steps[]` (each step binds `state` = observe snapshot *before* the action and `result.state` = snapshot *after* settle). Page bodies live in `states/sN.txt` using the same VOM format as `bsk observe`.
- **v2 fallback:** when the connected extension is older, export may contain only legacy `trace.json` with `pages[]` action context and no `states/` observation files. Update the extension for full v3 bundles.
- Each `states[]` entry is one **settled page observation** (captured after recording start, navigation landing, or action settle — not on a timer).
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
- The trace is a **record-only action log** (a `pages[]` dictionary + `navigate`/`click`/`fill`/`select`/`press` steps with `target` descriptors). It records *what the user did*; deciding which inputs are variable is left to the executing agent.
- `--purpose` is optional context metadata; it does **not** change what gets captured.
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
- There is **no** `bsk replay` — to redo a flow, read the trace and reuse the existing `session` / `snapshot` / `@eN` / `click` / `fill` tools. Follow **Stop when the goal is met**.
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
## Error handling
-1
View File
@@ -22,7 +22,6 @@ pub mod navigate;
pub mod network;
pub mod observe;
pub mod record;
pub mod record_recovery;
pub mod record_state;
pub mod render_error;
pub mod screenshot;
+64 -266
View File
@@ -1,30 +1,23 @@
//! `bsk record start|stop` — capture user actions in the Agent Window.
use std::path::{Path, PathBuf};
use std::fs;
use std::path::PathBuf;
use std::time::Duration;
use anyhow::Context;
use bsk_protocol::Method;
use bsk_protocol::tools::{
RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult, RecordStopParams,
RecordStopResult, RecordedTrace, TRACE_VERSION_V3,
RecordStopResult, Trace,
};
use bsk_protocol::{ErrorCode, Method};
use clap::{Args, Subcommand};
use crate::cli::TOOL_IPC_TIMEOUT;
mod export;
use crate::cli::business_rpc;
use crate::cli::ensure_daemon::ensure_daemon;
use crate::cli::error::{CliError, Format};
use crate::cli::record_recovery;
use crate::cli::record_state;
use crate::cli::session::{SessionStartOptions, start_session, stop_session};
use export::{
ExportMeta, export_with_recovery, states_dir_for_output, trace_json_path,
validate_record_output,
};
/// Max wait for the user to click 结束 in the browser (24 hours).
const RECORD_AWAIT_TIMEOUT_MS: u32 = 86_400_000;
@@ -57,8 +50,7 @@ pub struct RecordStartArgs {
pub tab_id: Option<i64>,
/// Navigate to this http(s) URL before recording. When omitted, defaults
/// to `https://example.com/`. If that page does not load, pass `--url`
/// with a site you can open in this browser.
/// to `https://example.com/`.
#[arg(long)]
pub url: Option<String>,
@@ -66,25 +58,15 @@ pub struct RecordStartArgs {
#[arg(long)]
pub purpose: Option<String>,
/// Max VOM tokens per page observation file (default 3000).
#[arg(long = "max-page-tokens")]
pub max_page_tokens: Option<u32>,
/// Redact all form values in page observations (`[filled]` only).
#[arg(long = "redact-values")]
pub redact_values: bool,
/// Output directory for the trace bundle (default `./trace`).
/// Writes `<dir>/trace.json` and `<dir>/states/*.txt`.
#[arg(long, default_value = "trace")]
/// Output path for the trace JSON (default `./trace.json`).
#[arg(long, default_value = "trace.json")]
pub output: PathBuf,
}
#[derive(Debug, Clone, Args)]
pub struct RecordStopArgs {
/// Output directory for the trace bundle (default `./trace`).
/// Writes `<dir>/trace.json` and `<dir>/states/*.txt`.
#[arg(long, default_value = "trace")]
/// Output path for the trace JSON (default `./trace.json`).
#[arg(long, default_value = "trace.json")]
pub output: PathBuf,
}
@@ -96,7 +78,11 @@ pub fn dispatch(cmd: RecordCmd, format: Format) -> Result<(), CliError> {
}
fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError> {
prepare_record_start(&args.output)?;
if record_state::read().is_ok() {
return Err(CliError::Local(anyhow::anyhow!(
"a recording is already in progress; run `bsk record stop` first"
)));
}
let info = ensure_daemon().context("ensure daemon is running")?;
let session = start_session(
@@ -110,12 +96,8 @@ fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError>
let start_params = RecordStartParams {
session_id: session.session_id.clone(),
tab_id: args.tab_id,
url: args.url.clone(),
url: args.url,
purpose: args.purpose.clone(),
max_page_tokens: args.max_page_tokens,
redact_values: Some(args.redact_values),
trace_version: Some(TRACE_VERSION_V3),
supports_tab_switch_steps: Some(true),
};
let start_result = business_rpc::call::<RecordStartParams, RecordStartResult>(
info.sock_path.clone(),
@@ -129,7 +111,7 @@ fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError>
Ok(result) => result,
Err(err) => {
let _ = stop_session(info.sock_path, &session.session_id);
return Err(annotate_default_start_page_error(err, args.url.as_deref()));
return Err(err);
}
};
@@ -160,8 +142,8 @@ fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError>
// Keep write/render inside a Result so `?` cannot skip session teardown.
let run_result: Result<(), CliError> = match await_result {
Ok(result) => (|| {
let exported = export_with_recovery(&args.output, &result.trace)?;
render_finish(&result.trace, &args.output, &exported, format)
write_trace_file(&args.output, &result.trace)?;
render_finish(&result.trace, &args.output, format)
})(),
Err(err) => Err(err),
};
@@ -175,12 +157,11 @@ fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError>
}
fn dispatch_stop(args: RecordStopArgs, format: Format) -> Result<(), CliError> {
validate_record_output(&args.output)?;
if let Ok(state) = record_state::read() {
let info = ensure_daemon().context("ensure daemon is running")?;
let session_id = state.session_id.clone();
let state = record_state::read().map_err(CliError::Local)?;
let info = ensure_daemon().context("ensure daemon is running")?;
let session_id = state.session_id.clone();
let run_result: Result<(), CliError> = (|| {
let params = RecordStopParams {
session_id: session_id.clone(),
};
@@ -191,85 +172,16 @@ fn dispatch_stop(args: RecordStopArgs, format: Format) -> Result<(), CliError> {
Some(params),
TOOL_IPC_TIMEOUT,
)?;
write_trace_file(&args.output, &result.trace)?;
render_stop(&result, &args.output, format)
})();
let run_result: Result<(), CliError> = (|| {
let exported = export_with_recovery(&args.output, &result.trace)?;
render_stop(&result, &args.output, &exported, format)
})();
let session_stop_result = stop_session(info.sock_path, &session_id);
record_state::clear();
let session_stop_result = stop_session(info.sock_path, &session_id);
record_state::clear();
run_result?;
session_stop_result?;
return Ok(());
}
let Some(trace) = record_recovery::load().map_err(CliError::Local)? else {
return Err(CliError::Local(anyhow::anyhow!(
"no recording in progress; run `bsk record start` first"
)));
};
let exported = export_with_recovery(&args.output, &trace)?;
render_finish(&trace, &args.output, &exported, format)
}
/// When `record start` omitted `--url` and the default example.com page
/// never loaded, rewrite the RPC error so the CLI hint points at `--url`.
fn annotate_default_start_page_error(err: CliError, user_url: Option<&str>) -> CliError {
if user_url.is_some() {
return err;
}
let CliError::Rpc {
code,
message,
data,
source,
} = err
else {
return err;
};
if !is_default_start_page_failure(code, &message) {
return CliError::Rpc {
code,
message,
data,
source,
};
}
CliError::Rpc {
code,
message,
data: Some(serde_json::json!({
"reason": crate::cli::render_error::reason::RECORD_START_PAGE_UNREACHABLE
})),
source,
}
}
fn is_default_start_page_failure(code: ErrorCode, message: &str) -> bool {
match code {
ErrorCode::CdpFailed | ErrorCode::Timeout => true,
ErrorCode::InvalidParams => {
message.contains("restricted URL") || message.contains("about:blank")
}
_ => false,
}
}
fn prepare_record_start(output: &Path) -> Result<(), CliError> {
if record_state::read().is_ok() {
return Err(CliError::Local(anyhow::anyhow!(
"a recording is already in progress; run `bsk record stop` first"
)));
}
if record_recovery::exists() {
return Err(CliError::Local(anyhow::anyhow!(
"a previous recording was not exported; run `bsk record stop --output <dir>` to recover it first"
)));
}
validate_record_output(output)
run_result?;
session_stop_result?;
Ok(())
}
fn record_await_ipc_timeout(timeout_ms: u32) -> Duration {
@@ -278,66 +190,39 @@ fn record_await_ipc_timeout(timeout_ms: u32) -> Duration {
.unwrap_or(Duration::from_secs(u64::from(timeout_ms / 1_000) + 15))
}
fn render_finish(
trace: &RecordedTrace,
output_dir: &PathBuf,
exported: &ExportMeta,
format: Format,
) -> Result<(), CliError> {
let trace_path = trace_json_path(output_dir);
fn write_trace_file(output: &PathBuf, trace: &Trace) -> Result<(), CliError> {
let json = serde_json::to_string_pretty(trace)
.context("serialize trace JSON")
.map_err(CliError::Local)?;
if let Some(parent) = output.parent() {
if !parent.as_os_str().is_empty() {
fs::create_dir_all(parent)
.with_context(|| format!("create output directory {}", parent.display()))
.map_err(CliError::Local)?;
}
}
fs::write(output, format!("{json}\n"))
.with_context(|| format!("write trace to {}", output.display()))
.map_err(CliError::Local)?;
Ok(())
}
fn render_finish(trace: &Trace, output: &PathBuf, format: Format) -> Result<(), CliError> {
match format {
Format::Json => {
let payload = match trace {
RecordedTrace::V3(t) => serde_json::json!({
"output": output_dir,
"trace_json": trace_path,
"trace_version": exported.trace_version,
"states_dir": exported.states_dir,
"trace": t,
"window_closed": true,
}),
RecordedTrace::V2(t) => serde_json::json!({
"output": output_dir,
"trace_json": trace_path,
"trace_version": exported.trace_version,
"states_dir": null,
"trace": t,
"window_closed": true,
}),
};
println!(
"{}",
serde_json::to_string(&payload).map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
serde_json::to_string(&serde_json::json!({
"output": output,
"trace": trace,
"window_closed": true,
}))
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
);
}
Format::Human => match trace {
RecordedTrace::V3(t) => {
let states_dir = exported
.states_dir
.as_ref()
.map(|p| p.display().to_string())
.unwrap_or_else(|| states_dir_for_output(output_dir).display().to_string());
println!(
"saved {} steps to {} and {} states to {}",
t.steps.len(),
trace_path.display(),
t.states.len(),
states_dir
);
}
RecordedTrace::V2(t) => {
if exported.v2_fallback {
eprintln!(
"note: extension returned trace v2 (no page observations); update the BrowserSkill extension for v3 bundles"
);
}
println!(
"saved {} steps to {} (trace v2)",
t.steps.len(),
trace_path.display()
);
}
},
Format::Human => {
println!("saved {} steps to {}", trace.steps.len(), output.display());
}
}
Ok(())
}
@@ -345,89 +230,33 @@ fn render_finish(
fn render_stop(
result: &RecordStopResult,
output: &PathBuf,
exported: &ExportMeta,
format: Format,
) -> Result<(), CliError> {
render_finish(&result.trace, output, exported, format)
render_finish(&result.trace, output, format)
}
#[cfg(test)]
mod tests {
use super::*;
fn with_temp_home<F: FnOnce()>(f: F) {
let _lock = record_recovery::test_env_lock();
let tmp = tempfile::tempdir().unwrap();
unsafe {
std::env::set_var(crate::daemon::paths::BSK_HOME_ENV, tmp.path());
}
f();
record_recovery::clear();
record_state::clear();
unsafe {
std::env::remove_var(crate::daemon::paths::BSK_HOME_ENV);
}
}
#[test]
fn prepare_record_start_rejects_existing_trace_json_file() {
with_temp_home(|| {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("trace.json");
std::fs::write(&output, "{}\n").unwrap();
let err = prepare_record_start(&output).unwrap_err();
let msg = err.to_string();
assert!(
msg.contains("JSON file") || msg.contains("json file"),
"{msg}"
);
assert!(
msg.contains("states/") || msg.contains("--output trace"),
"{msg}"
);
});
}
#[test]
fn prepare_record_start_rejects_unexported_recovery() {
with_temp_home(|| {
crate::daemon::paths::ensure_bsk_home().unwrap();
std::fs::write(
crate::daemon::paths::record_recovery_path().unwrap(),
"{}\n",
)
.unwrap();
let err = prepare_record_start(Path::new("trace")).unwrap_err();
let msg = err.to_string();
assert!(
msg.contains("not exported") || msg.contains("recover"),
"{msg}"
);
});
}
#[test]
fn default_output_is_trace_dir() {
fn default_output_is_trace_json() {
let args = RecordStopArgs {
output: PathBuf::from("trace"),
output: PathBuf::from("trace.json"),
};
assert_eq!(args.output, PathBuf::from("trace"));
assert_eq!(args.output, PathBuf::from("trace.json"));
}
#[test]
fn start_args_default_output_is_trace_dir() {
fn start_args_default_output_is_trace_json() {
let args = RecordStartArgs {
browser: None,
tab_id: None,
url: None,
purpose: None,
max_page_tokens: None,
redact_values: false,
output: PathBuf::from("trace"),
output: PathBuf::from("trace.json"),
};
assert_eq!(args.output, PathBuf::from("trace"));
assert_eq!(args.output, PathBuf::from("trace.json"));
}
#[test]
@@ -435,35 +264,4 @@ mod tests {
let got = record_await_ipc_timeout(RECORD_AWAIT_TIMEOUT_MS);
assert!(got >= Duration::from_secs(86_400));
}
#[test]
fn annotate_default_start_page_error_adds_url_reason() {
let err = CliError::from_rpc(bsk_protocol::RpcError {
code: ErrorCode::CdpFailed,
message: "Page.navigate rejected: net::ERR_ABORTED".into(),
data: None,
});
let annotated = annotate_default_start_page_error(err, None);
assert_eq!(
crate::cli::render_error::reason_for_data(annotated.data()),
Some(crate::cli::render_error::reason::RECORD_START_PAGE_UNREACHABLE)
);
let info =
crate::cli::render_error::info_for_error(annotated.code().unwrap(), annotated.data());
assert!(info.hint.unwrap().contains("--url"));
}
#[test]
fn annotate_default_start_page_error_skips_when_user_passed_url() {
let err = CliError::from_rpc(bsk_protocol::RpcError {
code: ErrorCode::CdpFailed,
message: "Page.navigate rejected: net::ERR_ABORTED".into(),
data: None,
});
let annotated = annotate_default_start_page_error(err, Some("https://www.example.org/"));
assert_eq!(
crate::cli::render_error::reason_for_data(annotated.data()),
None
);
}
}
-976
View File
@@ -1,976 +0,0 @@
use std::collections::HashSet;
use std::fs::{self, OpenOptions};
use std::io;
use std::path::{Path, PathBuf};
use anyhow::Context;
use bsk_protocol::tools::{RecordedTrace, TRACE_VERSION_V2, TRACE_VERSION_V3, TraceV2, TraceV3};
use serde::Serialize;
use crate::cli::error::CliError;
#[derive(Debug)]
pub(super) struct ExportMeta {
pub(super) states_dir: Option<PathBuf>,
pub(super) trace_version: u32,
/// True when the CLI requested v3 but the extension returned legacy v2.
pub(super) v2_fallback: bool,
}
pub(super) fn export_recorded_trace(
output_dir: &Path,
trace: &RecordedTrace,
) -> Result<ExportMeta, CliError> {
match trace {
RecordedTrace::V3(trace) => {
let states_dir = write_trace_bundle(output_dir, trace)?;
Ok(ExportMeta {
states_dir: Some(states_dir),
trace_version: TRACE_VERSION_V3,
v2_fallback: false,
})
}
RecordedTrace::V2(trace) => {
write_trace_v2(output_dir, trace)?;
Ok(ExportMeta {
states_dir: None,
trace_version: TRACE_VERSION_V2,
v2_fallback: true,
})
}
}
}
/// Save the completed Trace first, then write the bundle. Recovery is
/// removed only after the export commits so a bad `--output` cannot drop
/// a recording the extension already returned.
pub(super) fn export_with_recovery(
output_dir: &Path,
trace: &RecordedTrace,
) -> Result<ExportMeta, CliError> {
let save_result = crate::cli::record_recovery::save(trace);
match export_recorded_trace(output_dir, trace) {
Ok(meta) => {
crate::cli::record_recovery::clear();
Ok(meta)
}
Err(export_err) => {
let export_err = annotate_recovery_hint(export_err, save_result.is_ok());
if let Err(save_err) = save_result {
return Err(match export_err {
CliError::Local(inner) => CliError::Local(
inner.context(format!("also failed to save recovery data: {save_err}")),
),
other => other,
});
}
Err(export_err)
}
}
}
fn annotate_recovery_hint(err: CliError, recovery_saved: bool) -> CliError {
if !recovery_saved {
return err;
}
match err {
CliError::Local(inner) => CliError::Local(inner.context(
"export failed; the recorded trace was saved and can be recovered with `bsk record stop --output <dir>`",
)),
other => other,
}
}
fn looks_like_json_output(path: &Path) -> bool {
path.extension()
.and_then(|ext| ext.to_str())
.is_some_and(|ext| ext.eq_ignore_ascii_case("json"))
}
fn legacy_json_output_error(path: &Path) -> CliError {
CliError::Local(anyhow::anyhow!(
"--output {} is a JSON file path; Trace v3 writes a bundle directory \
(`<dir>/trace.json` and `<dir>/states/`), not a single file. \
Use `--output trace` or another directory.",
path.display()
))
}
/// Reject legacy `--output trace.json` (and any existing non-directory)
/// before a recording starts, so the user is not blocked only after Finish.
pub(super) fn validate_record_output(path: &Path) -> Result<(), CliError> {
match fs::symlink_metadata(path) {
Ok(metadata) if metadata.file_type().is_symlink() => Err(CliError::Local(anyhow::anyhow!(
"--output {} must not be a symlink",
path.display()
))),
Ok(metadata) if metadata.is_dir() => Ok(()),
Ok(_) if looks_like_json_output(path) => Err(legacy_json_output_error(path)),
Ok(_) => Err(CliError::Local(anyhow::anyhow!(
"--output {} is not a directory; Trace v3 writes a bundle directory \
(`<dir>/trace.json` and `<dir>/states/`). Use `--output trace`.",
path.display()
))),
Err(err) if err.kind() == io::ErrorKind::NotFound => {
if looks_like_json_output(path) {
Err(legacy_json_output_error(path))
} else {
Ok(())
}
}
Err(err) => Err(CliError::Local(
anyhow::Error::new(err).context(format!("inspect --output {}", path.display())),
)),
}
}
fn acquire_export_lock(output_dir: &Path) -> Result<std::fs::File, CliError> {
validate_bundle_directory(output_dir, "output bundle directory")?;
fs::create_dir_all(output_dir)
.with_context(|| format!("create output dir {}", output_dir.display()))
.map_err(CliError::Local)?;
let lock_path = output_dir.join(".bsk-record-export.lock");
let lock_file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(&lock_path)
.with_context(|| format!("open export lock {}", lock_path.display()))
.map_err(CliError::Local)?;
fs2::FileExt::lock_exclusive(&lock_file)
.with_context(|| format!("lock export bundle {}", output_dir.display()))
.map_err(CliError::Local)?;
Ok(lock_file)
}
pub(super) fn write_trace_v2(output_dir: &Path, trace: &TraceV2) -> Result<(), CliError> {
let trace_path = trace_json_path(output_dir);
let states_dir = states_dir_for_output(output_dir);
let json = serde_json::to_string_pretty(trace)
.context("serialize trace JSON")
.map_err(CliError::Local)?;
let _lock = acquire_export_lock(output_dir)?;
validate_replaceable_file(&trace_path, "trace JSON")?;
let transaction_id = uuid::Uuid::new_v4();
let staging_dir = output_dir.join(format!(".bsk-record-stage-{transaction_id}"));
fs::create_dir_all(&staging_dir)
.with_context(|| format!("create staging dir {}", staging_dir.display()))
.map_err(CliError::Local)?;
let staged_trace = staging_dir.join("trace.json");
let stage_result: Result<(), CliError> = (|| {
fs::write(&staged_trace, format!("{json}\n"))
.with_context(|| format!("write staged trace to {}", staged_trace.display()))
.map_err(CliError::Local)?;
Ok(())
})();
if let Err(err) = stage_result {
let _ = fs::remove_dir_all(&staging_dir);
return Err(err);
}
commit_export_transaction(
&staging_dir,
&states_dir,
&HashSet::new(),
transaction_id,
vec![(
staged_trace,
trace_path,
output_dir.join(format!(".trace.json.{transaction_id}.bak")),
)],
)?;
Ok(())
}
pub(super) fn states_dir_for_output(output_dir: &Path) -> PathBuf {
output_dir.join("states")
}
pub(super) fn trace_json_path(output_dir: &Path) -> PathBuf {
output_dir.join("trace.json")
}
fn is_canonical_state_id(id: &str) -> bool {
let Some(number) = id.strip_prefix('s') else {
return false;
};
let mut digits = number.bytes();
matches!(digits.next(), Some(b'1'..=b'9')) && digits.all(|byte| byte.is_ascii_digit())
}
fn validate_bundle_directory(path: &Path, description: &str) -> Result<(), CliError> {
match fs::symlink_metadata(path) {
Ok(metadata) if metadata.file_type().is_symlink() => Err(CliError::Local(anyhow::anyhow!(
"{description} {} must not be a symlink",
path.display()
))),
Ok(metadata) if metadata.is_dir() => Ok(()),
Ok(_) if looks_like_json_output(path) => Err(legacy_json_output_error(path)),
Ok(_) => Err(CliError::Local(anyhow::anyhow!(
"{description} {} is not a directory; Trace v3 writes `<dir>/trace.json` and `<dir>/states/`. Use `--output trace`.",
path.display()
))),
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
Err(err) => Err(CliError::Local(
anyhow::Error::new(err).context(format!("inspect {description} {}", path.display())),
)),
}
}
fn validate_replaceable_file(path: &Path, description: &str) -> Result<(), CliError> {
match fs::symlink_metadata(path) {
Ok(metadata) if metadata.file_type().is_file() => Ok(()),
Ok(_) => Err(CliError::Local(anyhow::anyhow!(
"{description} {} is not a regular file",
path.display()
))),
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
Err(err) => Err(CliError::Local(
anyhow::Error::new(err).context(format!("inspect {description} {}", path.display())),
)),
}
}
fn commit_staged_file(staged: &Path, target: &Path, backup: &Path) -> io::Result<Option<PathBuf>> {
let old_file = match fs::symlink_metadata(target) {
Ok(_) => {
fs::rename(target, backup)?;
Some(backup.to_path_buf())
}
Err(err) if err.kind() == io::ErrorKind::NotFound => None,
Err(err) => return Err(err),
};
if let Err(err) = fs::rename(staged, target) {
if let Some(old_file) = old_file.as_deref() {
if let Err(restore_err) = fs::rename(old_file, target) {
return Err(io::Error::new(
restore_err.kind(),
format!(
"{err}; additionally failed to restore {}: {restore_err}",
target.display()
),
));
}
}
return Err(err);
}
Ok(old_file)
}
fn rollback_installed_files(installed: &[(PathBuf, Option<PathBuf>)]) -> io::Result<()> {
let mut rollback_error = None;
for (target, backup) in installed.iter().rev() {
match fs::remove_file(target) {
Ok(()) => {}
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => {
rollback_error.get_or_insert(err);
continue;
}
}
if let Some(backup) = backup {
if let Err(err) = fs::rename(backup, target) {
rollback_error.get_or_insert(err);
}
}
}
match rollback_error {
Some(err) => Err(err),
None => Ok(()),
}
}
fn is_generated_state_name(name: &str) -> bool {
name.strip_suffix(".txt").is_some_and(is_canonical_state_id)
}
fn restore_staged_stale_states(staged: &[(PathBuf, PathBuf)]) -> io::Result<()> {
let mut restore_error = None;
for (original, backup) in staged.iter().rev() {
if let Err(err) = fs::rename(backup, original) {
restore_error.get_or_insert(err);
}
}
match restore_error {
Some(err) => Err(err),
None => Ok(()),
}
}
/// Move stale generated states aside so replacement and cleanup form one
/// rollback boundary. User-owned files, including arbitrary non-`sN.txt`, stay.
fn stage_stale_states(
states_dir: &Path,
keep: &HashSet<String>,
transaction_id: uuid::Uuid,
) -> Result<Vec<(PathBuf, PathBuf)>, CliError> {
let entries = match fs::read_dir(states_dir) {
Ok(entries) => entries,
Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()),
Err(err) => {
return Err(CliError::Local(
anyhow::Error::new(err)
.context(format!("read states dir {}", states_dir.display())),
));
}
};
let mut staged = Vec::new();
for entry in entries {
let entry = match entry {
Ok(entry) => entry,
Err(err) => {
let restore_result = restore_staged_stale_states(&staged);
let message = match restore_result {
Ok(()) => format!("read stale state entry: {err}"),
Err(restore_err) => {
format!("read stale state entry: {err}; restore also failed: {restore_err}")
}
};
return Err(CliError::Local(anyhow::anyhow!(message)));
}
};
let name = entry.file_name();
let Some(name) = name.to_str() else { continue };
if !is_generated_state_name(name) || keep.contains(name) {
continue;
}
let original = entry.path();
let file_type = match entry.file_type() {
Ok(file_type) => file_type,
Err(err) => {
let restore_result = restore_staged_stale_states(&staged);
let message = match restore_result {
Ok(()) => format!("inspect stale state {}: {err}", original.display()),
Err(restore_err) => format!(
"inspect stale state {}: {err}; restore also failed: {restore_err}",
original.display()
),
};
return Err(CliError::Local(anyhow::anyhow!(message)));
}
};
if file_type.is_dir() {
let restore_result = restore_staged_stale_states(&staged);
let message = match restore_result {
Ok(()) => format!("stale state {} is a directory", original.display()),
Err(restore_err) => format!(
"stale state {} is a directory; restore also failed: {restore_err}",
original.display()
),
};
return Err(CliError::Local(anyhow::anyhow!(message)));
}
let backup = states_dir.join(format!(".{name}.{transaction_id}.stale"));
if let Err(err) = fs::rename(&original, &backup) {
let restore_result = restore_staged_stale_states(&staged);
let message = match restore_result {
Ok(()) => format!("stage stale state {}: {err}", original.display()),
Err(restore_err) => format!(
"stage stale state {}: {err}; restore also failed: {restore_err}",
original.display()
),
};
return Err(CliError::Local(anyhow::anyhow!(message)));
}
staged.push((original, backup));
}
Ok(staged)
}
fn commit_export_transaction(
staging_dir: &Path,
states_dir: &Path,
keep_states: &HashSet<String>,
transaction_id: uuid::Uuid,
replacements: Vec<(PathBuf, PathBuf, PathBuf)>,
) -> Result<(), CliError> {
let mut installed = Vec::with_capacity(replacements.len());
for (staged, target, backup) in replacements {
match commit_staged_file(&staged, &target, &backup) {
Ok(old_file) => installed.push((target, old_file)),
Err(err) => {
let rollback_result = rollback_installed_files(&installed);
let _ = fs::remove_dir_all(staging_dir);
let message = match rollback_result {
Ok(()) => format!("commit replacement {}: {err}", target.display()),
Err(rollback_err) => format!(
"commit replacement {}: {err}; rollback also failed: {rollback_err}",
target.display()
),
};
return Err(CliError::Local(anyhow::anyhow!(message)));
}
}
}
let stale_states = match stage_stale_states(states_dir, keep_states, transaction_id) {
Ok(stale) => stale,
Err(err) => {
let rollback_result = rollback_installed_files(&installed);
let _ = fs::remove_dir_all(staging_dir);
let message = match rollback_result {
Ok(()) => err.to_string(),
Err(rollback_err) => {
format!("{err}; replacement rollback also failed: {rollback_err}")
}
};
return Err(CliError::Local(anyhow::anyhow!(message)));
}
};
for (_, backup) in &installed {
if let Some(backup) = backup {
let _ = fs::remove_file(backup);
}
}
for (_, backup) in stale_states {
let _ = fs::remove_file(backup);
}
let _ = fs::remove_dir_all(staging_dir);
Ok(())
}
#[derive(Serialize)]
struct BundleState<'a> {
id: &'a str,
url: &'a str,
#[serde(skip_serializing_if = "Option::is_none")]
title: Option<&'a str>,
page: String,
#[serde(skip_serializing_if = "std::ops::Not::not")]
truncated: bool,
}
#[derive(Serialize)]
struct BundleTrace<'a> {
version: u32,
#[serde(skip_serializing_if = "Option::is_none")]
purpose: Option<&'a str>,
#[serde(skip_serializing_if = "Option::is_none")]
started_at: Option<&'a str>,
recorded_at: &'a str,
stopped_by: &'a bsk_protocol::tools::StopReason,
entry: &'a bsk_protocol::tools::TraceEntry,
recorder: &'a bsk_protocol::tools::RecorderInfo,
states: Vec<BundleState<'a>>,
steps: &'a [bsk_protocol::tools::StepV3],
}
pub(super) fn write_trace_bundle(output_dir: &Path, trace: &TraceV3) -> Result<PathBuf, CliError> {
let states_dir = states_dir_for_output(output_dir);
let trace_path = trace_json_path(output_dir);
if trace.version != TRACE_VERSION_V3 {
return Err(CliError::Local(anyhow::anyhow!(
"trace version {} is not supported (expected {})",
trace.version,
TRACE_VERSION_V3
)));
}
let mut state_ids = HashSet::with_capacity(trace.states.len());
for state in &trace.states {
if !is_canonical_state_id(&state.id) {
return Err(CliError::Local(anyhow::anyhow!(
"state id {:?} is not canonical (expected s1, s2, ...)",
state.id
)));
}
if !state_ids.insert(state.id.as_str()) {
return Err(CliError::Local(anyhow::anyhow!(
"duplicate state id {:?}",
state.id
)));
}
}
let mut disk_states = Vec::with_capacity(trace.states.len());
let mut state_writes = Vec::new();
let mut keep = HashSet::new();
for state in &trace.states {
let filename = format!("{}.txt", state.id);
let state_path = states_dir.join(&filename);
if state_path.parent() != Some(states_dir.as_path()) {
return Err(CliError::Local(anyhow::anyhow!(
"state path {} escapes states directory {}",
state_path.display(),
states_dir.display()
)));
}
keep.insert(filename.clone());
state_writes.push((filename.clone(), state.body.as_str()));
disk_states.push(BundleState {
id: &state.id,
url: &state.url,
title: state.title.as_deref(),
page: filename,
truncated: state.truncated,
});
}
let disk_trace = BundleTrace {
version: trace.version,
purpose: trace.purpose.as_deref(),
started_at: trace.started_at.as_deref(),
recorded_at: &trace.recorded_at,
stopped_by: &trace.stopped_by,
entry: &trace.entry,
recorder: &trace.recorder,
states: disk_states,
steps: &trace.steps,
};
let json = serde_json::to_string_pretty(&disk_trace)
.context("serialize trace JSON")
.map_err(CliError::Local)?;
validate_bundle_directory(output_dir, "output bundle directory")?;
fs::create_dir_all(output_dir)
.with_context(|| format!("create output dir {}", output_dir.display()))
.map_err(CliError::Local)?;
validate_bundle_directory(&states_dir, "states directory")?;
let _lock = acquire_export_lock(output_dir)?;
fs::create_dir_all(&states_dir)
.with_context(|| format!("create states dir {}", states_dir.display()))
.map_err(CliError::Local)?;
for (filename, _) in &state_writes {
validate_replaceable_file(&states_dir.join(filename), "state observation")?;
}
validate_replaceable_file(&trace_path, "trace JSON")?;
let transaction_id = uuid::Uuid::new_v4();
let staging_dir = output_dir.join(format!(".bsk-record-stage-{transaction_id}"));
let staging_states = staging_dir.join("states");
fs::create_dir_all(&staging_states)
.with_context(|| format!("create staging dir {}", staging_states.display()))
.map_err(CliError::Local)?;
let stage_result: Result<(), CliError> = (|| {
for (filename, body) in &state_writes {
let staged_path = staging_states.join(filename);
fs::write(&staged_path, body)
.with_context(|| {
format!("write staged state observation {}", staged_path.display())
})
.map_err(CliError::Local)?;
}
let staged_trace = staging_dir.join("trace.json");
fs::write(&staged_trace, format!("{json}\n"))
.with_context(|| format!("write staged trace to {}", staged_trace.display()))
.map_err(CliError::Local)?;
Ok(())
})();
if let Err(err) = stage_result {
let _ = fs::remove_dir_all(&staging_dir);
return Err(err);
}
let mut replacements: Vec<(PathBuf, PathBuf, PathBuf)> =
Vec::with_capacity(state_writes.len() + 1);
for (filename, _) in &state_writes {
replacements.push((
staging_states.join(filename),
states_dir.join(filename),
states_dir.join(format!(".{filename}.{transaction_id}.bak")),
));
}
replacements.push((
staging_dir.join("trace.json"),
trace_path.clone(),
output_dir.join(format!(".trace.json.{transaction_id}.bak")),
));
commit_export_transaction(
&staging_dir,
&states_dir,
&keep,
transaction_id,
replacements,
)?;
Ok(states_dir)
}
#[cfg(test)]
mod tests {
use std::fs;
use super::*;
use bsk_protocol::tools::{
NavigationCause, PageRefV2 as PageRef, RecorderInfo, StepCommonV3 as StepCommon,
StepResultV3 as StepResult, StepV3 as Step, StopReason, TRACE_VERSION_V2, TRACE_VERSION_V3,
TraceEntry, TraceStateV3, TraceV2, TraceV3, VOM_FORMAT_VERSION,
};
fn sample_trace(state_id: &str, body: &str) -> TraceV3 {
TraceV3 {
version: TRACE_VERSION_V3,
recorded_at: "2026-08-10T02:12:55.360Z".into(),
stopped_by: StopReason::UserFinish,
entry: TraceEntry {
start_url: "https://example.com/".into(),
},
recorder: RecorderInfo {
bsk: "0.1.10".into(),
vom: VOM_FORMAT_VERSION,
},
states: vec![TraceStateV3 {
id: state_id.into(),
url: "https://example.com/".into(),
title: Some("Example".into()),
body: body.into(),
truncated: false,
}],
steps: vec![Step::Navigate {
common: StepCommon {
id: 1,
state: state_id.into(),
result: StepResult {
state: state_id.into(),
},
},
to: "https://example.com/".into(),
cause: NavigationCause::UserTyped,
}],
purpose: None,
started_at: None,
}
}
#[test]
fn write_trace_bundle_writes_dir_layout() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
let trace = sample_trace("s1", "@vom 1\nL1 page");
let states_dir = write_trace_bundle(&output, &trace).unwrap();
let trace_path = output.join("trace.json");
assert!(trace_path.exists());
assert_eq!(states_dir, output.join("states"));
assert!(states_dir.join("s1.txt").exists());
let disk: serde_json::Value =
serde_json::from_str(&fs::read_to_string(&trace_path).unwrap()).unwrap();
assert_eq!(disk["states"][0]["page"], "s1.txt");
assert!(disk["states"][0].get("body").is_none());
assert_eq!(
fs::read_to_string(states_dir.join("s1.txt")).unwrap(),
"@vom 1\nL1 page"
);
// Re-exporting drops states from the previous run but keeps the
// directory in place for the concurrent writer.
fs::write(states_dir.join("s9.txt"), "stale").unwrap();
fs::write(states_dir.join("notes.md"), "keep me").unwrap();
fs::write(states_dir.join("reference.txt"), "user file").unwrap();
write_trace_bundle(&output, &trace).unwrap();
assert!(!states_dir.join("s9.txt").exists());
assert!(states_dir.join("s1.txt").exists());
assert!(states_dir.join("notes.md").exists());
assert_eq!(
fs::read_to_string(states_dir.join("reference.txt")).unwrap(),
"user file"
);
}
#[test]
fn write_trace_bundle_rejects_traversal_state_id() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
let trace = sample_trace("../escaped", "@vom 1\nL1 page");
let err = write_trace_bundle(&output, &trace).unwrap_err();
assert!(err.to_string().contains("state id"));
assert!(!output.join("escaped.txt").exists());
assert!(!output.join("trace.json").exists());
}
#[test]
fn write_trace_bundle_rejects_duplicate_state_id() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
let mut trace = sample_trace("s1", "@vom 1\nL1 first");
trace.states.push(TraceStateV3 {
body: "@vom 1\nL1 duplicate".into(),
..trace.states[0].clone()
});
let err = write_trace_bundle(&output, &trace).unwrap_err();
assert!(err.to_string().contains("duplicate state id"));
assert!(!output.exists());
}
#[test]
fn unsupported_version_preserves_existing_bundle() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
write_trace_bundle(&output, &sample_trace("s1", "original")).unwrap();
let original_json = fs::read(output.join("trace.json")).unwrap();
let mut replacement = sample_trace("s2", "replacement");
replacement.version += 1;
let err = write_trace_bundle(&output, &replacement).unwrap_err();
assert!(err.to_string().contains("not supported"));
assert_eq!(fs::read(output.join("trace.json")).unwrap(), original_json);
assert_eq!(
fs::read_to_string(output.join("states/s1.txt")).unwrap(),
"original"
);
assert!(!output.join("states/s2.txt").exists());
}
#[test]
fn state_replacement_failure_preserves_existing_bundle() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
write_trace_bundle(&output, &sample_trace("s1", "original")).unwrap();
let original_json = fs::read(output.join("trace.json")).unwrap();
fs::write(output.join("states/notes.md"), "keep me").unwrap();
fs::create_dir(output.join("states/s2.txt")).unwrap();
let err = write_trace_bundle(&output, &sample_trace("s2", "replacement")).unwrap_err();
assert!(err.to_string().contains("state"));
assert_eq!(fs::read(output.join("trace.json")).unwrap(), original_json);
assert_eq!(
fs::read_to_string(output.join("states/s1.txt")).unwrap(),
"original"
);
assert_eq!(
fs::read_to_string(output.join("states/notes.md")).unwrap(),
"keep me"
);
}
#[test]
fn stale_state_cleanup_failure_rolls_back_replacement() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
write_trace_bundle(&output, &sample_trace("s1", "original")).unwrap();
let original_json = fs::read(output.join("trace.json")).unwrap();
fs::create_dir(output.join("states/s9.txt")).unwrap();
let err = write_trace_bundle(&output, &sample_trace("s2", "replacement")).unwrap_err();
assert!(err.to_string().contains("stale state"));
assert_eq!(fs::read(output.join("trace.json")).unwrap(), original_json);
assert_eq!(
fs::read_to_string(output.join("states/s1.txt")).unwrap(),
"original"
);
assert!(!output.join("states/s2.txt").exists());
assert!(output.join("states/s9.txt").is_dir());
}
#[cfg(unix)]
#[test]
fn write_trace_bundle_rejects_symlinked_states_directory() {
use std::os::unix::fs::symlink;
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
let external_states = dir.path().join("external-states");
fs::create_dir_all(&output).unwrap();
fs::create_dir_all(&external_states).unwrap();
fs::write(external_states.join("s9.txt"), "external").unwrap();
symlink(&external_states, output.join("states")).unwrap();
let err = write_trace_bundle(&output, &sample_trace("s1", "replacement")).unwrap_err();
assert!(err.to_string().contains("symlink"));
assert_eq!(
fs::read_to_string(external_states.join("s9.txt")).unwrap(),
"external"
);
assert!(!external_states.join("s1.txt").exists());
assert!(!output.join("trace.json").exists());
}
#[cfg(unix)]
#[test]
fn write_trace_bundle_rejects_symlinked_output_directory() {
use std::os::unix::fs::symlink;
let dir = tempfile::tempdir().unwrap();
let external_output = dir.path().join("external-output");
let output = dir.path().join("flow");
fs::create_dir_all(&external_output).unwrap();
fs::write(external_output.join("notes.md"), "external").unwrap();
symlink(&external_output, &output).unwrap();
let err = write_trace_bundle(&output, &sample_trace("s1", "replacement")).unwrap_err();
assert!(err.to_string().contains("symlink"));
assert_eq!(
fs::read_to_string(external_output.join("notes.md")).unwrap(),
"external"
);
assert!(!external_output.join("trace.json").exists());
assert!(!external_output.join("states").exists());
}
#[test]
fn write_trace_v2_writes_single_json_and_prunes_stale_states() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
fs::create_dir_all(output.join("states")).unwrap();
fs::write(output.join("states/s1.txt"), "stale").unwrap();
let trace = TraceV2 {
recorded_at: "2026-08-10T02:12:55.360Z".into(),
started_at: None,
purpose: None,
entry: bsk_protocol::tools::TraceEntry {
start_url: "https://example.com/".into(),
},
pages: vec![PageRef {
id: "p1".into(),
url: "https://example.com/".into(),
title: None,
}],
steps: vec![],
};
write_trace_v2(&output, &trace).unwrap();
assert!(output.join("trace.json").exists());
assert!(!output.join("states/s1.txt").exists());
}
#[test]
fn write_trace_v2_stale_cleanup_failure_rolls_back_trace() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
write_trace_bundle(&output, &sample_trace("s1", "original")).unwrap();
let original_json = fs::read(output.join("trace.json")).unwrap();
fs::create_dir(output.join("states/s9.txt")).unwrap();
let trace = TraceV2 {
recorded_at: "2026-08-10T02:12:55.360Z".into(),
started_at: None,
purpose: None,
entry: bsk_protocol::tools::TraceEntry {
start_url: "https://example.com/".into(),
},
pages: vec![],
steps: vec![],
};
let err = write_trace_v2(&output, &trace).unwrap_err();
assert!(err.to_string().contains("stale state"));
assert_eq!(fs::read(output.join("trace.json")).unwrap(), original_json);
assert_eq!(
fs::read_to_string(output.join("states/s1.txt")).unwrap(),
"original"
);
assert!(output.join("states/s9.txt").is_dir());
}
#[test]
fn export_recorded_trace_accepts_v2() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
let trace = RecordedTrace::V2(TraceV2 {
recorded_at: "2026-08-10T02:12:55.360Z".into(),
started_at: None,
purpose: None,
entry: bsk_protocol::tools::TraceEntry {
start_url: "https://example.com/".into(),
},
pages: vec![],
steps: vec![],
});
let exported = export_recorded_trace(&output, &trace).unwrap();
assert_eq!(exported.trace_version, TRACE_VERSION_V2);
assert!(exported.states_dir.is_none());
assert!(output.join("trace.json").exists());
}
#[test]
fn existing_trace_json_file_is_rejected_with_bundle_usage() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("trace.json");
fs::write(&output, "{}\n").unwrap();
let err = validate_record_output(&output).unwrap_err();
let msg = err.to_string();
assert!(
msg.contains("JSON file") || msg.contains("json file"),
"should name the legacy JSON file usage: {msg}"
);
assert!(
msg.contains("states/") || msg.contains("--output trace"),
"should hint at the new bundle directory usage: {msg}"
);
assert_eq!(fs::read_to_string(&output).unwrap(), "{}\n");
}
#[test]
fn missing_trace_json_path_is_rejected_as_legacy_output() {
let err = validate_record_output(Path::new("trace.json")).unwrap_err();
let msg = err.to_string();
assert!(
msg.contains("JSON file") || msg.contains("json file"),
"{msg}"
);
assert!(
msg.contains("states/") || msg.contains("--output trace"),
"{msg}"
);
}
#[test]
fn export_failure_keeps_recovery_for_retry() {
let _lock = crate::cli::record_recovery::test_env_lock();
let tmp = tempfile::tempdir().unwrap();
unsafe {
std::env::set_var(crate::daemon::paths::BSK_HOME_ENV, tmp.path());
}
let dir = tempfile::tempdir().unwrap();
let bad_output = dir.path().join("trace.json");
fs::write(&bad_output, "{}\n").unwrap();
let trace = RecordedTrace::V3(sample_trace("s1", "@vom 1\nL1 page"));
let err = export_with_recovery(&bad_output, &trace).unwrap_err();
let msg = err.to_string();
assert!(
msg.contains("recover") || msg.contains("record stop"),
"export failure should tell the user how to recover: {msg}"
);
assert!(
crate::cli::record_recovery::exists(),
"recovery data must survive a failed bundle export"
);
let recovered = crate::cli::record_recovery::load()
.unwrap()
.expect("recovery file");
assert_eq!(recovered, trace);
assert_eq!(fs::read_to_string(&bad_output).unwrap(), "{}\n");
let good_output = dir.path().join("bundle");
export_with_recovery(&good_output, &recovered).unwrap();
assert!(good_output.join("trace.json").exists());
assert!(good_output.join("states/s1.txt").exists());
assert!(
!crate::cli::record_recovery::exists(),
"successful export must delete recovery data"
);
unsafe {
std::env::remove_var(crate::daemon::paths::BSK_HOME_ENV);
}
}
}
-112
View File
@@ -1,112 +0,0 @@
//! Persist a completed Trace so a failed bundle export can be retried.
use std::fs;
use anyhow::{Context, Result};
use bsk_protocol::tools::RecordedTrace;
use crate::daemon::paths;
pub fn save(trace: &RecordedTrace) -> Result<()> {
let path = paths::record_recovery_path()?;
paths::ensure_bsk_home()?;
let json = serde_json::to_string_pretty(trace).context("serialize record recovery")?;
let tmp = path.with_extension("tmp");
fs::write(&tmp, format!("{json}\n")).with_context(|| format!("write {}", tmp.display()))?;
fs::rename(&tmp, &path).with_context(|| format!("commit {}", path.display()))?;
Ok(())
}
pub fn load() -> Result<Option<RecordedTrace>> {
let path = paths::record_recovery_path()?;
if !path.exists() {
return Ok(None);
}
let raw = fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?;
let trace = serde_json::from_str(&raw).with_context(|| format!("parse {}", path.display()))?;
Ok(Some(trace))
}
pub fn exists() -> bool {
paths::record_recovery_path()
.map(|path| path.exists())
.unwrap_or(false)
}
pub fn clear() {
if let Ok(path) = paths::record_recovery_path() {
let _ = fs::remove_file(path);
}
}
#[cfg(test)]
pub(crate) fn test_env_lock() -> std::sync::MutexGuard<'static, ()> {
crate::daemon::paths::test_env_lock()
}
#[cfg(test)]
mod tests {
use super::*;
use bsk_protocol::tools::{
RecorderInfo, StopReason, TRACE_VERSION_V3, TraceEntry, TraceStateV3, TraceV3,
VOM_FORMAT_VERSION,
};
fn sample_trace() -> RecordedTrace {
RecordedTrace::V3(TraceV3 {
version: TRACE_VERSION_V3,
recorded_at: "2026-08-10T02:12:55.360Z".into(),
stopped_by: StopReason::UserFinish,
entry: TraceEntry {
start_url: "https://example.com/".into(),
},
recorder: RecorderInfo {
bsk: "0.1.10".into(),
vom: VOM_FORMAT_VERSION,
},
states: vec![TraceStateV3 {
id: "s1".into(),
url: "https://example.com/".into(),
title: Some("Example".into()),
body: "@vom 1\nL1 page".into(),
truncated: false,
}],
steps: vec![],
purpose: None,
started_at: None,
})
}
fn with_temp_home<F: FnOnce()>(f: F) {
let _lock = test_env_lock();
let tmp = tempfile::tempdir().unwrap();
unsafe {
std::env::set_var(crate::daemon::paths::BSK_HOME_ENV, tmp.path());
}
f();
clear();
unsafe {
std::env::remove_var(crate::daemon::paths::BSK_HOME_ENV);
}
}
#[test]
fn save_load_round_trips() {
with_temp_home(|| {
let trace = sample_trace();
save(&trace).unwrap();
assert!(exists());
assert_eq!(load().unwrap(), Some(trace));
});
}
#[test]
fn clear_removes_recovery_file() {
with_temp_home(|| {
save(&sample_trace()).unwrap();
clear();
assert!(!exists());
assert_eq!(load().unwrap(), None);
});
}
}
+7 -1
View File
@@ -59,9 +59,15 @@ fn started_at_unix_ms() -> String {
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Mutex;
fn env_guard() -> &'static Mutex<()> {
static GUARD: Mutex<()> = Mutex::new(());
&GUARD
}
fn with_temp_home<F: FnOnce()>(f: F) {
let _lock = crate::daemon::paths::test_env_lock();
let _lock = env_guard().lock().unwrap_or_else(|e| e.into_inner());
let tmp = tempfile::tempdir().unwrap();
unsafe {
std::env::set_var(crate::daemon::paths::BSK_HOME_ENV, tmp.path());
-27
View File
@@ -50,7 +50,6 @@ pub mod reason {
pub const BORROW_CONFLICT: &str = "borrow_conflict";
pub const SCREENSHOT_CAPTURE_FAILED: &str = "screenshot_capture_failed";
pub const SESSION_BUSY: &str = crate::rpc_reason::SESSION_BUSY;
pub const RECORD_START_PAGE_UNREACHABLE: &str = "record_start_page_unreachable";
}
/// Per-code rendering metadata. The `summary` is what the CLI prints
@@ -270,16 +269,6 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
),
exit_code: base.exit_code,
},
(
ErrorCode::CdpFailed | ErrorCode::Timeout | ErrorCode::InvalidParams,
reason::RECORD_START_PAGE_UNREACHABLE,
) => RenderInfo {
summary: "default start page https://example.com/ could not be opened",
hint: Some(
"that site may be unreachable from your network. Pass --url with a page you can open, e.g. `bsk record start --url https://<site>/`",
),
exit_code: base.exit_code,
},
_ => base,
}
}
@@ -389,22 +378,6 @@ mod tests {
assert_eq!(info.exit_code, 4);
}
#[test]
fn record_start_page_unreachable_overrides_cdp_failed_copy() {
let data = serde_json::json!({ "reason": reason::RECORD_START_PAGE_UNREACHABLE });
let info = info_for_error(ErrorCode::CdpFailed, Some(&data));
assert_eq!(
info.summary,
"default start page https://example.com/ could not be opened"
);
assert!(
info.hint.unwrap().contains("--url"),
"expected --url hint, got {:?}",
info.hint
);
assert_eq!(info.exit_code, 3);
}
#[test]
fn screenshot_capture_failed_overrides_cdp_failed_copy() {
let data = serde_json::json!({ "reason": reason::SCREENSHOT_CAPTURE_FAILED });
+9 -21
View File
@@ -100,14 +100,6 @@ pub fn record_session_path() -> Result<PathBuf> {
Ok(bsk_home()?.join("record-session.json"))
}
/// Path to a completed Trace saved before bundle export (`record-recovery.json`).
///
/// Kept until export succeeds so a failed `--output` write cannot drop the
/// recording the extension already returned.
pub fn record_recovery_path() -> Result<PathBuf> {
Ok(bsk_home()?.join("record-recovery.json"))
}
/// Windows named-pipe name. Include the resolved `BSK_HOME` path in the
/// token so test homes and custom installs do not share a predictable
/// per-username pipe.
@@ -127,13 +119,6 @@ pub fn pipe_name() -> String {
render_pipe_name(&user, &home)
}
/// Process-wide lock for tests that mutate `BSK_HOME`.
#[cfg(test)]
pub(crate) fn test_env_lock() -> std::sync::MutexGuard<'static, ()> {
static GUARD: std::sync::Mutex<()> = std::sync::Mutex::new(());
GUARD.lock().unwrap_or_else(|e| e.into_inner())
}
/// Render the Windows named-pipe name for a user/home pair. Kept
/// platform-agnostic so unit tests on any host can pin the output
/// character set.
@@ -151,12 +136,19 @@ fn render_pipe_name(user: &str, home: &str) -> String {
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Mutex;
use tempfile::TempDir;
/// Serialise tests that mutate the global `BSK_HOME` env var.
fn env_guard() -> &'static Mutex<()> {
static GUARD: Mutex<()> = Mutex::new(());
&GUARD
}
fn with_temp_home<F: FnOnce(&Path)>(f: F) {
let _lock = test_env_lock();
let _lock = env_guard().lock().unwrap_or_else(|e| e.into_inner());
let tmp = TempDir::new().unwrap();
// SAFETY: serialised by test_env_lock above.
// SAFETY: serialised by env_guard above.
unsafe {
std::env::set_var(BSK_HOME_ENV, tmp.path().join("bsk"));
}
@@ -195,10 +187,6 @@ mod tests {
record_session_path().unwrap(),
home.join("record-session.json")
);
assert_eq!(
record_recovery_path().unwrap(),
home.join("record-recovery.json")
);
});
}
+29 -97
View File
@@ -15,6 +15,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::path::Path;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::{Duration, Instant};
use anyhow::{Context, Result};
@@ -26,7 +27,7 @@ use crate::daemon::{
browsers::{BROWSER_LIVENESS_TICK, BROWSER_LIVENESS_TIMEOUT, EXTENSION_CONNECT_WAIT},
info as daemon_info, ipc, lockfile, paths,
sessions::{StopSessionError, forget_session, stop_session},
state::{DaemonState, PROTOCOL_VERSION},
state::DaemonState,
ws,
};
@@ -290,14 +291,15 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
});
let started_at = Instant::now();
let activity = Arc::new(Mutex::new(IpcActivityState::new(started_at)));
let activity: Arc<Mutex<Instant>> = Arc::new(Mutex::new(started_at));
let active_ipc_connections = Arc::new(AtomicUsize::new(0));
let (ipc_shutdown_tx, ipc_shutdown_rx) = tokio::sync::oneshot::channel::<()>();
let status = ipc::DaemonStatus {
started_at,
ws_port,
sock_path: sock_path.clone(),
daemon_version: env!("CARGO_PKG_VERSION"),
protocol_version: PROTOCOL_VERSION,
protocol_version: "1.0",
};
let handler = ipc::full_handler(status, Arc::clone(&state));
@@ -312,14 +314,18 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
let handler = handler.clone();
let ipc_open = {
let activity = activity.clone();
let active = active_ipc_connections.clone();
move || {
record_ipc_open(&activity);
active.fetch_add(1, Ordering::SeqCst);
record_activity(&activity);
}
};
let ipc_close = {
let activity = activity.clone();
let active = active_ipc_connections.clone();
move || {
record_ipc_close(&activity);
active.fetch_sub(1, Ordering::SeqCst);
record_activity(&activity);
}
};
tokio::spawn(ipc::serve(
@@ -338,6 +344,7 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
let idle_task = {
let activity = activity.clone();
let state = Arc::clone(&state);
let active_ipc_connections = active_ipc_connections.clone();
let daemon_idle = cfg.daemon_idle;
tokio::spawn(async move {
let tick = (daemon_idle / 4).max(Duration::from_millis(250));
@@ -345,19 +352,22 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
ticker.tick().await;
let ipc_is_idle =
lock_activity(&activity).should_exit_for_idle(Instant::now(), daemon_idle);
if !ipc_is_idle {
let last = match activity.lock() {
Ok(a) => *a,
Err(p) => *p.into_inner(),
};
if last.elapsed() < daemon_idle {
continue;
}
// IPC liveness (open-connection count + idle window) is already
// enforced above by `should_exit_for_idle`, which reads
// `IpcActivityState` under one lock and returns true only when
// there are zero connections AND the idle interval has elapsed.
// Here we additionally hold the daemon alive while any browser
// is paired or any session is live (design §3.2, M4/M5
// registries).
if !state.browsers.is_empty() || !state.sessions.is_empty() {
// Bridge from M2/M3 (which only knew about connection
// counts) to M4/M5 (browser + session registries):
// hold the daemon alive while any IPC client is
// connected, any browser is paired, or any session
// is live (design §3.2).
if active_ipc_connections.load(Ordering::SeqCst) > 0
|| !state.browsers.is_empty()
|| !state.sessions.is_empty()
{
continue;
}
info!(
@@ -672,55 +682,10 @@ fn restart_start_args(cfg: &DaemonConfig) -> StartArgs {
}
}
#[derive(Debug)]
struct IpcActivityState {
last_activity: Instant,
active_connections: usize,
}
impl IpcActivityState {
fn new(now: Instant) -> Self {
Self {
last_activity: now,
active_connections: 0,
}
fn record_activity(activity: &Arc<Mutex<Instant>>) {
if let Ok(mut a) = activity.lock() {
*a = Instant::now();
}
fn touch(&mut self, now: Instant) {
self.last_activity = now;
}
fn opened(&mut self, now: Instant) {
self.active_connections = self.active_connections.saturating_add(1);
self.last_activity = now;
}
fn closed(&mut self, now: Instant) {
self.active_connections = self.active_connections.saturating_sub(1);
self.last_activity = now;
}
fn should_exit_for_idle(&self, now: Instant, idle: Duration) -> bool {
self.active_connections == 0 && now.saturating_duration_since(self.last_activity) >= idle
}
}
fn lock_activity(
activity: &Arc<Mutex<IpcActivityState>>,
) -> std::sync::MutexGuard<'_, IpcActivityState> {
activity.lock().unwrap_or_else(|poison| poison.into_inner())
}
fn record_activity(activity: &Arc<Mutex<IpcActivityState>>) {
lock_activity(activity).touch(Instant::now());
}
fn record_ipc_open(activity: &Arc<Mutex<IpcActivityState>>) {
lock_activity(activity).opened(Instant::now());
}
fn record_ipc_close(activity: &Arc<Mutex<IpcActivityState>>) {
lock_activity(activity).closed(Instant::now());
}
/// Initialise tracing for the daemon: write a daily-rotated file in
@@ -1103,37 +1068,4 @@ mod tests {
assert_eq!(args.session_idle, Some(Duration::from_secs(11)));
assert_eq!(args.daemon_idle, Some(Duration::from_secs(22)));
}
#[test]
fn ipc_close_restarts_the_idle_window_atomically() {
let started = Instant::now();
let idle = Duration::from_secs(10);
let mut activity = IpcActivityState::new(started);
activity.opened(started);
let closed_at = started + Duration::from_secs(30);
assert!(
!activity.should_exit_for_idle(closed_at, idle),
"an open IPC connection must hold the daemon alive"
);
activity.closed(closed_at);
assert_eq!(activity.active_connections, 0);
assert_eq!(activity.last_activity, closed_at);
assert!(!activity.should_exit_for_idle(closed_at, idle));
assert!(!activity.should_exit_for_idle(closed_at + idle - Duration::from_millis(1), idle));
assert!(activity.should_exit_for_idle(closed_at + idle, idle));
}
#[test]
fn ipc_activity_keeps_the_daemon_alive_until_a_full_idle_interval_passes() {
let started = Instant::now();
let idle = Duration::from_secs(10);
let mut activity = IpcActivityState::new(started);
let request_at = started + Duration::from_secs(20);
activity.touch(request_at);
assert!(!activity.should_exit_for_idle(request_at + Duration::from_secs(9), idle));
assert!(activity.should_exit_for_idle(request_at + idle, idle));
}
}
+1 -1
View File
@@ -16,7 +16,7 @@ use super::start::DaemonConfig;
use super::ws::WsHandle;
pub const DAEMON_VERSION: &str = env!("CARGO_PKG_VERSION");
pub const PROTOCOL_VERSION: &str = "1.1";
pub const PROTOCOL_VERSION: &str = "1.0";
/// Lowest **protocol** version peers must speak (e.g. `"1.0"`).
pub const MIN_COMPATIBLE_PROTOCOL: &str = "1.0";
/// Legacy app-semver floor used only when `HandshakeResult.min_compatible_peer`
+10 -10
View File
@@ -108,12 +108,12 @@ async fn send_handshake_with_floors(
async fn handshake_ok_when_protocol_matches() {
let (handle, _sock) = spawn_daemon().await;
let mut ws = open_ws(handle.ws_addr()).await;
let resp = send_handshake(&mut ws, "1.1", env!("CARGO_PKG_VERSION")).await;
let resp = send_handshake(&mut ws, "1.0", env!("CARGO_PKG_VERSION")).await;
let result: HandshakeResult = match resp.body {
ResponseBody::Ok(v) => serde_json::from_value(v).unwrap(),
ResponseBody::Err(e) => panic!("expected ok handshake, got {e:?}"),
};
assert_eq!(result.protocol_version, "1.1");
assert_eq!(result.protocol_version, "1.0");
assert_eq!(
result
.min_compatible_peer
@@ -135,7 +135,7 @@ async fn handshake_ok_when_app_versions_differ_but_protocol_matches() {
let (handle, _sock) = spawn_daemon().await;
let mut ws = open_ws(handle.ws_addr()).await;
let resp =
send_handshake_with_floors(&mut ws, "1.1", "9.9.9", Some("0.0.0"), Some("1.1")).await;
send_handshake_with_floors(&mut ws, "1.0", "9.9.9", Some("0.0.0"), Some("1.0")).await;
match resp.body {
ResponseBody::Ok(_) => {}
other => panic!("expected ok when protocol matches, got {other:?}"),
@@ -149,10 +149,10 @@ async fn handshake_skew_when_protocol_minor_differs() {
let mut ws = open_ws(handle.ws_addr()).await;
let resp = send_handshake_with_floors(
&mut ws,
"1.2",
"1.1",
env!("CARGO_PKG_VERSION"),
Some("0.0.0"),
Some("1.1"),
Some("1.0"),
)
.await;
match resp.body {
@@ -210,7 +210,7 @@ async fn handshake_legacy_ext_without_protocol_floor_still_ok() {
let mut ws = open_ws(handle.ws_addr()).await;
let resp = send_handshake_with_floors(
&mut ws,
"1.1",
"1.0",
env!("CARGO_PKG_VERSION"),
Some("0.1.0"),
None,
@@ -235,7 +235,7 @@ async fn status_surfaces_version_skew_for_skewed_browser() {
browser_name: "chrome".into(),
browser_version: "131.0".into(),
extension_version: "9.9.9".into(),
extension_protocol_version: "1.2".into(),
extension_protocol_version: "1.1".into(),
label: "Older".into(),
sink: bsk::daemon::browsers::BrowserSink { tx },
pending: Mutex::new(bsk::daemon::browsers::Pending::default()),
@@ -263,8 +263,8 @@ async fn status_surfaces_version_skew_for_skewed_browser() {
.iter()
.find(|s| s.instance_id == "skew-only-test")
.expect("status must list our skew client");
assert_eq!(skew.client_protocol_version, "1.2");
assert_eq!(skew.server_protocol_version, "1.1");
assert_eq!(skew.client_protocol_version, "1.1");
assert_eq!(skew.server_protocol_version, "1.0");
assert_eq!(skew.client_version, "9.9.9");
let entry = status
.browsers
@@ -281,7 +281,7 @@ async fn handshake_rejects_when_local_below_peer_min_compatible_protocol() {
let mut ws = open_ws(handle.ws_addr()).await;
let resp = send_handshake_with_floors(
&mut ws,
"1.1",
"1.0",
env!("CARGO_PKG_VERSION"),
Some("0.0.0"),
Some("99.0.0"),
@@ -1,327 +0,0 @@
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use bsk::daemon::info::DaemonInfo;
use bsk::daemon::paths::BSK_HOME_ENV;
use bsk::daemon::{self, DaemonConfig};
use bsk::ipc_client::IpcClient;
use bsk_protocol::system::{HandshakeParams, HandshakeResult};
use bsk_protocol::tools::{
RecordStopResult, RecordedTrace, RecorderInfo, SessionStartParams, SessionStartResult,
StopReason, TraceEntry, TraceStateV3, TraceV3, VOM_FORMAT_VERSION,
};
use bsk_protocol::{BrowserPeerInfo, Frame, Method, RequestFrame, ResponseBody, ResponseFrame};
use futures_util::{SinkExt, StreamExt};
use serde_json::json;
use tokio::process::Command;
use tokio_tungstenite::tungstenite::handshake::client::generate_key;
use tokio_tungstenite::tungstenite::http::Request;
use tokio_tungstenite::tungstenite::protocol::Message;
const TEST_EXT_ID: &str = "abcdefghijklmnopabcdefghijklmnop";
type Ws =
tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>>;
fn bsk_bin() -> PathBuf {
PathBuf::from(env!("CARGO_BIN_EXE_bsk"))
}
async fn connect_extension(addr: std::net::SocketAddr) -> Ws {
let origin = format!("chrome-extension://{TEST_EXT_ID}");
let request = Request::builder()
.method("GET")
.uri(format!("ws://{addr}/"))
.header("Host", addr.to_string())
.header("Upgrade", "websocket")
.header("Connection", "Upgrade")
.header("Sec-WebSocket-Version", "13")
.header("Sec-WebSocket-Key", generate_key())
.header("Origin", origin)
.body(())
.unwrap();
tokio_tungstenite::connect_async(request).await.unwrap().0
}
async fn handshake(ws: &mut Ws) {
let request = RequestFrame {
id: "handshake".into(),
method: Method::SystemHandshake,
params: Some(
serde_json::to_value(HandshakeParams {
client: "browser-skill-extension".into(),
version: "0.1.10".parse().unwrap(),
protocol_version: "1.1".into(),
instance_id: TEST_EXT_ID.into(),
browser: BrowserPeerInfo {
name: "chrome".into(),
version: "131.0".into(),
},
min_compatible_peer: Some("0.1.10".parse().unwrap()),
min_compatible_protocol: Some("1.1".into()),
label: "Record export recovery test".into(),
})
.unwrap(),
),
};
ws.send(Message::Text(serde_json::to_string(&request).unwrap()))
.await
.unwrap();
let Message::Text(response) = ws.next().await.unwrap().unwrap() else {
panic!("expected text handshake response");
};
let response: ResponseFrame = serde_json::from_str(&response).unwrap();
let ResponseBody::Ok(value) = response.body else {
panic!("handshake failed");
};
let _: HandshakeResult = serde_json::from_value(value).unwrap();
}
fn sample_trace() -> TraceV3 {
TraceV3 {
version: 3,
purpose: Some("export recovery".into()),
started_at: Some("2026-08-12T10:00:00Z".into()),
recorded_at: "2026-08-12T10:01:00Z".into(),
stopped_by: StopReason::CliStop,
entry: TraceEntry {
start_url: "https://example.com/".into(),
},
recorder: RecorderInfo {
bsk: "0.1.10".into(),
vom: VOM_FORMAT_VERSION,
},
states: vec![TraceStateV3 {
id: "s1".into(),
url: "https://example.com/".into(),
title: Some("Example".into()),
body: "@vom 1\nL1 page".into(),
truncated: false,
}],
steps: vec![],
}
}
fn run_extension(
mut ws: Ws,
record_stop_calls: Arc<AtomicUsize>,
session_stop_calls: Arc<AtomicUsize>,
) {
tokio::spawn(async move {
while let Some(Ok(Message::Text(text))) = ws.next().await {
let Ok(Frame::Request(request)) = serde_json::from_str::<Frame>(&text) else {
continue;
};
let body = match request.method {
Method::ToolSessionStart => {
let _: SessionStartParams =
serde_json::from_value(request.params.clone().unwrap()).unwrap();
ResponseBody::Ok(
serde_json::to_value(SessionStartResult {
agent_window_id: Some(100),
})
.unwrap(),
)
}
Method::ToolRecordStop => {
record_stop_calls.fetch_add(1, Ordering::SeqCst);
ResponseBody::Ok(
serde_json::to_value(RecordStopResult {
trace: RecordedTrace::V3(sample_trace()),
})
.unwrap(),
)
}
Method::ToolSessionStop => {
session_stop_calls.fetch_add(1, Ordering::SeqCst);
ResponseBody::Ok(json!({}))
}
_ => ResponseBody::Err(bsk_protocol::RpcError {
code: bsk_protocol::ErrorCode::ProtocolError,
message: format!("unexpected method {:?}", request.method),
data: None,
}),
};
let response = ResponseFrame {
id: request.id,
body,
};
ws.send(Message::Text(serde_json::to_string(&response).unwrap()))
.await
.unwrap();
}
});
}
async fn start_session(sock: &Path) -> String {
#[derive(serde::Deserialize)]
struct StartResult {
session_id: String,
}
let mut client = IpcClient::connect(sock).await.unwrap();
client
.call::<(), StartResult>(
"session-start",
Method::SessionStart,
None,
Duration::from_secs(5),
)
.await
.unwrap()
.unwrap()
.session_id
}
fn write_record_session(home: &Path, session_id: &str) {
std::fs::create_dir_all(home).unwrap();
std::fs::write(
home.join("record-session.json"),
format!("{{\"session_id\":\"{session_id}\",\"started_at\":\"1\"}}\n"),
)
.unwrap();
}
async fn run_record_stop(home: &Path, output: &Path) -> std::process::Output {
Command::new(bsk_bin())
.arg("record")
.arg("stop")
.arg("--output")
.arg(output)
.env(BSK_HOME_ENV, home)
.output()
.await
.unwrap()
}
async fn setup_daemon(temp: &Path) -> (bsk::daemon::DaemonHandle, PathBuf, PathBuf) {
let home = temp.join("bsk-home");
let sock = temp.join("daemon.sock");
let daemon = daemon::run(DaemonConfig::new(0), Some(sock.clone()))
.await
.unwrap();
bsk::daemon::info::write_to_path(
&DaemonInfo::now(
std::process::id(),
sock.clone(),
daemon.ws_addr().port(),
env!("CARGO_PKG_VERSION"),
),
&home.join("daemon.json"),
)
.unwrap();
(daemon, home, sock)
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn existing_trace_json_file_is_rejected_before_record_stop() {
let temp = tempfile::Builder::new()
.prefix("bsk-record-json-")
.tempdir()
.unwrap();
let (daemon, home, sock) = setup_daemon(temp.path()).await;
let output = temp.path().join("trace.json");
std::fs::write(&output, "{}\n").unwrap();
let mut extension = connect_extension(daemon.ws_addr()).await;
handshake(&mut extension).await;
let record_stop_calls = Arc::new(AtomicUsize::new(0));
let session_stop_calls = Arc::new(AtomicUsize::new(0));
run_extension(
extension,
Arc::clone(&record_stop_calls),
Arc::clone(&session_stop_calls),
);
let session_id = start_session(&sock).await;
write_record_session(&home, &session_id);
let first = run_record_stop(&home, &output).await;
let stderr = String::from_utf8_lossy(&first.stderr);
assert!(
!first.status.success(),
"legacy JSON --output should fail: {stderr}"
);
assert!(
stderr.contains("JSON file") || stderr.contains("bundle directory"),
"{stderr}"
);
let state: serde_json::Value =
serde_json::from_str(&std::fs::read_to_string(home.join("record-session.json")).unwrap())
.unwrap();
assert_eq!(state["session_id"], session_id);
assert_eq!(
record_stop_calls.load(Ordering::SeqCst),
0,
"existing trace.json must be rejected before the extension returns the Trace"
);
assert_eq!(session_stop_calls.load(Ordering::SeqCst), 0);
assert_eq!(std::fs::read_to_string(&output).unwrap(), "{}\n");
daemon.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn failed_bundle_export_can_be_recovered() {
let temp = tempfile::Builder::new()
.prefix("bsk-record-recov-")
.tempdir()
.unwrap();
let (daemon, home, sock) = setup_daemon(temp.path()).await;
let output = temp.path().join("trace");
std::fs::create_dir_all(&output).unwrap();
std::fs::write(output.join("states"), "not a directory\n").unwrap();
let mut extension = connect_extension(daemon.ws_addr()).await;
handshake(&mut extension).await;
let record_stop_calls = Arc::new(AtomicUsize::new(0));
let session_stop_calls = Arc::new(AtomicUsize::new(0));
run_extension(
extension,
Arc::clone(&record_stop_calls),
Arc::clone(&session_stop_calls),
);
let session_id = start_session(&sock).await;
write_record_session(&home, &session_id);
let first = run_record_stop(&home, &output).await;
let stderr = String::from_utf8_lossy(&first.stderr);
assert!(
!first.status.success(),
"blocked states path should fail export: {stderr}"
);
assert!(
home.join("record-recovery.json").exists(),
"completed Trace must be saved before the failed export"
);
assert!(
!home.join("record-session.json").exists(),
"session state is cleared after the extension returns the Trace"
);
assert_eq!(record_stop_calls.load(Ordering::SeqCst), 1);
assert_eq!(session_stop_calls.load(Ordering::SeqCst), 1);
std::fs::remove_file(output.join("states")).unwrap();
let second = run_record_stop(&home, &output).await;
assert!(
second.status.success(),
"recovery export failed: {}",
String::from_utf8_lossy(&second.stderr)
);
assert_eq!(
record_stop_calls.load(Ordering::SeqCst),
1,
"recovery must not call the extension again"
);
assert!(output.join("trace.json").exists());
assert!(output.join("states/s1.txt").exists());
assert!(
!home.join("record-recovery.json").exists(),
"successful export must delete recovery data"
);
daemon.shutdown().await;
}
-271
View File
@@ -1,271 +0,0 @@
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use bsk::cli::record_state;
use bsk::daemon::info::DaemonInfo;
use bsk::daemon::paths::BSK_HOME_ENV;
use bsk::daemon::{self, DaemonConfig};
use bsk::ipc_client::IpcClient;
use bsk_protocol::system::{HandshakeParams, HandshakeResult};
use bsk_protocol::tools::{
RecordStopResult, RecordedTrace, RecorderInfo, SessionStartParams, SessionStartResult,
StopReason, TraceEntry, TraceStateV3, TraceV3, VOM_FORMAT_VERSION,
};
use bsk_protocol::{
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
};
use futures_util::{SinkExt, StreamExt};
use serde_json::json;
use tokio::process::Command;
use tokio_tungstenite::tungstenite::handshake::client::generate_key;
use tokio_tungstenite::tungstenite::http::Request;
use tokio_tungstenite::tungstenite::protocol::Message;
const TEST_EXT_ID: &str = "abcdefghijklmnopabcdefghijklmnop";
type Ws =
tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>>;
fn bsk_bin() -> PathBuf {
PathBuf::from(env!("CARGO_BIN_EXE_bsk"))
}
async fn connect_extension(addr: std::net::SocketAddr) -> Ws {
let origin = format!("chrome-extension://{TEST_EXT_ID}");
let request = Request::builder()
.method("GET")
.uri(format!("ws://{addr}/"))
.header("Host", addr.to_string())
.header("Upgrade", "websocket")
.header("Connection", "Upgrade")
.header("Sec-WebSocket-Version", "13")
.header("Sec-WebSocket-Key", generate_key())
.header("Origin", origin)
.body(())
.unwrap();
tokio_tungstenite::connect_async(request).await.unwrap().0
}
async fn handshake(ws: &mut Ws) {
let request = RequestFrame {
id: "handshake".into(),
method: Method::SystemHandshake,
params: Some(
serde_json::to_value(HandshakeParams {
client: "browser-skill-extension".into(),
version: "0.1.10".parse().unwrap(),
protocol_version: "1.1".into(),
instance_id: TEST_EXT_ID.into(),
browser: BrowserPeerInfo {
name: "chrome".into(),
version: "131.0".into(),
},
min_compatible_peer: Some("0.1.10".parse().unwrap()),
min_compatible_protocol: Some("1.1".into()),
label: "Record stop retry test".into(),
})
.unwrap(),
),
};
ws.send(Message::Text(serde_json::to_string(&request).unwrap()))
.await
.unwrap();
let Message::Text(response) = ws.next().await.unwrap().unwrap() else {
panic!("expected text handshake response");
};
let response: ResponseFrame = serde_json::from_str(&response).unwrap();
let ResponseBody::Ok(value) = response.body else {
panic!("handshake failed");
};
let _: HandshakeResult = serde_json::from_value(value).unwrap();
}
fn sample_trace() -> TraceV3 {
TraceV3 {
version: 3,
purpose: Some("retry record stop".into()),
started_at: Some("2026-08-12T10:00:00Z".into()),
recorded_at: "2026-08-12T10:01:00Z".into(),
stopped_by: StopReason::CliStop,
entry: TraceEntry {
start_url: "https://example.com/".into(),
},
recorder: RecorderInfo {
bsk: "0.1.10".into(),
vom: VOM_FORMAT_VERSION,
},
states: vec![TraceStateV3 {
id: "s1".into(),
url: "https://example.com/".into(),
title: Some("Example".into()),
body: "@vom 1\nL1 page".into(),
truncated: false,
}],
steps: vec![],
}
}
fn run_extension(
mut ws: Ws,
record_stop_calls: Arc<AtomicUsize>,
session_stop_calls: Arc<AtomicUsize>,
) {
tokio::spawn(async move {
while let Some(Ok(Message::Text(text))) = ws.next().await {
let Ok(Frame::Request(request)) = serde_json::from_str::<Frame>(&text) else {
continue;
};
let body = match request.method {
Method::ToolSessionStart => {
let _: SessionStartParams =
serde_json::from_value(request.params.clone().unwrap()).unwrap();
ResponseBody::Ok(
serde_json::to_value(SessionStartResult {
agent_window_id: Some(100),
})
.unwrap(),
)
}
Method::ToolRecordStop => {
if record_stop_calls.fetch_add(1, Ordering::SeqCst) == 0 {
ResponseBody::Err(RpcError {
code: ErrorCode::ProtocolError,
message: "recording remains active; retry record stop".into(),
data: Some(json!({ "recording": true, "retryable": true })),
})
} else {
ResponseBody::Ok(
serde_json::to_value(RecordStopResult {
trace: RecordedTrace::V3(sample_trace()),
})
.unwrap(),
)
}
}
Method::ToolSessionStop => {
session_stop_calls.fetch_add(1, Ordering::SeqCst);
ResponseBody::Ok(json!({}))
}
_ => ResponseBody::Err(RpcError {
code: ErrorCode::ProtocolError,
message: format!("unexpected method {:?}", request.method),
data: None,
}),
};
let response = ResponseFrame {
id: request.id,
body,
};
ws.send(Message::Text(serde_json::to_string(&response).unwrap()))
.await
.unwrap();
}
});
}
async fn start_session(sock: &Path) -> String {
#[derive(serde::Deserialize)]
struct StartResult {
session_id: String,
}
let mut client = IpcClient::connect(sock).await.unwrap();
client
.call::<(), StartResult>(
"session-start",
Method::SessionStart,
None,
Duration::from_secs(5),
)
.await
.unwrap()
.unwrap()
.session_id
}
async fn run_record_stop(home: &Path, output: &Path) -> std::process::Output {
Command::new(bsk_bin())
.arg("record")
.arg("stop")
.arg("--output")
.arg(output)
.env(BSK_HOME_ENV, home)
.output()
.await
.unwrap()
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn failed_record_stop_keeps_state_and_session_for_retry() {
// Keep the Unix-domain socket below macOS's SUN_LEN even from a deep worktree.
let temp = tempfile::Builder::new()
.prefix("bsk-record-stop-")
.tempdir()
.unwrap();
let home = temp.path().join("bsk-home");
let sock = temp.path().join("daemon.sock");
let output = temp.path().join("trace");
let daemon = daemon::run(DaemonConfig::new(0), Some(sock.clone()))
.await
.unwrap();
bsk::daemon::info::write_to_path(
&DaemonInfo::now(
std::process::id(),
sock.clone(),
daemon.ws_addr().port(),
env!("CARGO_PKG_VERSION"),
),
&home.join("daemon.json"),
)
.unwrap();
let mut extension = connect_extension(daemon.ws_addr()).await;
handshake(&mut extension).await;
let record_stop_calls = Arc::new(AtomicUsize::new(0));
let session_stop_calls = Arc::new(AtomicUsize::new(0));
run_extension(
extension,
Arc::clone(&record_stop_calls),
Arc::clone(&session_stop_calls),
);
let session_id = start_session(&sock).await;
unsafe {
std::env::set_var(BSK_HOME_ENV, &home);
}
record_state::write(&session_id).unwrap();
let first = run_record_stop(&home, &output).await;
assert!(
!first.status.success(),
"first stop unexpectedly succeeded: {}",
String::from_utf8_lossy(&first.stdout)
);
assert_eq!(record_state::read().unwrap().session_id, session_id);
assert_eq!(record_stop_calls.load(Ordering::SeqCst), 1);
assert_eq!(
session_stop_calls.load(Ordering::SeqCst),
0,
"failed record stop must leave the session active"
);
let second = run_record_stop(&home, &output).await;
assert!(
second.status.success(),
"retry failed: {}",
String::from_utf8_lossy(&second.stderr)
);
assert!(record_state::read().is_err());
assert_eq!(record_stop_calls.load(Ordering::SeqCst), 2);
assert_eq!(session_stop_calls.load(Ordering::SeqCst), 1);
assert!(output.join("trace.json").exists());
assert!(output.join("states/s1.txt").exists());
unsafe {
std::env::remove_var(BSK_HOME_ENV);
}
daemon.shutdown().await;
}
+1 -1
View File
@@ -61,7 +61,7 @@ fn bsk_status_json_returns_structured_payload() {
assert!(parsed["pid"].as_u64().unwrap() > 0);
assert!(!parsed["daemon_version"].as_str().unwrap().is_empty());
assert_eq!(parsed["protocol_version"], "1.1");
assert_eq!(parsed["protocol_version"], "1.0");
assert!(parsed["sock_path"].as_str().is_some());
assert_eq!(parsed["browsers"], serde_json::json!([]));
assert_eq!(parsed["sessions"], serde_json::json!([]));
+2 -2
View File
@@ -53,7 +53,7 @@ pub async fn send_handshake(
let params = HandshakeParams {
client: "browser-skill-extension".into(),
version: "0.1.0-dev.0".parse().unwrap(),
protocol_version: "1.1".into(),
protocol_version: "1.0".into(),
instance_id: instance_id.into(),
browser: BrowserPeerInfo {
name: "chrome".into(),
@@ -91,7 +91,7 @@ async fn ws_handshake_registers_browser_in_state() {
let mut ws = connect_ext(handle.ws_addr(), &origin).await;
let result = send_handshake(&mut ws, TEST_EXT_ID).await;
assert_eq!(result.server, "browser-skill-daemon");
assert_eq!(result.protocol_version, "1.1");
assert_eq!(result.protocol_version, "1.0");
let state = handle.state();
let browsers = state.browsers.snapshot();
@@ -7,18 +7,10 @@
],
"properties": {
"trace": {
"$ref": "#/definitions/RecordedTrace"
"$ref": "#/definitions/Trace"
}
},
"definitions": {
"FillCommit": {
"type": "string",
"enum": [
"enter",
"suggestion",
"blur"
]
},
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
@@ -29,19 +21,7 @@
"shift"
]
},
"NavigationCause": {
"type": "string",
"enum": [
"user_typed",
"link",
"form_submit",
"reload",
"history",
"script",
"browser"
]
},
"PageRefV2": {
"PageRef": {
"description": "Page context dictionary entry — referenced by steps via `page` id.",
"type": "object",
"required": [
@@ -63,42 +43,7 @@
}
}
},
"RecordedTrace": {
"if": {
"required": [
"version"
],
"properties": {
"version": {
"type": "integer"
}
}
},
"then": {
"$ref": "#/definitions/TraceV3"
},
"else": {
"$ref": "#/definitions/TraceV2"
}
},
"RecorderInfo": {
"type": "object",
"required": [
"bsk",
"vom"
],
"properties": {
"bsk": {
"type": "string"
},
"vom": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
}
}
},
"SelectedOptionV2": {
"SelectedOption": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
@@ -116,53 +61,11 @@
}
}
},
"SelectedOptionV3": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
"value"
],
"properties": {
"label": {
"type": [
"string",
"null"
]
},
"value": {
"type": "string"
}
}
},
"StepEffectV2": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
}
}
},
"StepResultV3": {
"type": "object",
"required": [
"state"
],
"properties": {
"state": {
"type": "string"
}
}
},
"StepV2": {
"description": "One recorded user action — discriminated union by `op` (v2).",
"Step": {
"description": "One recorded user action — discriminated union by `op`.",
"oneOf": [
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -174,7 +77,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
@@ -202,7 +105,7 @@
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -214,7 +117,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
@@ -237,12 +140,12 @@
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
"$ref": "#/definitions/TargetDescriptor"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -254,7 +157,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
@@ -277,12 +180,12 @@
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
"$ref": "#/definitions/TargetDescriptor"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -295,7 +198,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
@@ -324,7 +227,7 @@
]
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
"$ref": "#/definitions/TargetDescriptor"
},
"value": {
"type": "string"
@@ -332,7 +235,7 @@
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -345,7 +248,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
@@ -370,16 +273,16 @@
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV2"
"$ref": "#/definitions/SelectedOption"
}
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
"$ref": "#/definitions/TargetDescriptor"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -391,7 +294,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
@@ -428,7 +331,7 @@
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV2"
"$ref": "#/definitions/TargetDescriptor"
},
{
"type": "null"
@@ -439,324 +342,21 @@
}
]
},
"StepV3": {
"description": "One recorded user action — discriminated union by `op`.",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"cause",
"id",
"op",
"result",
"state",
"to"
],
"properties": {
"cause": {
"$ref": "#/definitions/NavigationCause"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"commit",
"id",
"op",
"result",
"state",
"target",
"value"
],
"properties": {
"commit": {
"$ref": "#/definitions/FillCommit"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"redacted": {
"type": "boolean"
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV3"
}
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV3"
},
{
"type": "null"
}
]
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"scroll"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
"StepEffect": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
}
]
}
},
"StopReason": {
"type": "string",
"enum": [
"user_finish",
"cli_stop"
]
},
"TargetDescriptorV2": {
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
"TargetDescriptor": {
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
"type": "object",
"required": [
"tag"
@@ -797,83 +397,8 @@
}
}
},
"TargetDescriptorV3": {
"description": "Stable semantic handle for an interacted element within a page observation.",
"type": "object",
"properties": {
"ctx": {
"type": [
"string",
"null"
]
},
"name": {
"type": [
"string",
"null"
]
},
"ref": {
"type": [
"string",
"null"
]
},
"role": {
"type": [
"string",
"null"
]
},
"unmatched": {
"type": "boolean"
}
}
},
"TraceEntry": {
"type": "object",
"required": [
"start_url"
],
"properties": {
"start_url": {
"type": "string"
}
}
},
"TraceStateV3": {
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
"type": "object",
"required": [
"body",
"id",
"url"
],
"properties": {
"body": {
"description": "Full page observation (front matter + VOM body + annotations).",
"type": "string"
},
"id": {
"type": "string"
},
"title": {
"type": [
"string",
"null"
]
},
"truncated": {
"type": "boolean"
},
"url": {
"type": "string"
}
}
},
"TraceV2": {
"title": "TraceV2",
"description": "Persisted user-action trace exported by legacy `tool.record_stop` / `await`.\n\nUnknown extension fields are ignored so older traces remain readable. `states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace` classification.",
"Trace": {
"description": "Persisted user-action trace exported by `tool.record_stop` / `await`.",
"type": "object",
"required": [
"entry",
@@ -888,10 +413,11 @@
"pages": {
"type": "array",
"items": {
"$ref": "#/definitions/PageRefV2"
"$ref": "#/definitions/PageRef"
}
},
"purpose": {
"description": "Optional user-provided goal from `--purpose` (metadata only).",
"type": [
"string",
"null"
@@ -907,76 +433,25 @@
"null"
]
},
"states": false,
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/StepV2"
}
},
"version": {
"description": "Numeric version selects Trace v3. Legacy v2 envelopes omit this field.",
"not": {
"type": "integer"
"$ref": "#/definitions/Step"
}
}
}
},
"TraceV3": {
"description": "Wire trace returned by `tool.record_stop` / `await`.",
"TraceEntry": {
"description": "Recording entry point — first URL the flow starts from.",
"type": "object",
"required": [
"entry",
"recorded_at",
"recorder",
"states",
"steps",
"stopped_by",
"version"
"start_url"
],
"properties": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"purpose": {
"type": [
"string",
"null"
]
},
"recorded_at": {
"start_url": {
"type": "string"
},
"recorder": {
"$ref": "#/definitions/RecorderInfo"
},
"started_at": {
"type": [
"string",
"null"
]
},
"states": {
"type": "array",
"items": {
"$ref": "#/definitions/TraceStateV3"
}
},
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/StepV3"
}
},
"stopped_by": {
"$ref": "#/definitions/StopReason"
},
"version": {
"type": "integer",
"const": 3
}
},
"additionalProperties": false
}
}
}
}
@@ -6,36 +6,15 @@
"session_id"
],
"properties": {
"max_page_tokens": {
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 0.0
},
"purpose": {
"type": [
"string",
"null"
]
},
"redact_values": {
"type": [
"boolean",
"null"
]
},
"session_id": {
"type": "string"
},
"supports_tab_switch_steps": {
"description": "Client can decode the v3 `switch_tab` step variant.",
"type": [
"boolean",
"null"
]
},
"tab_id": {
"type": [
"integer",
@@ -43,15 +22,6 @@
],
"format": "int64"
},
"trace_version": {
"description": "Desired trace export format. Omitted means v2; `3` requests v3.",
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 0.0
},
"url": {
"type": [
"string",
@@ -7,18 +7,10 @@
],
"properties": {
"trace": {
"$ref": "#/definitions/RecordedTrace"
"$ref": "#/definitions/Trace"
}
},
"definitions": {
"FillCommit": {
"type": "string",
"enum": [
"enter",
"suggestion",
"blur"
]
},
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
@@ -29,19 +21,7 @@
"shift"
]
},
"NavigationCause": {
"type": "string",
"enum": [
"user_typed",
"link",
"form_submit",
"reload",
"history",
"script",
"browser"
]
},
"PageRefV2": {
"PageRef": {
"description": "Page context dictionary entry — referenced by steps via `page` id.",
"type": "object",
"required": [
@@ -63,42 +43,7 @@
}
}
},
"RecordedTrace": {
"if": {
"required": [
"version"
],
"properties": {
"version": {
"type": "integer"
}
}
},
"then": {
"$ref": "#/definitions/TraceV3"
},
"else": {
"$ref": "#/definitions/TraceV2"
}
},
"RecorderInfo": {
"type": "object",
"required": [
"bsk",
"vom"
],
"properties": {
"bsk": {
"type": "string"
},
"vom": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
}
}
},
"SelectedOptionV2": {
"SelectedOption": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
@@ -116,53 +61,11 @@
}
}
},
"SelectedOptionV3": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
"value"
],
"properties": {
"label": {
"type": [
"string",
"null"
]
},
"value": {
"type": "string"
}
}
},
"StepEffectV2": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
}
}
},
"StepResultV3": {
"type": "object",
"required": [
"state"
],
"properties": {
"state": {
"type": "string"
}
}
},
"StepV2": {
"description": "One recorded user action — discriminated union by `op` (v2).",
"Step": {
"description": "One recorded user action — discriminated union by `op`.",
"oneOf": [
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -174,7 +77,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
@@ -202,7 +105,7 @@
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -214,7 +117,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
@@ -237,12 +140,12 @@
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
"$ref": "#/definitions/TargetDescriptor"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -254,7 +157,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
@@ -277,12 +180,12 @@
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
"$ref": "#/definitions/TargetDescriptor"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -295,7 +198,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
@@ -324,7 +227,7 @@
]
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
"$ref": "#/definitions/TargetDescriptor"
},
"value": {
"type": "string"
@@ -332,7 +235,7 @@
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -345,7 +248,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
@@ -370,16 +273,16 @@
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV2"
"$ref": "#/definitions/SelectedOption"
}
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
"$ref": "#/definitions/TargetDescriptor"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -391,7 +294,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
@@ -428,7 +331,7 @@
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV2"
"$ref": "#/definitions/TargetDescriptor"
},
{
"type": "null"
@@ -439,324 +342,21 @@
}
]
},
"StepV3": {
"description": "One recorded user action — discriminated union by `op`.",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"cause",
"id",
"op",
"result",
"state",
"to"
],
"properties": {
"cause": {
"$ref": "#/definitions/NavigationCause"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"commit",
"id",
"op",
"result",
"state",
"target",
"value"
],
"properties": {
"commit": {
"$ref": "#/definitions/FillCommit"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"redacted": {
"type": "boolean"
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV3"
}
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV3"
},
{
"type": "null"
}
]
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"scroll"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
"StepEffect": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
}
]
}
},
"StopReason": {
"type": "string",
"enum": [
"user_finish",
"cli_stop"
]
},
"TargetDescriptorV2": {
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
"TargetDescriptor": {
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
"type": "object",
"required": [
"tag"
@@ -797,83 +397,8 @@
}
}
},
"TargetDescriptorV3": {
"description": "Stable semantic handle for an interacted element within a page observation.",
"type": "object",
"properties": {
"ctx": {
"type": [
"string",
"null"
]
},
"name": {
"type": [
"string",
"null"
]
},
"ref": {
"type": [
"string",
"null"
]
},
"role": {
"type": [
"string",
"null"
]
},
"unmatched": {
"type": "boolean"
}
}
},
"TraceEntry": {
"type": "object",
"required": [
"start_url"
],
"properties": {
"start_url": {
"type": "string"
}
}
},
"TraceStateV3": {
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
"type": "object",
"required": [
"body",
"id",
"url"
],
"properties": {
"body": {
"description": "Full page observation (front matter + VOM body + annotations).",
"type": "string"
},
"id": {
"type": "string"
},
"title": {
"type": [
"string",
"null"
]
},
"truncated": {
"type": "boolean"
},
"url": {
"type": "string"
}
}
},
"TraceV2": {
"title": "TraceV2",
"description": "Persisted user-action trace exported by legacy `tool.record_stop` / `await`.\n\nUnknown extension fields are ignored so older traces remain readable. `states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace` classification.",
"Trace": {
"description": "Persisted user-action trace exported by `tool.record_stop` / `await`.",
"type": "object",
"required": [
"entry",
@@ -888,10 +413,11 @@
"pages": {
"type": "array",
"items": {
"$ref": "#/definitions/PageRefV2"
"$ref": "#/definitions/PageRef"
}
},
"purpose": {
"description": "Optional user-provided goal from `--purpose` (metadata only).",
"type": [
"string",
"null"
@@ -907,76 +433,25 @@
"null"
]
},
"states": false,
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/StepV2"
}
},
"version": {
"description": "Numeric version selects Trace v3. Legacy v2 envelopes omit this field.",
"not": {
"type": "integer"
"$ref": "#/definitions/Step"
}
}
}
},
"TraceV3": {
"description": "Wire trace returned by `tool.record_stop` / `await`.",
"TraceEntry": {
"description": "Recording entry point — first URL the flow starts from.",
"type": "object",
"required": [
"entry",
"recorded_at",
"recorder",
"states",
"steps",
"stopped_by",
"version"
"start_url"
],
"properties": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"purpose": {
"type": [
"string",
"null"
]
},
"recorded_at": {
"start_url": {
"type": "string"
},
"recorder": {
"$ref": "#/definitions/RecorderInfo"
},
"started_at": {
"type": [
"string",
"null"
]
},
"states": {
"type": "array",
"items": {
"$ref": "#/definitions/TraceStateV3"
}
},
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/StepV3"
}
},
"stopped_by": {
"$ref": "#/definitions/StopReason"
},
"version": {
"type": "integer",
"const": 3
}
},
"additionalProperties": false
}
}
}
}
+76 -601
View File
@@ -1,31 +1,49 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "RecordedTrace",
"if": {
"required": [
"version"
],
"properties": {
"version": {
"type": "integer"
"title": "Trace",
"description": "Persisted user-action trace exported by `tool.record_stop` / `await`.",
"type": "object",
"required": [
"entry",
"pages",
"recorded_at",
"steps"
],
"properties": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"pages": {
"type": "array",
"items": {
"$ref": "#/definitions/PageRef"
}
},
"purpose": {
"description": "Optional user-provided goal from `--purpose` (metadata only).",
"type": [
"string",
"null"
]
},
"recorded_at": {
"description": "RFC 3339 timestamp when recording stopped.",
"type": "string"
},
"started_at": {
"type": [
"string",
"null"
]
},
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/Step"
}
}
},
"then": {
"$ref": "#/definitions/TraceV3"
},
"else": {
"$ref": "#/definitions/TraceV2"
},
"definitions": {
"FillCommit": {
"type": "string",
"enum": [
"enter",
"suggestion",
"blur"
]
},
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
@@ -36,19 +54,7 @@
"shift"
]
},
"NavigationCause": {
"type": "string",
"enum": [
"user_typed",
"link",
"form_submit",
"reload",
"history",
"script",
"browser"
]
},
"PageRefV2": {
"PageRef": {
"description": "Page context dictionary entry — referenced by steps via `page` id.",
"type": "object",
"required": [
@@ -70,24 +76,7 @@
}
}
},
"RecorderInfo": {
"type": "object",
"required": [
"bsk",
"vom"
],
"properties": {
"bsk": {
"type": "string"
},
"vom": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
}
}
},
"SelectedOptionV2": {
"SelectedOption": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
@@ -105,53 +94,11 @@
}
}
},
"SelectedOptionV3": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
"value"
],
"properties": {
"label": {
"type": [
"string",
"null"
]
},
"value": {
"type": "string"
}
}
},
"StepEffectV2": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
}
}
},
"StepResultV3": {
"type": "object",
"required": [
"state"
],
"properties": {
"state": {
"type": "string"
}
}
},
"StepV2": {
"description": "One recorded user action — discriminated union by `op` (v2).",
"Step": {
"description": "One recorded user action — discriminated union by `op`.",
"oneOf": [
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -163,7 +110,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
@@ -191,7 +138,7 @@
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -203,7 +150,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
@@ -226,12 +173,12 @@
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
"$ref": "#/definitions/TargetDescriptor"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -243,7 +190,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
@@ -266,12 +213,12 @@
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
"$ref": "#/definitions/TargetDescriptor"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -284,7 +231,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
@@ -313,7 +260,7 @@
]
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
"$ref": "#/definitions/TargetDescriptor"
},
"value": {
"type": "string"
@@ -321,7 +268,7 @@
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -334,7 +281,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
@@ -359,16 +306,16 @@
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV2"
"$ref": "#/definitions/SelectedOption"
}
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
"$ref": "#/definitions/TargetDescriptor"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -380,7 +327,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
@@ -417,7 +364,7 @@
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV2"
"$ref": "#/definitions/TargetDescriptor"
},
{
"type": "null"
@@ -428,324 +375,21 @@
}
]
},
"StepV3": {
"description": "One recorded user action — discriminated union by `op`.",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"cause",
"id",
"op",
"result",
"state",
"to"
],
"properties": {
"cause": {
"$ref": "#/definitions/NavigationCause"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"commit",
"id",
"op",
"result",
"state",
"target",
"value"
],
"properties": {
"commit": {
"$ref": "#/definitions/FillCommit"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"redacted": {
"type": "boolean"
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV3"
}
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV3"
},
{
"type": "null"
}
]
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"scroll"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
"StepEffect": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
}
]
}
},
"StopReason": {
"type": "string",
"enum": [
"user_finish",
"cli_stop"
]
},
"TargetDescriptorV2": {
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
"TargetDescriptor": {
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
"type": "object",
"required": [
"tag"
@@ -786,40 +430,8 @@
}
}
},
"TargetDescriptorV3": {
"description": "Stable semantic handle for an interacted element within a page observation.",
"type": "object",
"properties": {
"ctx": {
"type": [
"string",
"null"
]
},
"name": {
"type": [
"string",
"null"
]
},
"ref": {
"type": [
"string",
"null"
]
},
"role": {
"type": [
"string",
"null"
]
},
"unmatched": {
"type": "boolean"
}
}
},
"TraceEntry": {
"description": "Recording entry point — first URL the flow starts from.",
"type": "object",
"required": [
"start_url"
@@ -829,143 +441,6 @@
"type": "string"
}
}
},
"TraceStateV3": {
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
"type": "object",
"required": [
"body",
"id",
"url"
],
"properties": {
"body": {
"description": "Full page observation (front matter + VOM body + annotations).",
"type": "string"
},
"id": {
"type": "string"
},
"title": {
"type": [
"string",
"null"
]
},
"truncated": {
"type": "boolean"
},
"url": {
"type": "string"
}
}
},
"TraceV2": {
"title": "TraceV2",
"description": "Persisted user-action trace exported by legacy `tool.record_stop` / `await`.\n\nUnknown extension fields are ignored so older traces remain readable. `states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace` classification.",
"type": "object",
"required": [
"entry",
"pages",
"recorded_at",
"steps"
],
"properties": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"pages": {
"type": "array",
"items": {
"$ref": "#/definitions/PageRefV2"
}
},
"purpose": {
"type": [
"string",
"null"
]
},
"recorded_at": {
"description": "RFC 3339 timestamp when recording stopped.",
"type": "string"
},
"started_at": {
"type": [
"string",
"null"
]
},
"states": false,
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/StepV2"
}
},
"version": {
"description": "Numeric version selects Trace v3. Legacy v2 envelopes omit this field.",
"not": {
"type": "integer"
}
}
}
},
"TraceV3": {
"description": "Wire trace returned by `tool.record_stop` / `await`.",
"type": "object",
"required": [
"entry",
"recorded_at",
"recorder",
"states",
"steps",
"stopped_by",
"version"
],
"properties": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"purpose": {
"type": [
"string",
"null"
]
},
"recorded_at": {
"type": "string"
},
"recorder": {
"$ref": "#/definitions/RecorderInfo"
},
"started_at": {
"type": [
"string",
"null"
]
},
"states": {
"type": "array",
"items": {
"$ref": "#/definitions/TraceStateV3"
}
},
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/StepV3"
}
},
"stopped_by": {
"$ref": "#/definitions/StopReason"
},
"version": {
"type": "integer",
"const": 3
}
},
"additionalProperties": false
}
}
}
File diff suppressed because it is too large Load Diff
@@ -1,368 +0,0 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "StepV2",
"description": "One recorded user action — discriminated union by `op` (v2).",
"oneOf": [
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"to"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target",
"value"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"redacted": {
"type": [
"boolean",
"null"
]
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"selection",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV2"
}
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"page"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV2"
},
{
"type": "null"
}
]
}
}
}
],
"definitions": {
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
"enum": [
"alt",
"ctrl",
"meta",
"shift"
]
},
"SelectedOptionV2": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
"value"
],
"properties": {
"label": {
"type": [
"string",
"null"
]
},
"value": {
"type": "string"
}
}
},
"StepEffectV2": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
}
}
},
"TargetDescriptorV2": {
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
"type": "object",
"required": [
"tag"
],
"properties": {
"name": {
"type": [
"string",
"null"
]
},
"name_attr": {
"type": [
"string",
"null"
]
},
"nearby_label": {
"type": [
"string",
"null"
]
},
"placeholder": {
"type": [
"string",
"null"
]
},
"role": {
"type": [
"string",
"null"
]
},
"tag": {
"type": "string"
}
}
}
}
}
@@ -1,405 +0,0 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "StepV3",
"description": "One recorded user action — discriminated union by `op`.",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"cause",
"id",
"op",
"result",
"state",
"to"
],
"properties": {
"cause": {
"$ref": "#/definitions/NavigationCause"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"commit",
"id",
"op",
"result",
"state",
"target",
"value"
],
"properties": {
"commit": {
"$ref": "#/definitions/FillCommit"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"redacted": {
"type": "boolean"
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV3"
}
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV3"
},
{
"type": "null"
}
]
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"scroll"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
}
],
"definitions": {
"FillCommit": {
"type": "string",
"enum": [
"enter",
"suggestion",
"blur"
]
},
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
"enum": [
"alt",
"ctrl",
"meta",
"shift"
]
},
"NavigationCause": {
"type": "string",
"enum": [
"user_typed",
"link",
"form_submit",
"reload",
"history",
"script",
"browser"
]
},
"SelectedOptionV3": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
"value"
],
"properties": {
"label": {
"type": [
"string",
"null"
]
},
"value": {
"type": "string"
}
}
},
"StepResultV3": {
"type": "object",
"required": [
"state"
],
"properties": {
"state": {
"type": "string"
}
}
},
"TargetDescriptorV3": {
"description": "Stable semantic handle for an interacted element within a page observation.",
"type": "object",
"properties": {
"ctx": {
"type": [
"string",
"null"
]
},
"name": {
"type": [
"string",
"null"
]
},
"ref": {
"type": [
"string",
"null"
]
},
"role": {
"type": [
"string",
"null"
]
},
"unmatched": {
"type": "boolean"
}
}
}
}
}
-451
View File
@@ -1,451 +0,0 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "TraceV2",
"description": "Persisted user-action trace exported by legacy `tool.record_stop` / `await`.\n\nUnknown extension fields are ignored so older traces remain readable. `states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace` classification.",
"type": "object",
"required": [
"entry",
"pages",
"recorded_at",
"steps"
],
"properties": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"pages": {
"type": "array",
"items": {
"$ref": "#/definitions/PageRefV2"
}
},
"purpose": {
"type": [
"string",
"null"
]
},
"recorded_at": {
"description": "RFC 3339 timestamp when recording stopped.",
"type": "string"
},
"started_at": {
"type": [
"string",
"null"
]
},
"states": false,
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/StepV2"
}
},
"version": {
"description": "Numeric version selects Trace v3. Legacy v2 envelopes omit this field.",
"not": {
"type": "integer"
}
}
},
"definitions": {
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
"enum": [
"alt",
"ctrl",
"meta",
"shift"
]
},
"PageRefV2": {
"description": "Page context dictionary entry — referenced by steps via `page` id.",
"type": "object",
"required": [
"id",
"url"
],
"properties": {
"id": {
"type": "string"
},
"title": {
"type": [
"string",
"null"
]
},
"url": {
"type": "string"
}
}
},
"SelectedOptionV2": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
"value"
],
"properties": {
"label": {
"type": [
"string",
"null"
]
},
"value": {
"type": "string"
}
}
},
"StepEffectV2": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
}
}
},
"StepV2": {
"description": "One recorded user action — discriminated union by `op` (v2).",
"oneOf": [
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"to"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target",
"value"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"redacted": {
"type": [
"boolean",
"null"
]
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"selection",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV2"
}
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"page"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV2"
},
{
"type": "null"
}
]
}
}
}
]
},
"TargetDescriptorV2": {
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
"type": "object",
"required": [
"tag"
],
"properties": {
"name": {
"type": [
"string",
"null"
]
},
"name_attr": {
"type": [
"string",
"null"
]
},
"nearby_label": {
"type": [
"string",
"null"
]
},
"placeholder": {
"type": [
"string",
"null"
]
},
"role": {
"type": [
"string",
"null"
]
},
"tag": {
"type": "string"
}
}
},
"TraceEntry": {
"type": "object",
"required": [
"start_url"
],
"properties": {
"start_url": {
"type": "string"
}
}
}
}
}
-526
View File
@@ -1,526 +0,0 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "TraceV3",
"description": "Wire trace returned by `tool.record_stop` / `await`.",
"type": "object",
"required": [
"entry",
"recorded_at",
"recorder",
"states",
"steps",
"stopped_by",
"version"
],
"properties": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"purpose": {
"type": [
"string",
"null"
]
},
"recorded_at": {
"type": "string"
},
"recorder": {
"$ref": "#/definitions/RecorderInfo"
},
"started_at": {
"type": [
"string",
"null"
]
},
"states": {
"type": "array",
"items": {
"$ref": "#/definitions/TraceStateV3"
}
},
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/StepV3"
}
},
"stopped_by": {
"$ref": "#/definitions/StopReason"
},
"version": {
"type": "integer",
"const": 3
}
},
"additionalProperties": false,
"definitions": {
"FillCommit": {
"type": "string",
"enum": [
"enter",
"suggestion",
"blur"
]
},
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
"enum": [
"alt",
"ctrl",
"meta",
"shift"
]
},
"NavigationCause": {
"type": "string",
"enum": [
"user_typed",
"link",
"form_submit",
"reload",
"history",
"script",
"browser"
]
},
"RecorderInfo": {
"type": "object",
"required": [
"bsk",
"vom"
],
"properties": {
"bsk": {
"type": "string"
},
"vom": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
}
}
},
"SelectedOptionV3": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
"value"
],
"properties": {
"label": {
"type": [
"string",
"null"
]
},
"value": {
"type": "string"
}
}
},
"StepResultV3": {
"type": "object",
"required": [
"state"
],
"properties": {
"state": {
"type": "string"
}
}
},
"StepV3": {
"description": "One recorded user action — discriminated union by `op`.",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"cause",
"id",
"op",
"result",
"state",
"to"
],
"properties": {
"cause": {
"$ref": "#/definitions/NavigationCause"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"commit",
"id",
"op",
"result",
"state",
"target",
"value"
],
"properties": {
"commit": {
"$ref": "#/definitions/FillCommit"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"redacted": {
"type": "boolean"
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV3"
}
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV3"
},
{
"type": "null"
}
]
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"scroll"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
}
]
},
"StopReason": {
"type": "string",
"enum": [
"user_finish",
"cli_stop"
]
},
"TargetDescriptorV3": {
"description": "Stable semantic handle for an interacted element within a page observation.",
"type": "object",
"properties": {
"ctx": {
"type": [
"string",
"null"
]
},
"name": {
"type": [
"string",
"null"
]
},
"ref": {
"type": [
"string",
"null"
]
},
"role": {
"type": [
"string",
"null"
]
},
"unmatched": {
"type": "boolean"
}
}
},
"TraceEntry": {
"type": "object",
"required": [
"start_url"
],
"properties": {
"start_url": {
"type": "string"
}
}
},
"TraceStateV3": {
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
"type": "object",
"required": [
"body",
"id",
"url"
],
"properties": {
"body": {
"description": "Full page observation (front matter + VOM body + annotations).",
"type": "string"
},
"id": {
"type": "string"
},
"title": {
"type": [
"string",
"null"
]
},
"truncated": {
"type": "boolean"
},
"url": {
"type": "string"
}
}
}
}
}
+2 -6
View File
@@ -113,12 +113,8 @@ fn main() {
dump!(RequestHelpParams, "tool_request_help_params");
dump!(RequestHelpResult, "tool_request_help_result");
dump!(TraceV2, "trace_v2");
dump!(TraceV3, "trace_v3");
dump!(RecordedTrace, "trace");
dump!(StepV2, "trace_step_v2");
dump!(StepV3, "trace_step_v3");
dump!(RecordedStep, "trace_step");
dump!(Trace, "trace");
dump!(Step, "trace_step");
dump!(RecordStartParams, "tool_record_start_params");
dump!(RecordStartResult, "tool_record_start_result");
dump!(RecordStopParams, "tool_record_stop_params");
-3
View File
@@ -9,9 +9,6 @@ pub mod navigation;
pub mod network;
pub mod observation;
pub mod record;
mod record_common;
mod record_v2;
mod record_v3;
pub mod script;
pub mod session;
pub mod tabs;
File diff suppressed because it is too large Load Diff
@@ -1,7 +0,0 @@
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct TraceEntry {
pub start_url: String,
}
-330
View File
@@ -1,330 +0,0 @@
//! Trace v2 — record-only user-action log with `pages[]` and step `page` refs.
//!
//! Legacy wire format with no top-level `version` field.
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use super::interaction::KeyModifier;
use super::record_common::TraceEntry;
/// Stable semantic handle for an interacted element (v2).
///
/// `name` and `nearby_label` are **untrusted page text**.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct TargetDescriptorV2 {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub role: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
pub tag: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name_attr: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub placeholder: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub nearby_label: Option<String>,
}
/// Page context dictionary entry — referenced by steps via `page` id.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct PageRefV2 {
pub id: String,
pub url: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub title: Option<String>,
}
/// One selected option (`select` op).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct SelectedOptionV2 {
pub value: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub label: Option<String>,
}
/// Observed navigation after a step (objective fact only).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct StepEffectV2 {
/// Reference into `pages[]` for the destination page.
pub navigated_to: String,
}
/// Fields shared by every v2 step variant (flattened in JSON).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct StepCommonV2 {
pub id: u32,
/// Reference into `pages[]`.
pub page: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub effect: Option<StepEffectV2>,
}
/// One recorded user action — discriminated union by `op` (v2).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[serde(tag = "op", rename_all = "snake_case")]
pub enum StepV2 {
Navigate {
#[serde(flatten)]
common: StepCommonV2,
to: String,
},
Click {
#[serde(flatten)]
common: StepCommonV2,
target: TargetDescriptorV2,
},
Hover {
#[serde(flatten)]
common: StepCommonV2,
target: TargetDescriptorV2,
},
Fill {
#[serde(flatten)]
common: StepCommonV2,
target: TargetDescriptorV2,
value: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
redacted: Option<bool>,
},
Select {
#[serde(flatten)]
common: StepCommonV2,
target: TargetDescriptorV2,
selection: Vec<SelectedOptionV2>,
},
Press {
#[serde(flatten)]
common: StepCommonV2,
key: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
modifiers: Option<Vec<KeyModifier>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
target: Option<TargetDescriptorV2>,
},
}
/// Persisted user-action trace exported by legacy `tool.record_stop` / `await`.
///
/// Unknown extension fields are ignored so older traces remain readable.
/// `states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace`
/// classification — serde still ignores them, but the JSON Schema rejects them.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct TraceV2 {
/// RFC 3339 timestamp when recording stopped.
pub recorded_at: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub started_at: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub purpose: Option<String>,
pub entry: TraceEntry,
pub pages: Vec<PageRefV2>,
pub steps: Vec<StepV2>,
}
fn constrain_trace_v2_schema(mut schema: schemars::schema::Schema) -> schemars::schema::Schema {
let schemars::schema::Schema::Object(obj) = &mut schema else {
return schema;
};
obj.metadata().title = Some("TraceV2".into());
obj.metadata().description = Some(
"Persisted user-action trace exported by legacy `tool.record_stop` / `await`.\n\n\
Unknown extension fields are ignored so older traces remain readable. \
`states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace` classification."
.into(),
);
let object = obj.object.get_or_insert_with(Default::default);
object
.properties
.insert("states".into(), schemars::schema::Schema::Bool(false));
object.properties.insert(
"version".into(),
schemars::schema::SchemaObject {
metadata: Some(Box::new(schemars::schema::Metadata {
description: Some(
"Numeric version selects Trace v3. Legacy v2 envelopes omit this field.".into(),
),
..Default::default()
})),
subschemas: Some(Box::new(schemars::schema::SubschemaValidation {
not: Some(Box::new(
schemars::schema::SchemaObject {
instance_type: Some(schemars::schema::InstanceType::Integer.into()),
..Default::default()
}
.into(),
)),
..Default::default()
})),
..Default::default()
}
.into(),
);
schema
}
impl JsonSchema for TraceV2 {
fn schema_name() -> String {
"TraceV2".into()
}
fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::schema::Schema {
#[derive(JsonSchema)]
#[allow(dead_code)]
struct TraceV2Shape {
/// RFC 3339 timestamp when recording stopped.
recorded_at: String,
started_at: Option<String>,
purpose: Option<String>,
entry: TraceEntry,
pages: Vec<PageRefV2>,
steps: Vec<StepV2>,
}
constrain_trace_v2_schema(TraceV2Shape::json_schema(generator))
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn sample_common(id: u32) -> StepCommonV2 {
StepCommonV2 {
id,
page: "p1".into(),
effect: None,
}
}
fn sample_target() -> TargetDescriptorV2 {
TargetDescriptorV2 {
role: Some("button".into()),
name: Some("发布".into()),
tag: "button".into(),
name_attr: None,
placeholder: None,
nearby_label: None,
}
}
#[test]
fn trace_v2_has_no_version_field() {
let trace = TraceV2 {
recorded_at: "2026-07-17T09:01:10Z".into(),
started_at: None,
purpose: None,
entry: TraceEntry {
start_url: "https://example.com/".into(),
},
pages: vec![PageRefV2 {
id: "p1".into(),
url: "https://example.com/".into(),
title: None,
}],
steps: vec![StepV2::Click {
common: sample_common(1),
target: sample_target(),
}],
};
let v = serde_json::to_value(&trace).unwrap();
assert!(v.get("version").is_none());
assert!(v.get("pages").is_some());
assert!(v.get("states").is_none());
}
#[test]
fn extension_v2_trace_deserializes() {
let v = json!({
"recorded_at": "2026-07-21T08:00:00Z",
"started_at": "2026-07-21T07:59:00Z",
"purpose": "demo",
"entry": { "start_url": "https://example.com/editor" },
"pages": [
{ "id": "p1", "url": "https://example.com/editor" },
{ "id": "p2", "url": "https://example.com/p/99" }
],
"steps": [
{
"op": "fill",
"id": 1,
"page": "p1",
"target": { "tag": "input", "role": "textbox", "name": "标题" },
"value": "hello"
},
{
"op": "click",
"id": 2,
"page": "p1",
"target": { "tag": "button", "role": "button", "name": "发布" },
"effect": { "navigated_to": "p2" }
}
]
});
let trace: TraceV2 = serde_json::from_value(v).unwrap();
assert_eq!(trace.pages.len(), 2);
assert_eq!(trace.steps.len(), 2);
}
#[test]
fn v2_hover_steps_round_trip() {
let value = json!({
"recorded_at": "2026-07-21T08:00:00Z",
"entry": { "start_url": "https://example.com/" },
"pages": [{ "id": "p1", "url": "https://example.com/" }],
"steps": [{
"op": "hover",
"id": 1,
"page": "p1",
"target": { "tag": "span", "role": "button", "name": "Account" }
}]
});
let trace: TraceV2 = serde_json::from_value(value).unwrap();
assert!(matches!(trace.steps.as_slice(), [StepV2::Hover { .. }]));
assert_eq!(
serde_json::to_value(&trace).unwrap()["steps"][0]["op"],
"hover"
);
}
#[test]
fn trace_v2_ignores_unknown_extension_fields() {
let value = json!({
"recorded_at": "2026-07-21T08:00:00Z",
"entry": { "start_url": "https://example.com/" },
"pages": [{ "id": "p1", "url": "https://example.com/" }],
"steps": [],
"meta": { "tool": "legacy-exporter" }
});
let trace: TraceV2 = serde_json::from_value(value).unwrap();
assert_eq!(trace.pages.len(), 1);
assert!(trace.steps.is_empty());
}
#[test]
fn trace_v2_schema_allows_additional_properties() {
let schema = serde_json::to_value(schemars::schema_for!(TraceV2)).unwrap();
assert_ne!(
schema.get("additionalProperties"),
Some(&serde_json::Value::Bool(false)),
"Trace v2 must keep accepting traces with unknown extension fields"
);
}
#[test]
fn trace_v2_schema_forbids_classification_keys() {
let schema = serde_json::to_value(schemars::schema_for!(TraceV2)).unwrap();
assert_eq!(
schema["properties"]["states"],
json!(false),
"Trace v2 schema must reject states[] (reserved for v3)"
);
assert_eq!(
schema["properties"]["version"]["not"]["type"], "integer",
"Trace v2 schema must reject a numeric version (that selects the v3 path)"
);
}
}
-445
View File
@@ -1,445 +0,0 @@
//! Trace v3 — state-action-state log with `states[]` and step `state` refs.
//!
//! Wire format with top-level `version: 3`.
use schemars::JsonSchema;
use serde::{Deserialize, Deserializer, Serialize};
use super::interaction::KeyModifier;
use super::record_common::TraceEntry;
pub const TRACE_VERSION_V3: u32 = 3;
pub const VOM_FORMAT_VERSION: u32 = 1;
fn deserialize_trace_v3_version<'de, D>(deserializer: D) -> Result<u32, D::Error>
where
D: Deserializer<'de>,
{
let version = u32::deserialize(deserializer)?;
if version != TRACE_VERSION_V3 {
return Err(serde::de::Error::custom(format!(
"unsupported trace version {version} (expected {TRACE_VERSION_V3})"
)));
}
Ok(version)
}
fn trace_v3_version_schema(_: &mut schemars::r#gen::SchemaGenerator) -> schemars::schema::Schema {
schemars::schema::SchemaObject {
instance_type: Some(schemars::schema::InstanceType::Integer.into()),
const_value: Some(serde_json::json!(TRACE_VERSION_V3)),
..Default::default()
}
.into()
}
/// Stable semantic handle for an interacted element within a page observation.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct TargetDescriptorV3 {
#[serde(default, skip_serializing_if = "Option::is_none", rename = "ref")]
pub element_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub role: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ctx: Option<String>,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub unmatched: bool,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct RecorderInfo {
pub bsk: String,
pub vom: u32,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum StopReason {
UserFinish,
CliStop,
}
/// Page observation dictionary entry — referenced by steps via `state` / `result.state`.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct TraceStateV3 {
pub id: String,
pub url: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub title: Option<String>,
/// Full page observation (front matter + VOM body + annotations).
pub body: String,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub truncated: bool,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct StepResultV3 {
pub state: String,
}
/// Fields shared by every step variant (flattened in JSON).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct StepCommonV3 {
pub id: u32,
/// Observation id immediately before this action.
pub state: String,
pub result: StepResultV3,
}
/// One selected option (`select` op).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct SelectedOptionV3 {
pub value: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub label: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum NavigationCause {
UserTyped,
Link,
FormSubmit,
Reload,
History,
Script,
Browser,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum FillCommit {
Enter,
Suggestion,
Blur,
}
/// One recorded user action — discriminated union by `op`.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[serde(tag = "op", rename_all = "snake_case")]
pub enum StepV3 {
Navigate {
#[serde(flatten)]
common: StepCommonV3,
to: String,
cause: NavigationCause,
},
SwitchTab {
#[serde(flatten)]
common: StepCommonV3,
},
Click {
#[serde(flatten)]
common: StepCommonV3,
target: TargetDescriptorV3,
},
Hover {
#[serde(flatten)]
common: StepCommonV3,
target: TargetDescriptorV3,
},
Fill {
#[serde(flatten)]
common: StepCommonV3,
target: TargetDescriptorV3,
value: String,
commit: FillCommit,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
redacted: bool,
},
Select {
#[serde(flatten)]
common: StepCommonV3,
target: TargetDescriptorV3,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
selection: Vec<SelectedOptionV3>,
},
Press {
#[serde(flatten)]
common: StepCommonV3,
key: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
modifiers: Option<Vec<KeyModifier>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
target: Option<TargetDescriptorV3>,
},
Scroll {
#[serde(flatten)]
common: StepCommonV3,
},
}
/// Wire trace returned by `tool.record_stop` / `await`.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct TraceV3 {
#[serde(deserialize_with = "deserialize_trace_v3_version")]
#[schemars(schema_with = "trace_v3_version_schema")]
pub version: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub purpose: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub started_at: Option<String>,
pub recorded_at: String,
pub stopped_by: StopReason,
pub entry: TraceEntry,
pub recorder: RecorderInfo,
pub states: Vec<TraceStateV3>,
pub steps: Vec<StepV3>,
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn sample_common(id: u32, state: &str, result_state: &str) -> StepCommonV3 {
StepCommonV3 {
id,
state: state.into(),
result: StepResultV3 {
state: result_state.into(),
},
}
}
fn sample_target() -> TargetDescriptorV3 {
TargetDescriptorV3 {
element_ref: Some("e21".into()),
role: Some("button".into()),
name: Some("发布".into()),
ctx: Some("金桔柠檬 6 号".into()),
unmatched: false,
}
}
fn sample_trace() -> TraceV3 {
TraceV3 {
version: TRACE_VERSION_V3,
purpose: Some("把草稿商品发布上架".into()),
started_at: Some("2026-08-10T02:10:41.080Z".into()),
recorded_at: "2026-08-10T02:12:55.360Z".into(),
stopped_by: StopReason::UserFinish,
entry: TraceEntry {
start_url: "https://example.com/".into(),
},
recorder: RecorderInfo {
bsk: "0.1.10".into(),
vom: VOM_FORMAT_VERSION,
},
states: vec![
TraceStateV3 {
id: "s1".into(),
url: "https://example.com/".into(),
title: Some("Example Domain".into()),
body: "@vom 1\nRootWebArea \"Example Domain\"".into(),
truncated: false,
},
TraceStateV3 {
id: "s2".into(),
url: "https://shop.example.com/products?status=draft".into(),
title: Some("商品管理".into()),
body: "@vom 1\nRootWebArea \"商品管理\"".into(),
truncated: false,
},
],
steps: vec![
StepV3::Navigate {
common: sample_common(1, "s1", "s2"),
to: "https://shop.example.com/products?status=draft".into(),
cause: NavigationCause::UserTyped,
},
StepV3::Fill {
common: sample_common(2, "s2", "s2"),
target: TargetDescriptorV3 {
element_ref: Some("e12".into()),
role: Some("textbox".into()),
name: Some("搜索商品".into()),
ctx: None,
unmatched: false,
},
value: "金桔柠檬".into(),
commit: FillCommit::Enter,
redacted: false,
},
StepV3::Click {
common: sample_common(3, "s2", "s2"),
target: sample_target(),
},
],
}
}
#[test]
fn step_click_round_trips() {
let step = StepV3::Click {
common: sample_common(1, "s1", "s2"),
target: sample_target(),
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v.get("op").and_then(|v| v.as_str()), Some("click"));
assert_eq!(v.get("state").and_then(|v| v.as_str()), Some("s1"));
assert_eq!(v["result"]["state"], "s2");
assert_eq!(v["target"]["ref"], "e21");
let round: StepV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn step_fill_with_commit_round_trips() {
let step = StepV3::Fill {
common: sample_common(2, "s2", "s3"),
target: TargetDescriptorV3 {
element_ref: Some("e12".into()),
role: Some("textbox".into()),
name: Some("搜索商品".into()),
ctx: None,
unmatched: false,
},
value: "browser skill".into(),
commit: FillCommit::Enter,
redacted: false,
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["op"], "fill");
assert_eq!(v["commit"], "enter");
assert!(v.get("redacted").is_none());
let round: StepV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn step_fill_password_is_redacted() {
let step = StepV3::Fill {
common: sample_common(1, "s1", "s1"),
target: TargetDescriptorV3 {
element_ref: Some("e3".into()),
role: Some("textbox".into()),
name: Some("密码".into()),
ctx: None,
unmatched: false,
},
value: "***".into(),
commit: FillCommit::Blur,
redacted: true,
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["value"], "***");
assert_eq!(v["redacted"], true);
}
#[test]
fn step_navigate_with_cause_round_trips() {
let step = StepV3::Navigate {
common: sample_common(1, "s1", "s2"),
to: "https://example.com".into(),
cause: NavigationCause::UserTyped,
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["op"], "navigate");
assert_eq!(v["cause"], "user_typed");
let round: StepV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn step_switch_tab_round_trips() {
let step = StepV3::SwitchTab {
common: sample_common(2, "s2", "s5"),
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["op"], "switch_tab");
assert_eq!(v["state"], "s2");
assert_eq!(v["result"]["state"], "s5");
let round: StepV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn trace_v3_round_trips() {
let trace = sample_trace();
let v = serde_json::to_value(&trace).unwrap();
assert_eq!(v.get("version").and_then(|v| v.as_u64()), Some(3));
assert!(v.get("pages").is_none());
assert_eq!(v["recorder"]["vom"], 1);
assert_eq!(v["states"].as_array().unwrap().len(), 2);
let round: TraceV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, trace);
}
#[test]
fn default_fields_are_omitted() {
let step = StepV3::Click {
common: sample_common(1, "s1", "s1"),
target: TargetDescriptorV3 {
element_ref: Some("e1".into()),
role: Some("button".into()),
name: Some("OK".into()),
ctx: None,
unmatched: false,
},
};
let v = serde_json::to_value(&step).unwrap();
assert!(v.get("unmatched").is_none());
assert!(v["target"].get("ctx").is_none());
assert!(v["target"].get("unmatched").is_none());
}
#[test]
fn unmatched_target_serializes_flag() {
let step = StepV3::Click {
common: sample_common(1, "s1", "s2"),
target: TargetDescriptorV3 {
element_ref: None,
role: Some("button".into()),
name: Some("发布".into()),
ctx: None,
unmatched: true,
},
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["target"]["unmatched"], true);
assert!(v["target"].get("ref").is_none());
}
#[test]
fn extension_trace_deserializes() {
let v = json!({
"version": 3,
"recorded_at": "2026-07-21T08:00:00Z",
"started_at": "2026-07-21T07:59:00Z",
"purpose": "demo",
"stopped_by": "user_finish",
"entry": { "start_url": "https://example.com/editor" },
"recorder": { "bsk": "0.1.10", "vom": 1 },
"states": [
{ "id": "s1", "url": "https://example.com/editor", "body": "@vom 1" },
{ "id": "s2", "url": "https://example.com/p/99", "body": "@vom 1" }
],
"steps": [
{
"op": "fill",
"id": 1,
"state": "s1",
"result": { "state": "s1" },
"target": { "ref": "e1", "role": "textbox", "name": "标题" },
"value": "hello",
"commit": "blur"
},
{
"op": "click",
"id": 2,
"state": "s1",
"result": { "state": "s2" },
"target": { "ref": "e2", "role": "button", "name": "发布" }
}
]
});
let trace: TraceV3 = serde_json::from_value(v).unwrap();
assert_eq!(trace.version, 3);
assert_eq!(trace.states.len(), 2);
assert_eq!(trace.steps.len(), 2);
}
}
+31 -10
View File
@@ -20,22 +20,43 @@ extension keep owning the actual browser control — this package is a thin, wel
| `browser_snapshot` | `bsk snapshot` | Indented aria-tree snapshot with `@eN` refs for interaction tools. |
| `browser_observe` | `bsk observe` | Semantic VOM observation (read-only) with `@eN` refs. |
| `browser_click` | `bsk click` | Click a snapshot ref or CSS selector (button / click-count options). |
| `browser_hover` | `bsk hover` | Hover a ref or selector and allow hover-triggered UI to settle. |
| `browser_fill` | `bsk fill` | Fill an input / textarea / contenteditable (clears first by default). |
| `browser_select` | `bsk select` | Select one or more option values on a `<select>`. |
| `browser_press` | `bsk press` | Dispatch a key or combo, optionally focusing a target first. |
| `browser_screenshot` | `bsk screenshot` | PNG capture of the tab or a ref-cropped element; inlines the image when the deployment supports image input, otherwise returns a file path. |
| `browser_emulate` | `bsk emulate` | Apply or clear mobile device emulation on the active tab. |
| `browser_tab_list` | `bsk tab list` | List user and/or Agent Window tabs visible to an owned session. |
| `browser_tab_create` | `bsk tab create` | Create a foreground or background tab in the Agent Window. |
| `browser_tab_select` | `bsk tab select` | Focus an Agent Window tab. |
| `browser_tab_close` | `bsk tab close` | Close an Agent Window tab. |
| `browser_tab_borrow` | `bsk tab borrow` | Move a user tab into the Agent Window for controlled interaction. |
| `browser_tab_return` | `bsk tab return` | Return a borrowed tab to its original user window. |
| `browser_navigate_back` / `browser_navigate_forward` | `bsk navigate-back` / `bsk navigate-forward` | Traverse tab history and wait for a lifecycle phase. |
| `browser_reload` | `bsk reload` | Reload the active tab, optionally bypassing cache. |
| `browser_wait_for_navigation` | `bsk wait-for-navigation` | Wait explicitly for a page lifecycle event. |
| `browser_request_help` | `bsk request-help` | Pause for login, captcha, OTP, confirmation, or another human browser step. |
| `browser_get_html` | `bsk get-html` | Read raw document or ref-scoped HTML after semantic observation is insufficient. |
| `browser_console` | `bsk console` | Read buffered console/log/exception entries without evaluating JavaScript. |
| `browser_network` | `bsk network` | Read buffered network response/failure metadata. |
| `browser_window_resize` | `bsk window resize` | Resize the owned session's Agent Window. |
`bsk evaluate` and `bsk record` are intentionally not exposed by this plugin: arbitrary page
evaluation is a higher-risk capability, while recording is long-running and needs a dedicated
DeepSeek Harness lifecycle before it can be added safely.
## Agent skill (progressive disclosure)
Beyond the tools, the plugin publishes the **`browser-skill` agent skill** through the harness's
official skill seam (`ctx.skills.register`): the catalog entry (name + routing description) is
resident in `<available_skills>`, and the body is loaded only when the model invokes the `skill`
tool. The body is assembled at build time from two parts, so there is exactly one source of
truth: a dsh-specific prelude (`skill/prelude.md` — tool↔CLI map, owned-session semantics,
plugin-only overrides) followed verbatim by the canonical CLI skill (`skill/SKILL.md` at the
repo root — the same file `crates/bsk-cli/build.rs` mirrors for the CLI package; workflows,
stop-when-done rules, refs usage, sandbox rules). Registration and every pre-step catalog
snapshot are pure in-memory reads (no disk/process/daemon); compositions without the skill
tool. Its single source is the DSH-specific `skill/SKILL.md`, which documents only structured
`browser_*` calls and their plugin semantics. The repository-root CLI skill is intentionally not
concatenated: its command examples belong to a different execution interface and would bypass
the plugin's ownership, live observation UI, cancellation, and cleanup path if followed directly.
The build rejects internal CLI-name leakage, command-line code blocks, unknown browser tools, and
missing supported browser tools before embedding the Markdown. Registration and every pre-step
catalog snapshot are pure in-memory reads (no disk/process/daemon); compositions without the skill
seam degrade silently.
## Multi-session model
@@ -88,7 +109,7 @@ All fields are optional and validated through the plugin's Schemastery `Config`:
# lazyTools: true # reveal browser_* tools only after the skill is invoked
```
- **`lazyTools` (default `true`)** — the final progressive-disclosure stage: the eleven
- **`lazyTools` (default `true`)** — the final progressive-disclosure stage: the 28
`browser_*` tool schemas stay OUT of the system prompt (zero schema tokens) until the
`browser-skill` skill is actually invoked — the skill catalog entry is the only
advertisement. One successful invocation (model tool call, or a `/browser-skill` user
@@ -152,7 +173,7 @@ shell's theme cannot bleed back in.
card). Screenshots additionally attach the image itself when the host mounts an attachment store
and the active model route declares image input; otherwise the PNG path is returned.
- **Web UI toolview (browser half)**: the package is dual-face. `dsh.client` (platform `web`) ships
`lib/client.js`, which registers a keyed `tool.call.toolview` view for `browser_screenshot`. The
`lib/client.cjs`, which registers a keyed `tool.call.toolview` view for `browser_screenshot`. The
custom view keeps the terminal block (command + output) and, when the settled result carries an
image block, resolves the durable attachment through the client session's authorized
`readAttachment` RPC and renders it with the shared `MessageImage` thumbnail/lightbox atoms.
@@ -183,8 +204,8 @@ as `@wxg-prc-cpg/browser-skill-dsh-plugin`. The Cordis plugin id stays
Trigger it by pushing a tag that matches `package.json`'s `version`:
```sh
git tag dsh-plugin-v0.1.0
git push origin dsh-plugin-v0.1.0
git tag dsh-plugin-v0.1.2
git push origin dsh-plugin-v0.1.2
```
Or run the workflow from the Actions tab (`workflow_dispatch`). The job reads the
@@ -1,7 +1,7 @@
{
"name": "@wxg-prc-cpg/browser-skill-dsh-plugin",
"description": "DeepSeek Harness tool plugin that exposes BrowserSkill (bsk) browser automation to the model",
"version": "0.1.1",
"description": "DeepSeek Harness tool plugin that exposes BrowserSkill browser automation (browser_* tools) to the model",
"version": "0.1.2",
"license": "MIT",
"publishConfig": {
"access": "public",
@@ -1,20 +1,15 @@
// Assemble the plugin's skill content: a dsh-specific routing prelude, the
// canonical CLI skill body verbatim (repo-root skill/SKILL.md — the same source
// crates/bsk-cli's build.rs mirrors), then a dsh routing reminder postlude. The
// original's YAML frontmatter is stripped: name/description are supplied by the
// registration.
// Embed the DeepSeek Harness-specific browser skill at build time. The model-facing
// skill deliberately documents only injected browser_* tools; the repository's CLI
// skill is a separate interface and must never be concatenated here.
// Output: src/skill-content.generated.ts (gitignored, built before tsdown and
// before vitest) so the markdown ships inlined in the bundle and registration
// is a pure in-memory read — never a per-step disk access.
// before vitest), so registration is a pure in-memory read.
import { readFileSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
const here = dirname(fileURLToPath(import.meta.url));
const pkg = join(here, "..");
const prelude = readFileSync(join(pkg, "skill", "prelude.md"), "utf8");
const upstream = readFileSync(join(pkg, "..", "..", "skill", "SKILL.md"), "utf8");
const postlude = readFileSync(join(pkg, "skill", "postlude.md"), "utf8");
const source = readFileSync(join(pkg, "skill", "SKILL.md"), "utf8");
function stripFrontmatter(markdown) {
if (!markdown.startsWith("---\n")) return markdown;
@@ -22,18 +17,69 @@ function stripFrontmatter(markdown) {
return end === -1 ? markdown : markdown.slice(end + 5).replace(/^\s*\n/, "");
}
const SUPPORTED_BROWSER_TOOLS = [
"browser_session_start",
"browser_session_stop",
"browser_session_list",
"browser_navigate",
"browser_navigate_back",
"browser_navigate_forward",
"browser_reload",
"browser_wait_for_navigation",
"browser_snapshot",
"browser_observe",
"browser_get_html",
"browser_screenshot",
"browser_console",
"browser_network",
"browser_click",
"browser_hover",
"browser_fill",
"browser_select",
"browser_press",
"browser_tab_list",
"browser_tab_create",
"browser_tab_select",
"browser_tab_close",
"browser_tab_borrow",
"browser_tab_return",
"browser_request_help",
"browser_emulate",
"browser_window_resize",
];
function validateModelFacingSkill(markdown) {
if (/\bbsk\b/i.test(markdown)) {
throw new Error("DSH skill must not expose the internal CLI name");
}
if (/```(?:bash|sh|shell)\b/i.test(markdown)) {
throw new Error("DSH skill must not contain command-line code blocks");
}
const mentioned = new Set(markdown.match(/\bbrowser_[a-z][a-z_]*\b/g) ?? []);
const supported = new Set(SUPPORTED_BROWSER_TOOLS);
const unknown = [...mentioned].filter((name) => !supported.has(name));
const missing = SUPPORTED_BROWSER_TOOLS.filter((name) => !mentioned.has(name));
if (unknown.length > 0) {
throw new Error(`DSH skill mentions unsupported browser tools: ${unknown.join(", ")}`);
}
if (missing.length > 0) {
throw new Error(`DSH skill omits supported browser tools: ${missing.join(", ")}`);
}
}
const NAME = "browser-skill";
const DESCRIPTION =
"Browser automation against the user's logged-in Chromium through this plugin's injected browser_* tools; never invoke bsk through bash or shell. " +
"tools (bsk engine): visit and read pages, fill forms, click through flows, scrape data, " +
"smoke-test UI changes. Use when a task needs a real browser; requires the bsk CLI and the " +
"browser-skill extension connected.";
"Browser automation against the user's logged-in Chromium through this plugin's injected " +
"browser_* tools. Use to visit and read pages, fill forms, click through flows, inspect tabs, " +
"debug page activity, or smoke-test UI changes in a managed Agent Window.";
const content = `${prelude.trim()}\n\n${stripFrontmatter(upstream).trim()}\n\n${postlude.trim()}\n`;
const content = `${stripFrontmatter(source).trim()}\n`;
validateModelFacingSkill(content);
const banner =
"// GENERATED by scripts/build-skill-content.mjs — do not edit.\n" +
"// Sources: skill/prelude.md + ../../../skill/SKILL.md + skill/postlude.md.\n";
"// Source: skill/SKILL.md (DeepSeek Harness browser_* tool instructions).\n";
writeFileSync(
join(pkg, "src", "skill-content.generated.ts"),
@@ -0,0 +1,271 @@
---
name: browser-skill
description: |
Use the injected browser_* tools to automate the user's logged-in Chromium
browser in a managed Agent Window.
---
# browser-skill for DeepSeek Harness
Drive the user's real Chromium browser through the structured `browser_*` tools provided by this
plugin. The browser keeps the user's existing logins and cookies while isolating automation in an
Agent Window. Use the tool schemas exactly as exposed by DeepSeek Harness; arguments are structured
fields, not command-line flags.
## Tool availability
Loading this skill reveals the browser tools for the rest of the conversation. If the tools were
hidden before this skill was loaded, start browser work only after the skill invocation succeeds.
All browser operations must use the injected tools directly. Do not translate the examples in this
document into terminal commands or invoke another process to control the browser. Direct tool calls
preserve session ownership, the live observation window, tool cards, cancellation, and cleanup.
## When to use
- Visit and read pages that require the user's existing browser session.
- Fill forms, select options, click through bounded flows, or smoke-test a web UI.
- Inspect tabs, page structure, screenshots, console messages, or network metadata.
- Work with a user tab after explicitly borrowing it into the Agent Window.
Do not use browser automation for tasks that need only local files, a purpose-built API, or an
explanation. Never extract credentials, cookies, tokens, or other secrets. Do not keep control of a
personal tab longer than the immediate task requires.
## Mandatory workflow
Every browser task follows this lifecycle:
1. Call `browser_session_start` and retain its returned `sessionId`.
2. Use that session for every browser operation. Pass `session` explicitly when more than one
session exists.
3. Call `browser_session_stop` when the bounded goal is reached or an error ends the task. Treat
cleanup like a `finally` path unless the user explicitly asks to keep the session open.
Conceptual tool-call sequence:
```text
browser_session_start({ url: "https://example.com" })
browser_observe({ session: sessionId })
browser_click({ session: sessionId, target: "@e4" })
browser_observe({ session: sessionId })
browser_session_stop({ session: sessionId })
```
The plugin owns only sessions created by `browser_session_start`. `browser_session_list` reports
only those owned sessions. Never guess or reuse a session id from another program. When `session`
is omitted, most tools use the most recently started or used owned session; explicit ids are safer
for multi-session work.
## Stop when the goal is met
Browser work is a bounded task, not open-ended browsing.
1. Define one observable success condition from the user's request.
2. Take the shortest path: observe, act, then perform at most one relevant check.
3. Stop immediately when success is visible. Do not refresh, navigate, click, or perform extra
verification after success.
4. If a human-only step appears, use `browser_request_help`. If an action fails twice without
progress, pause instead of brute-forcing.
5. When uncertain, make at most one fresh observation. If the state is still ambiguous, ask the
user rather than continuing to click.
## Core interaction loop
Use `browser_navigate` to open a destination in the active Agent Window tab, then inspect the page
before interacting:
1. `browser_observe` is the primary semantic view. It returns roles, states, text, and `@eN` refs,
including hints about conditional hover or focus surfaces.
2. `browser_snapshot` is the static accessibility-tree fallback when the semantic view is
insufficient or a stricter tree is useful.
3. Prefer a fresh `@eN` ref over a raw CSS selector for `browser_click`, `browser_hover`,
`browser_fill`, `browser_select`, and targeted `browser_press` calls.
4. Observe again after navigation or a meaningful DOM change before using another ref.
Refs invalidate after navigation and may also become stale after large DOM changes. Never carry a
ref to a new page. If an observation marks a control as hover-triggered, call `browser_hover` on the
trigger, observe again, and then act on the newly visible ref.
Use interaction tools according to their schemas:
- `browser_click`: click a ref or selector; optionally choose mouse button and click count.
- `browser_hover`: move to a ref or selector and allow hover UI to settle; then observe again.
- `browser_fill`: clear and fill a text-capable element; set `noClear` only when appending is
intentional.
- `browser_select`: pass `values` as an array, including for a single option.
- `browser_press`: send a key or combo, optionally focusing `target` first.
Actions may trigger a navigation after the action itself returns. Use
`browser_wait_for_navigation` only when that lifecycle wait is actually needed, then observe the
new page.
## Observation and page reading priority
Escalate page reading only as needed:
1. `browser_observe` for normal semantic understanding and interaction refs.
2. `browser_snapshot` for a stricter static accessibility tree.
3. `browser_get_html` for hidden DOM, metadata, or exact markup that the semantic views cannot
provide. A `ref` must come from a fresh observation; keep `maxBytes` bounded.
4. `browser_screenshot` for layout, styling, canvas, image content, or another genuinely visual
question. Pass a fresh `ref` to crop to one element, or omit it for the visible tab.
Do not start with raw HTML or a screenshot merely to discover ordinary controls. Screenshots may
return an inline image or a saved path depending on the deployment.
## Tabs and the Agent Window boundary
Write actions normally affect only Agent Window tabs.
- `browser_tab_list` lists `agent`, `user`, or `all` tabs visible to the owned session. Use
`scope: "user"` to locate a user tab before borrowing it.
- `browser_tab_create` creates an Agent Window tab. It is active by default; set `active: false`
for a background tab.
- `browser_tab_select` focuses an Agent Window tab.
- `browser_tab_close` closes an Agent Window tab.
- `browser_tab_borrow` moves a chosen user tab into the Agent Window before any write action.
- `browser_tab_return` returns a borrowed tab to its original window and position. Return it as
soon as the immediate task is finished; stopping the session is only the fallback cleanup.
Use only a `tabId` returned by `browser_tab_list` or `browser_tab_create`. Do not assume a tab id
from page position or title.
## Navigation and lifecycle
- `browser_navigate` opens a URL and can wait for a requested lifecycle phase.
- `browser_navigate_back` and `browser_navigate_forward` move one history entry.
- `browser_reload` reloads the current tab; use `hard: true` only when bypassing cache is necessary.
- `browser_wait_for_navigation` waits for a lifecycle event after an action that did not already
wait. Avoid speculative waits when no navigation is expected.
After navigation, history traversal, or reload, discard old refs and call `browser_observe` or
`browser_snapshot` again.
## Ask the user for browser help
Call `browser_request_help` for login, captcha, OTP, payment confirmation, consent, or another step
that requires the user. Give a clear `prompt`; when the prompt refers to a concrete control, pass
its fresh ref or selector in `targets` so the overlay can highlight it. Use `title` only when it
adds clarity.
Optional `completionCriteria` can finish the pause automatically when the page has an explicit,
stable success signal. It supports `any` or `all` conditions using `urlContains`, `urlMatches`,
`selectorExists`, `selectorMissing`, `textExists`, or `textMissing`, plus `stableForMs`.
```text
browser_request_help({
session: sessionId,
title: "Sign in required",
prompt: "Complete sign-in, then return control after the account page appears.",
targets: ["@e7"],
timeoutMs: 300000,
completionCriteria: {
any: [{ urlContains: "/account" }, { selectorExists: "[data-testid='account-menu']" }],
stableForMs: 1000
}
})
```
Interpret outcomes deliberately:
- `continued` or `completed`: the user returned control or the explicit detector matched. Observe
again before reasoning about the new state or using refs.
- `cancelled`: stop the requested flow.
- `timed_out`: report the timeout; do not repeatedly reopen the prompt.
- `disabled`: unattended mode does not allow help; proceed only if safe or stop gracefully.
- `navigated`: legacy outcome; do not treat navigation alone as proof of success.
## Debugging without page scripting
Use the read-only debugging tools only when relevant to the user's task:
- `browser_console` reads buffered console messages, browser logs, and JavaScript exceptions.
Set `includeStack: true` when stack frames are useful.
- `browser_network` reads buffered response and failure metadata. It does not return request or
response headers or bodies and does not modify traffic.
Both tools accept `tabId`, `since`, `limit`, and `maxTextChars`. Their result returns `nextSince`;
pass that value as the next `since` cursor instead of repeatedly requesting the same entries.
Arbitrary page script evaluation and interaction recording are intentionally unsupported in this
DeepSeek Harness plugin. Do not invent tool names or route around this limitation. If the task
strictly requires either capability, explain that it is not available in this phase.
## Window and device controls
- `browser_window_resize` changes the Agent Window's outer `width` and `height`; each dimension
must be from 100 through 7680 CSS pixels.
- `browser_emulate` applies a built-in device preset or explicit viewport settings to the active
tab. Explicit `width` and `height` must be supplied together. `mobile: true` also requires both
dimensions. Use `off: true` alone to clear emulation. Emulation is per-tab and new tabs do not
inherit it.
## Tool reference
### Sessions
| Tool | Purpose |
| --- | --- |
| `browser_session_start` | Start an owned Agent Window session, optionally with URL, size, browser instance, focus behavior, or device preset. |
| `browser_session_stop` | Stop an owned session and close its Agent Window. |
| `browser_session_list` | List owned sessions and identify the current one. |
### Navigation and reading
| Tool | Purpose |
| --- | --- |
| `browser_navigate` | Navigate the active tab to a URL. |
| `browser_navigate_back` | Go back one history entry. |
| `browser_navigate_forward` | Go forward one history entry. |
| `browser_reload` | Reload the active tab, optionally bypassing cache. |
| `browser_wait_for_navigation` | Wait for an expected page lifecycle event. |
| `browser_observe` | Read the primary semantic page representation with refs. |
| `browser_snapshot` | Read the static accessibility-tree fallback with refs. |
| `browser_get_html` | Read raw document or ref-scoped HTML as a last resort. |
| `browser_screenshot` | Capture the visible tab or a ref-cropped element. |
### Interaction
| Tool | Purpose |
| --- | --- |
| `browser_click` | Click a ref or CSS selector. |
| `browser_hover` | Hover a ref or selector to reveal conditional UI. |
| `browser_fill` | Fill a text-capable element. |
| `browser_select` | Select one or more option values. |
| `browser_press` | Send a keyboard key or combo. |
### Tabs
| Tool | Purpose |
| --- | --- |
| `browser_tab_list` | List visible user and Agent Window tabs. |
| `browser_tab_create` | Create an Agent Window tab. |
| `browser_tab_select` | Focus an Agent Window tab. |
| `browser_tab_close` | Close an Agent Window tab. |
| `browser_tab_borrow` | Move a user tab into the Agent Window. |
| `browser_tab_return` | Return a borrowed tab to its original window. |
### Assistance, diagnostics, and display
| Tool | Purpose |
| --- | --- |
| `browser_request_help` | Pause for an explicit user browser step. |
| `browser_console` | Read buffered console and exception entries. |
| `browser_network` | Read buffered response and failure metadata. |
| `browser_window_resize` | Resize the Agent Window. |
| `browser_emulate` | Apply or clear per-tab device emulation. |
## Error handling and final checklist
On a stale ref, observe again and retry once with a fresh ref. On a closed or unknown tab, list
tabs instead of guessing. On an unknown session, list owned sessions or start a new one; never try
foreign ids. On timeout, decide whether one longer, purposeful wait is justified. Avoid repeated
actions with unchanged state.
Before finishing a browser task, verify that:
- The user's observable goal is met, or the blocker is clearly reported.
- Any borrowed tab has been returned when practical.
- The owned session has been stopped unless the user explicitly asked to keep it open.
- No further page action is taken after success.
@@ -1,10 +0,0 @@
---
## DSH routing reminder
The CLI-oriented reference above describes behavior, not the execution mechanism in
DeepSeek Harness. For every browser operation, call the injected `browser_*` model
tool. Do not translate these examples into `bash` commands and do not execute `bsk`
directly; the plugin invokes `bsk` internally. In particular, always begin with
`browser_session_start`, pass its returned session id to subsequent `browser_*`
tools, and finish with `browser_session_stop`.
@@ -1,71 +0,0 @@
# browser-skill (dsh edition)
## DSH tool-routing requirement (highest priority)
This skill is running inside DeepSeek Harness, where the plugin has injected
structured `browser_*` model tools. **Always call those injected `browser_*` tools
for browser work. Never use `bash`, `shell`, `exec`, or another tool to invoke the
`bsk` CLI directly.** The CLI command names shown below are only conceptual
mappings for understanding the tools; they are not instructions to run commands.
If a `browser_*` tool is available, using the CLI directly is incorrect because it
bypasses plugin session ownership, observation UI, tool cards, and cleanup.
Use the injected tool whose name matches the operation, for example:
`browser_session_start`, `browser_navigate`, `browser_observe`, `browser_snapshot`,
`browser_click`, `browser_fill`, `browser_press`, `browser_screenshot`, and
`browser_session_stop`. Pass arguments as the tool schema specifies (not CLI flags).
Drive the user's **real Chromium browser** (logins, cookies, Agent Window isolation)
through this plugin's `browser_*` tools. Each injected tool is implemented by the
plugin using the `bsk` engine internally — the model must not invoke that engine
itself. The workflow, judgment rules, and safety constraints remain identical to
the canonical CLI skill.
> **Tool availability**: invoking this skill (which you just did) makes the plugin's
> `browser_*` tools available for the rest of the session — they are injected on
> successful skill load. Until then they are deliberately hidden, so do not attempt
> a browser action before loading this skill.
## Tool ↔ CLI map
The tools below are thin, structured wrappers over the bsk CLI verbs of the same
name; `session` parameters take the 4-letter session id (no `--session` flag
spelling here):
| dsh tool | bsk equivalent |
|---|---|
| `browser_session_start` / `browser_session_stop` / `browser_session_list` | `bsk session start/stop/list` |
| `browser_navigate` | `bsk navigate` |
| `browser_snapshot` / `browser_observe` | `bsk snapshot` / `bsk observe` |
| `browser_click` / `browser_fill` / `browser_press` | `bsk click` / `bsk fill` / `bsk press` |
| `browser_screenshot` | `bsk screenshot` |
| `browser_emulate` | `bsk emulate` |
(`bsk console` / `bsk network` have no tool counterpart in this plugin yet.)
Plugin-specific semantics that override or narrow the CLI rules below:
- **Owned sessions only**: this plugin sees and acts on just the sessions IT
started via `browser_session_start`. `browser_session_list` shows only those
(with the current marker); passing a session id created by another program
(or person) sharing the bsk daemon is refused outright — never retry those
with the same id.
- **Current session**: omitted `session` parameters resolve to the most
recently started or used OWNED session. With several sessions in flight,
pass `session` explicitly to avoid surprises.
- **Screenshots**: when the model route accepts images, `browser_screenshot`
returns the PNG inline as an image attachment; otherwise it returns the
saved file path.
- **`browser_emulate mobile`**: requires `width`+`height` — the bsk daemon
refuses `--mobile` without viewport dimensions.
- **Live observation**: in the dsh Web UI, every owned session is watched by
the floating overlay (thumbnails, action, interrupt) — observational side
traffic is free and never steals your command turn; just keep acting.
The canonical CLI skill body follows verbatim (source of truth: `skill/SKILL.md`
in the BrowserSkill repo, mirrored by build into `crates/bsk-cli/skill/SKILL.md`).
Read `bsk <verb> --session <id>` as "the `browser_<verb>` tool with
`session: <id>`"; everything else — when to use, the stop-when-done rules, the
interaction loop, refs, sandbox rules — applies unchanged.
---
@@ -1,6 +1,6 @@
/**
* Lazy tool-schema injection ("progressive disclosure", final stage): with
* `lazyTools` on (the default), the 11 browser_* tool schemas stay OUT of the
* `lazyTools` on (the default), the 28 browser_* tool schemas stay OUT of the
* system prompt until the `browser-skill` skill has actually been invoked —
* the skill catalog entry is the only advertisement. One successful
* invocation (model tool call, or a user's `/browser-skill` gesture) reveals
@@ -96,7 +96,7 @@ export function hasSuccessfulSkillInvocation(events: readonly SessionEventLike[]
/**
* Arm the lazy reveal. Returns a disposer tearing down listeners and — when
* the reveal already happened — the tool suite itself.
* @param registerSuite - registers the 11 browser_* tools, returns their disposer.
* @param registerSuite - registers the 28 browser_* tools, returns their disposer.
*/
export function armLazyTools(ctx: Context, registerSuite: () => () => void): () => void {
let suiteDisposer: (() => void) | undefined;
@@ -400,14 +400,48 @@ export function actionForLabel(label: string): string {
return "observing";
case "click":
return "clicking";
case "hover":
return "hovering";
case "fill":
return "filling";
case "select":
return "selecting";
case "press":
return "pressing";
case "screenshot":
return "capturing";
case "emulate":
return "emulating";
case "tab list":
return "listing tabs";
case "tab create":
return "creating tab";
case "tab close":
return "closing tab";
case "tab select":
return "selecting tab";
case "tab borrow":
return "borrowing tab";
case "tab return":
return "returning tab";
case "navigate-back":
return "navigating back";
case "navigate-forward":
return "navigating forward";
case "reload":
return "reloading";
case "wait-for-navigation":
return "waiting for navigation";
case "request-help":
return "waiting for user";
case "get-html":
return "reading HTML";
case "console":
return "reading console";
case "network":
return "reading network";
case "window resize":
return "resizing window";
default:
return label;
}
@@ -0,0 +1,58 @@
import type { ToolDefinition, ToolResult, ToolRunContext } from "@deepseek-ai/dsh-tools";
import type { ToolDeps } from "./tools";
export type ToolRegistrar = (definition: ToolDefinition) => () => void;
type TerminalPresentation = { card: "terminal"; output: string; exitCode: number } | undefined;
/** Existing tool runtime seams reused by the phase-one capability modules. */
export interface PhaseOneRuntime {
run(
exec: ToolRunContext,
args: string[],
label: string,
observeSession?: string,
runnerTimeoutMs?: number,
): Promise<unknown>;
commandLine(args: string[]): string;
presentTerminalResult: (_args: never, result: ToolResult) => TerminalPresentation;
}
export function requireNonEmpty(value: string, name: string): void {
if (value.trim().length === 0) throw new Error(`${name} must be a non-empty string`);
}
export function requirePositive(value: number | undefined, name: string): void {
if (value !== undefined && value <= 0) throw new Error(`${name} must be greater than zero`);
}
/** CLI positional target detection is shared by hover/select/request-help. */
export function isSnapshotRef(target: string): boolean {
return /^@?e\d+$/.test(target);
}
/**
* Give the child enough time to honour a command-level timeout plus IPC
* settlement slack, without shortening the plugin's configured default.
*/
export function runnerTimeout(deps: ToolDeps, commandTimeoutMs: number | undefined): number {
if (commandTimeoutMs === undefined) return deps.config.defaultTimeoutMs;
return Math.max(deps.config.defaultTimeoutMs, commandTimeoutMs + 15_000);
}
export function appendTarget(args: string[], target: string): void {
args.push(target);
}
export function appendTabId(args: string[], tabId: number | undefined): void {
if (tabId !== undefined) args.push("--tab-id", String(tabId));
}
export function appendWaitOptions(
args: string[],
waitUntil: string | undefined,
timeoutMs: number | undefined,
): void {
if (waitUntil !== undefined) args.push("--wait-until", waitUntil);
if (timeoutMs !== undefined) args.push("--timeout", `${timeoutMs}ms`);
}
@@ -0,0 +1,197 @@
import { defineTool } from "@deepseek-ai/dsh-tools";
import {
appendTabId,
appendTarget,
type PhaseOneRuntime,
requireNonEmpty,
requirePositive,
runnerTimeout,
type ToolRegistrar,
} from "./phase-one-runtime";
import { SESSION_PARAM, TAB_ID_PARAM, TIMEOUT_MS_PARAM } from "./tool-params";
import type { ToolDeps } from "./tools";
const MODIFIERS = ["alt", "ctrl", "meta", "shift"] as const;
/** Add hover and select without bypassing session ownership or observation. */
export function registerPhaseOneInteractionTools(
deps: ToolDeps,
register: ToolRegistrar,
runtime: PhaseOneRuntime,
): void {
const { registry } = deps;
register(
defineTool({
name: "browser_hover",
description:
"Move the mouse over a snapshot ref or CSS selector to reveal hover-triggered UI. " +
"Run browser_observe or browser_snapshot again afterwards to discover newly visible refs.",
parameters: {
target: {
type: "string",
required: true,
description: "Snapshot ref (@e3 / e3) or CSS selector of the element to hover.",
},
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
modifiers: {
type: "array",
items: { type: "string", enum: MODIFIERS },
description: "Keyboard modifiers held during the mouse move.",
},
settleMs: {
type: "integer",
description: "Milliseconds to wait for hover-triggered UI to settle (default: 200).",
},
timeoutMs: TIMEOUT_MS_PARAM,
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
x: { type: "number", required: true },
y: { type: "number", required: true },
},
},
render: (_args, value) => [
{
type: "text",
text: `[session ${value.session}] hovered at (${value.x}, ${value.y}) on tab ${value.tabId}`,
},
],
},
async execute(args, exec) {
requireNonEmpty(args.target, "target");
requirePositive(args.timeoutMs, "timeoutMs");
requirePositive(args.settleMs, "settleMs");
const sessionId = registry.resolve(args.session, "browser_hover");
const cmdArgs = ["hover", "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
if (args.modifiers !== undefined && args.modifiers.length > 0) {
cmdArgs.push("--modifiers", args.modifiers.join(","));
}
if (args.settleMs !== undefined) cmdArgs.push("--settle", `${args.settleMs}ms`);
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
appendTarget(cmdArgs, args.target);
const reply = (await runtime.run(
exec,
cmdArgs,
"hover",
sessionId,
runnerTimeout(deps, args.timeoutMs),
)) as { tab_id: number; x: number; y: number };
return { session: sessionId, tabId: reply.tab_id, x: reply.x, y: reply.y };
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"hover",
args.target,
"--session",
args.session ?? "(current)",
]),
description: "Hover an element",
}),
presentResult: runtime.presentTerminalResult,
}),
);
register(
defineTool({
name: "browser_select",
description:
"Set a select element's option values. Pass one value for a normal select or multiple " +
"values for a multi-select; values replace the current selection.",
parameters: {
target: {
type: "string",
required: true,
description: "Snapshot ref (@e3 / e3) or CSS selector of the select element.",
},
values: {
type: "array",
required: true,
items: { type: "string" },
description: "Option value attributes to select; at least one value is required.",
},
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
timeoutMs: TIMEOUT_MS_PARAM,
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
multiple: { type: "boolean", required: true },
selectedValues: {
type: "array",
required: true,
items: { type: "string" },
},
selectedLabels: {
type: "array",
required: true,
items: { type: "string" },
},
},
},
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] selected ` +
`${value.selectedValues.join(", ") || "(none)"} on tab ${value.tabId}`,
},
],
},
async execute(args, exec) {
requireNonEmpty(args.target, "target");
requirePositive(args.timeoutMs, "timeoutMs");
if (args.values.length === 0) throw new Error("values must contain at least one option");
const sessionId = registry.resolve(args.session, "browser_select");
const cmdArgs = ["select", "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
for (const value of args.values) cmdArgs.push("--value", value);
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
appendTarget(cmdArgs, args.target);
const reply = (await runtime.run(
exec,
cmdArgs,
"select",
sessionId,
runnerTimeout(deps, args.timeoutMs),
)) as {
tab_id: number;
multiple: boolean;
selected_values: string[];
selected_labels: string[];
};
return {
session: sessionId,
tabId: reply.tab_id,
multiple: reply.multiple,
selectedValues: reply.selected_values,
selectedLabels: reply.selected_labels,
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"select",
args.target,
"--session",
args.session ?? "(current)",
]),
description: `Select ${args.values.length} option${args.values.length === 1 ? "" : "s"}`,
}),
presentResult: runtime.presentTerminalResult,
}),
);
}
@@ -0,0 +1,230 @@
import { defineTool } from "@deepseek-ai/dsh-tools";
import {
appendTabId,
appendWaitOptions,
type PhaseOneRuntime,
requirePositive,
runnerTimeout,
type ToolRegistrar,
} from "./phase-one-runtime";
import { SESSION_PARAM, TAB_ID_PARAM, TIMEOUT_MS_PARAM, WAIT_UNTIL_PARAM } from "./tool-params";
import type { ToolDeps } from "./tools";
interface RawHistoryResult {
tab_id: number;
previous_url?: string;
final_url?: string;
reached: string;
error_text?: string;
}
const HISTORY_OUTPUT = {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
previousUrl: { type: "string" },
finalUrl: { type: "string" },
reached: { type: "string", required: true },
errorText: { type: "string" },
},
} as const;
function mapHistory(sessionId: string, reply: RawHistoryResult) {
return {
session: sessionId,
tabId: reply.tab_id,
...(reply.previous_url !== undefined ? { previousUrl: reply.previous_url } : {}),
...(reply.final_url !== undefined ? { finalUrl: reply.final_url } : {}),
reached: reply.reached,
...(reply.error_text !== undefined ? { errorText: reply.error_text } : {}),
};
}
/** Add browser history, reload, and explicit lifecycle waiting. */
export function registerPhaseOneNavigationTools(
deps: ToolDeps,
register: ToolRegistrar,
runtime: PhaseOneRuntime,
): void {
const { registry } = deps;
const registerHistory = (direction: "back" | "forward") => {
const toolName = `browser_navigate_${direction}` as const;
const command = `navigate-${direction}`;
register(
defineTool({
name: toolName,
description: `Navigate the active Agent Window tab ${direction} by one history entry.`,
parameters: {
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
waitUntil: WAIT_UNTIL_PARAM,
timeoutMs: TIMEOUT_MS_PARAM,
},
output: {
schema: HISTORY_OUTPUT,
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] navigated ${direction} on tab ${value.tabId}` +
(value.finalUrl !== undefined ? ` to ${value.finalUrl}` : "") +
` (reached: ${value.reached})` +
(value.errorText !== undefined ? ` — ${value.errorText}` : ""),
},
],
},
async execute(args, exec) {
requirePositive(args.timeoutMs, "timeoutMs");
const sessionId = registry.resolve(args.session, toolName);
const cmdArgs = [command, "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
appendWaitOptions(cmdArgs, args.waitUntil, args.timeoutMs);
const reply = (await runtime.run(
exec,
cmdArgs,
command,
sessionId,
runnerTimeout(deps, args.timeoutMs),
)) as RawHistoryResult;
if (reply.final_url !== undefined) deps.observation.setUrl(sessionId, reply.final_url);
return mapHistory(sessionId, reply);
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([command, "--session", args.session ?? "(current)"]),
description: `Navigate ${direction}`,
}),
presentResult: runtime.presentTerminalResult,
}),
);
};
registerHistory("back");
registerHistory("forward");
register(
defineTool({
name: "browser_reload",
description:
"Reload the active Agent Window tab and wait for a lifecycle phase. Set hard=true to " +
"bypass the HTTP cache.",
parameters: {
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
waitUntil: WAIT_UNTIL_PARAM,
timeoutMs: TIMEOUT_MS_PARAM,
hard: { type: "boolean", description: "Bypass the HTTP cache while reloading." },
},
output: {
schema: HISTORY_OUTPUT,
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] reloaded tab ${value.tabId}` +
(value.finalUrl !== undefined ? ` at ${value.finalUrl}` : "") +
` (reached: ${value.reached})` +
(value.errorText !== undefined ? ` — ${value.errorText}` : ""),
},
],
},
async execute(args, exec) {
requirePositive(args.timeoutMs, "timeoutMs");
const sessionId = registry.resolve(args.session, "browser_reload");
const cmdArgs = ["reload", "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
appendWaitOptions(cmdArgs, args.waitUntil, args.timeoutMs);
if (args.hard === true) cmdArgs.push("--hard");
const reply = (await runtime.run(
exec,
cmdArgs,
"reload",
sessionId,
runnerTimeout(deps, args.timeoutMs),
)) as RawHistoryResult;
if (reply.final_url !== undefined) deps.observation.setUrl(sessionId, reply.final_url);
return mapHistory(sessionId, reply);
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"reload",
"--session",
args.session ?? "(current)",
...(args.hard === true ? ["--hard"] : []),
]),
description: "Reload the active tab",
}),
presentResult: runtime.presentTerminalResult,
}),
);
register(
defineTool({
name: "browser_wait_for_navigation",
description:
"Wait for a page lifecycle event on the active Agent Window tab. Use after an action " +
"that may navigate when the action result itself does not wait for navigation.",
parameters: {
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
waitUntil: WAIT_UNTIL_PARAM,
timeoutMs: TIMEOUT_MS_PARAM,
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
reached: { type: "string", required: true },
errorText: { type: "string" },
},
},
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] navigation wait on tab ${value.tabId}: ${value.reached}` +
(value.errorText !== undefined ? ` — ${value.errorText}` : ""),
},
],
},
async execute(args, exec) {
requirePositive(args.timeoutMs, "timeoutMs");
const sessionId = registry.resolve(args.session, "browser_wait_for_navigation");
const timeoutMs = args.timeoutMs ?? 30_000;
const cmdArgs = ["wait-for-navigation", "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
if (args.waitUntil !== undefined) cmdArgs.push("--wait-until", args.waitUntil);
cmdArgs.push("--timeout", `${timeoutMs}ms`);
const reply = (await runtime.run(
exec,
cmdArgs,
"wait-for-navigation",
sessionId,
runnerTimeout(deps, timeoutMs),
)) as { tab_id: number; reached: string; error_text?: string };
return {
session: sessionId,
tabId: reply.tab_id,
reached: reply.reached,
...(reply.error_text !== undefined ? { errorText: reply.error_text } : {}),
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"wait-for-navigation",
"--session",
args.session ?? "(current)",
]),
description: "Wait for page navigation",
}),
presentResult: runtime.presentTerminalResult,
}),
);
}
@@ -0,0 +1,672 @@
import { defineTool } from "@deepseek-ai/dsh-tools";
import {
appendTabId,
isSnapshotRef,
type PhaseOneRuntime,
requireNonEmpty,
requirePositive,
runnerTimeout,
type ToolRegistrar,
} from "./phase-one-runtime";
import { SESSION_PARAM, TAB_ID_PARAM, TIMEOUT_MS_PARAM } from "./tool-params";
import type { ToolDeps } from "./tools";
interface HelpConditionInput {
urlContains?: string;
urlMatches?: string;
selectorExists?: string;
selectorMissing?: string;
textExists?: string;
textMissing?: string;
}
interface CompletionCriteriaInput {
any?: HelpConditionInput[];
all?: HelpConditionInput[];
stableForMs?: number;
}
interface RawConsoleFrame {
function_name?: string;
url?: string;
line?: number;
column?: number;
}
interface RawConsoleEntry {
sequence: number;
kind: string;
level: string;
text: string;
url?: string;
line?: number;
column?: number;
timestamp?: number;
stack_trace?: RawConsoleFrame[];
truncated?: boolean;
}
interface RawNetworkEntry {
sequence: number;
kind: "response" | "failure";
method?: string;
url?: string;
status?: number;
status_text?: string;
mime_type?: string;
resource_type?: string;
error_text?: string;
timestamp?: number;
truncated?: boolean;
}
const HELP_CONDITION_SCHEMA = {
type: "object",
additionalProperties: false,
properties: {
urlContains: { type: "string" },
urlMatches: { type: "string" },
selectorExists: { type: "string" },
selectorMissing: { type: "string" },
textExists: { type: "string" },
textMissing: { type: "string" },
},
} as const;
const DEBUG_PARAMETERS = {
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
since: {
type: "integer",
description: "Return entries with a sequence strictly greater than this cursor.",
},
limit: {
type: "integer",
description: "Maximum entries to return (default: 50, daemon cap: 200).",
},
maxTextChars: {
type: "integer",
description: "Maximum characters per entry text/URL (default: 1000, daemon cap: 4096).",
},
} as const;
function mapCondition(condition: HelpConditionInput) {
return {
...(condition.urlContains !== undefined ? { url_contains: condition.urlContains } : {}),
...(condition.urlMatches !== undefined ? { url_matches: condition.urlMatches } : {}),
...(condition.selectorExists !== undefined
? { selector_exists: condition.selectorExists }
: {}),
...(condition.selectorMissing !== undefined
? { selector_missing: condition.selectorMissing }
: {}),
...(condition.textExists !== undefined ? { text_exists: condition.textExists } : {}),
...(condition.textMissing !== undefined ? { text_missing: condition.textMissing } : {}),
};
}
function mapCompletionCriteria(criteria: CompletionCriteriaInput) {
return {
...(criteria.any !== undefined ? { any: criteria.any.map(mapCondition) } : {}),
...(criteria.all !== undefined ? { all: criteria.all.map(mapCondition) } : {}),
...(criteria.stableForMs !== undefined ? { stable_for_ms: criteria.stableForMs } : {}),
};
}
function validateDebugArgs(args: { since?: number; limit?: number; maxTextChars?: number }): void {
if (args.since !== undefined && args.since < 0) {
throw new Error("since must be zero or greater");
}
requirePositive(args.limit, "limit");
requirePositive(args.maxTextChars, "maxTextChars");
}
function appendDebugOptions(
args: string[],
options: { tabId?: number; since?: number; limit?: number; maxTextChars?: number },
): void {
appendTabId(args, options.tabId);
if (options.since !== undefined) args.push("--since", String(options.since));
if (options.limit !== undefined) args.push("--limit", String(options.limit));
if (options.maxTextChars !== undefined) {
args.push("--max-text-chars", String(options.maxTextChars));
}
}
/** Human help, raw HTML, diagnostics, and Agent Window sizing. */
export function registerPhaseOneSupportTools(
deps: ToolDeps,
register: ToolRegistrar,
runtime: PhaseOneRuntime,
): void {
const { registry } = deps;
register(
defineTool({
name: "browser_request_help",
description:
"Pause browser automation and ask the user to complete an in-page step such as login, " +
"captcha, OTP, or confirmation. After continued/completed, observe again before using refs.",
parameters: {
prompt: {
type: "string",
required: true,
description: "Clear instructions shown to the user in the browser help overlay.",
},
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
title: { type: "string", description: "Optional title for the help overlay." },
targets: {
type: "array",
items: { type: "string" },
description: "Snapshot refs or CSS selectors to scroll to and highlight for the user.",
},
timeoutMs: {
...TIMEOUT_MS_PARAM,
description: "Maximum wait for the user in milliseconds (default: 300000).",
},
completionCriteria: {
type: "object",
additionalProperties: false,
description: "Optional explicit success detector for automatic completion.",
properties: {
any: { type: "array", items: HELP_CONDITION_SCHEMA },
all: { type: "array", items: HELP_CONDITION_SCHEMA },
stableForMs: {
type: "integer",
description: "How long criteria must remain true before completing.",
},
},
},
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
outcome: {
type: "string",
required: true,
enum: ["continued", "cancelled", "timed_out", "completed", "navigated", "disabled"],
},
completedBy: { type: "string" },
note: { type: "string" },
resolvedTargets: {
type: "array",
items: {
type: "object",
additionalProperties: false,
properties: {
matched: { type: "boolean", required: true },
ref: { type: "string" },
selector: { type: "string" },
},
},
},
},
},
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] user help outcome on tab ${value.tabId}: ${value.outcome}` +
(value.note !== undefined ? ` — ${value.note}` : "") +
(value.resolvedTargets !== undefined
? `\ntargets: ${value.resolvedTargets
.map(
(target) =>
`${target.ref ?? target.selector ?? "(unknown)"}=${target.matched ? "matched" : "not found"}`,
)
.join(", ")}`
: ""),
},
],
},
async execute(args, exec) {
requireNonEmpty(args.prompt, "prompt");
if (args.title !== undefined) requireNonEmpty(args.title, "title");
requirePositive(args.timeoutMs, "timeoutMs");
for (const target of args.targets ?? []) requireNonEmpty(target, "target");
const criteria = args.completionCriteria as CompletionCriteriaInput | undefined;
if (criteria?.stableForMs !== undefined && criteria.stableForMs < 0) {
throw new Error("completionCriteria.stableForMs must be zero or greater");
}
const sessionId = registry.resolve(args.session, "browser_request_help");
const timeoutMs = args.timeoutMs ?? 300_000;
const cmdArgs = [
"request-help",
"--session",
sessionId,
"--prompt",
args.prompt,
"--timeout",
`${timeoutMs}ms`,
];
appendTabId(cmdArgs, args.tabId);
if (args.title !== undefined) cmdArgs.push("--title", args.title);
for (const target of args.targets ?? []) cmdArgs.push("--target", target);
if (criteria !== undefined) {
cmdArgs.push("--completion-criteria", JSON.stringify(mapCompletionCriteria(criteria)));
}
const reply = (await runtime.run(
exec,
cmdArgs,
"request-help",
sessionId,
runnerTimeout(deps, timeoutMs),
)) as {
outcome: "continued" | "cancelled" | "timed_out" | "completed" | "navigated" | "disabled";
completed_by?: string;
note?: string;
tab_id: number;
resolved_targets?: { matched: boolean; ref?: string; selector?: string }[];
};
return {
session: sessionId,
tabId: reply.tab_id,
outcome: reply.outcome,
...(reply.completed_by !== undefined ? { completedBy: reply.completed_by } : {}),
...(reply.note !== undefined ? { note: reply.note } : {}),
...(reply.resolved_targets !== undefined
? { resolvedTargets: reply.resolved_targets }
: {}),
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine(["request-help", "--session", args.session ?? "(current)"]),
description: "Ask the user for browser help",
}),
presentResult: runtime.presentTerminalResult,
}),
);
register(
defineTool({
name: "browser_get_html",
description:
"Read raw DOM HTML from the active Agent Window tab, optionally scoped to a fresh " +
"snapshot ref. Use only when browser_observe/browser_snapshot cannot answer the question.",
parameters: {
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
ref: { type: "string", description: "Fresh snapshot ref that scopes the HTML subtree." },
maxBytes: {
type: "integer",
description: "Maximum returned HTML bytes before truncation (default: 524288).",
},
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
html: { type: "string", required: true },
truncated: { type: "boolean", required: true },
byteSize: { type: "integer", required: true },
},
},
render: (_args, value) => [
{
type: "text",
text:
value.html +
(value.truncated ? `\n(HTML truncated; original size ${value.byteSize} bytes)` : ""),
},
],
},
isConcurrencySafe: () => true,
async execute(args, exec) {
if (args.ref !== undefined) {
requireNonEmpty(args.ref, "ref");
if (!isSnapshotRef(args.ref)) throw new Error("ref must be a snapshot ref such as @e3");
}
requirePositive(args.maxBytes, "maxBytes");
const sessionId = registry.resolve(args.session, "browser_get_html");
const cmdArgs = ["get-html", "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
if (args.ref !== undefined) cmdArgs.push("--ref", args.ref);
if (args.maxBytes !== undefined) cmdArgs.push("--max-bytes", String(args.maxBytes));
const reply = (await runtime.run(exec, cmdArgs, "get-html", sessionId)) as {
tab_id: number;
html: string;
truncated?: boolean;
byte_size: number;
};
return {
session: sessionId,
tabId: reply.tab_id,
html: reply.html,
truncated: reply.truncated ?? false,
byteSize: reply.byte_size,
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"get-html",
"--session",
args.session ?? "(current)",
...(args.ref !== undefined ? ["--ref", args.ref] : []),
]),
description: "Read raw page HTML",
}),
presentResult: runtime.presentTerminalResult,
}),
);
register(
defineTool({
name: "browser_console",
description:
"Read buffered console messages, browser log entries, and JavaScript exceptions from a tab. " +
"This is read-only and does not evaluate JavaScript.",
parameters: {
...DEBUG_PARAMETERS,
includeStack: {
type: "boolean",
description: "Include structured stack frames in returned console entries.",
},
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
entries: {
type: "array",
required: true,
items: {
type: "object",
additionalProperties: false,
properties: {
sequence: { type: "integer", required: true },
kind: { type: "string", required: true },
level: { type: "string", required: true },
text: { type: "string", required: true },
url: { type: "string" },
line: { type: "integer" },
column: { type: "integer" },
timestamp: { type: "number" },
stackTrace: {
type: "array",
required: true,
items: {
type: "object",
additionalProperties: false,
properties: {
functionName: { type: "string" },
url: { type: "string" },
line: { type: "integer" },
column: { type: "integer" },
},
},
},
truncated: { type: "boolean", required: true },
},
},
},
nextSince: { type: "integer", required: true },
truncated: { type: "boolean", required: true },
},
},
render: (_args, value) => [
{
type: "text",
text:
value.entries.length === 0
? "(no console messages captured)"
: value.entries
.map((entry) => {
const location =
entry.url === undefined
? ""
: ` ${entry.url}${entry.line !== undefined ? `:${entry.line}` : ""}${entry.column !== undefined ? `:${entry.column}` : ""}`;
const stack = entry.stackTrace
.map(
(frame) =>
`\n at ${frame.functionName ?? "<anonymous>"} ${frame.url ?? ""}` +
`${frame.line !== undefined ? `:${frame.line}` : ""}` +
`${frame.column !== undefined ? `:${frame.column}` : ""}`,
)
.join("");
return `#${entry.sequence} ${entry.level} ${entry.kind}${location} ${entry.text}${stack}`;
})
.join("\n") +
(value.truncated
? `\n(output truncated; continue with since=${value.nextSince})`
: ""),
},
],
},
isConcurrencySafe: () => true,
async execute(args, exec) {
validateDebugArgs(args);
const sessionId = registry.resolve(args.session, "browser_console");
const cmdArgs = ["console", "--session", sessionId];
appendDebugOptions(cmdArgs, args);
if (args.includeStack === true) cmdArgs.push("--include-stack");
const reply = (await runtime.run(exec, cmdArgs, "console", sessionId)) as {
tab_id: number;
entries?: RawConsoleEntry[];
next_since: number;
truncated?: boolean;
};
return {
session: sessionId,
tabId: reply.tab_id,
entries: (reply.entries ?? []).map((entry) => ({
sequence: entry.sequence,
kind: entry.kind,
level: entry.level,
text: entry.text,
...(entry.url !== undefined ? { url: entry.url } : {}),
...(entry.line !== undefined ? { line: entry.line } : {}),
...(entry.column !== undefined ? { column: entry.column } : {}),
...(entry.timestamp !== undefined ? { timestamp: entry.timestamp } : {}),
stackTrace: (entry.stack_trace ?? []).map((frame) => ({
...(frame.function_name !== undefined ? { functionName: frame.function_name } : {}),
...(frame.url !== undefined ? { url: frame.url } : {}),
...(frame.line !== undefined ? { line: frame.line } : {}),
...(frame.column !== undefined ? { column: frame.column } : {}),
})),
truncated: entry.truncated ?? false,
})),
nextSince: reply.next_since,
truncated: reply.truncated ?? false,
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine(["console", "--session", args.session ?? "(current)"]),
description: "Read browser console messages",
}),
presentResult: runtime.presentTerminalResult,
}),
);
register(
defineTool({
name: "browser_network",
description:
"Read buffered network responses and failures from a tab. Returns metadata only; request " +
"and response headers and bodies are not captured.",
parameters: DEBUG_PARAMETERS,
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
entries: {
type: "array",
required: true,
items: {
type: "object",
additionalProperties: false,
properties: {
sequence: { type: "integer", required: true },
kind: { type: "string", required: true, enum: ["response", "failure"] },
method: { type: "string" },
url: { type: "string" },
status: { type: "integer" },
statusText: { type: "string" },
mimeType: { type: "string" },
resourceType: { type: "string" },
errorText: { type: "string" },
timestamp: { type: "number" },
truncated: { type: "boolean", required: true },
},
},
},
nextSince: { type: "integer", required: true },
truncated: { type: "boolean", required: true },
},
},
render: (_args, value) => [
{
type: "text",
text:
value.entries.length === 0
? "(no network activity captured)"
: value.entries
.map((entry) =>
entry.kind === "failure"
? `#${entry.sequence} FAILED ${entry.method ?? "?"} ${entry.url ?? "(unknown)"} — ${entry.errorText ?? "failed"}`
: `#${entry.sequence} ${entry.status ?? "?"} ${entry.method ?? "?"} ${entry.url ?? "(unknown)"}`,
)
.join("\n") +
(value.truncated
? `\n(output truncated; continue with since=${value.nextSince})`
: ""),
},
],
},
isConcurrencySafe: () => true,
async execute(args, exec) {
validateDebugArgs(args);
const sessionId = registry.resolve(args.session, "browser_network");
const cmdArgs = ["network", "--session", sessionId];
appendDebugOptions(cmdArgs, args);
const reply = (await runtime.run(exec, cmdArgs, "network", sessionId)) as {
tab_id: number;
entries?: RawNetworkEntry[];
next_since: number;
truncated?: boolean;
};
return {
session: sessionId,
tabId: reply.tab_id,
entries: (reply.entries ?? []).map((entry) => ({
sequence: entry.sequence,
kind: entry.kind,
...(entry.method !== undefined ? { method: entry.method } : {}),
...(entry.url !== undefined ? { url: entry.url } : {}),
...(entry.status !== undefined ? { status: entry.status } : {}),
...(entry.status_text !== undefined ? { statusText: entry.status_text } : {}),
...(entry.mime_type !== undefined ? { mimeType: entry.mime_type } : {}),
...(entry.resource_type !== undefined ? { resourceType: entry.resource_type } : {}),
...(entry.error_text !== undefined ? { errorText: entry.error_text } : {}),
...(entry.timestamp !== undefined ? { timestamp: entry.timestamp } : {}),
truncated: entry.truncated ?? false,
})),
nextSince: reply.next_since,
truncated: reply.truncated ?? false,
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine(["network", "--session", args.session ?? "(current)"]),
description: "Read browser network activity",
}),
presentResult: runtime.presentTerminalResult,
}),
);
register(
defineTool({
name: "browser_window_resize",
description: "Resize the owned session's Agent Window using outer CSS-pixel dimensions.",
parameters: {
width: {
type: "integer",
required: true,
description: "Agent Window outer width in CSS pixels (100..=7680).",
},
height: {
type: "integer",
required: true,
description: "Agent Window outer height in CSS pixels (100..=7680).",
},
session: SESSION_PARAM,
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
windowId: { type: "integer", required: true },
width: { type: "integer", required: true },
height: { type: "integer", required: true },
},
},
render: (_args, value) => [
{
type: "text",
text: `[session ${value.session}] resized window ${value.windowId} to ${value.width}x${value.height}`,
},
],
},
async execute(args, exec) {
if (args.width < 100 || args.width > 7680 || args.height < 100 || args.height > 7680) {
throw new Error("width and height must each be in the range 100..=7680");
}
const sessionId = registry.resolve(args.session, "browser_window_resize");
const reply = (await runtime.run(
exec,
[
"window",
"resize",
"--session",
sessionId,
"--width",
String(args.width),
"--height",
String(args.height),
],
"window resize",
sessionId,
)) as { window_id: number; width: number; height: number };
return {
session: sessionId,
windowId: reply.window_id,
width: reply.width,
height: reply.height,
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"window",
"resize",
"--session",
args.session ?? "(current)",
"--width",
String(args.width),
"--height",
String(args.height),
]),
description: "Resize the Agent Window",
}),
presentResult: runtime.presentTerminalResult,
}),
);
}
@@ -0,0 +1,402 @@
import { defineTool } from "@deepseek-ai/dsh-tools";
import type { PhaseOneRuntime, ToolRegistrar } from "./phase-one-runtime";
import { requireNonEmpty } from "./phase-one-runtime";
import { SESSION_PARAM } from "./tool-params";
import type { ToolDeps } from "./tools";
const TAB_ID_REQUIRED = {
type: "integer",
required: true,
description: "Chrome tab id returned by browser_tab_list or browser_tab_create.",
} as const;
interface RawTabInfo {
tab_id: number;
title?: string;
url?: string;
window_id?: number;
active?: boolean;
scope?: "user" | "agent";
}
/** Add Agent Window tab management while preserving the owned-session boundary. */
export function registerPhaseOneTabTools(
deps: ToolDeps,
register: ToolRegistrar,
runtime: PhaseOneRuntime,
): void {
const { registry } = deps;
register(
defineTool({
name: "browser_tab_list",
description:
"List tabs visible to an owned browser session. Other sessions' Agent Windows remain " +
"hidden; use scope=user to find a user tab before borrowing it.",
parameters: {
session: SESSION_PARAM,
scope: {
type: "string",
enum: ["user", "agent", "all"],
description: "Which tabs to list (default: all).",
},
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabs: {
type: "array",
required: true,
items: {
type: "object",
additionalProperties: false,
properties: {
tabId: { type: "integer", required: true },
title: { type: "string" },
url: { type: "string" },
windowId: { type: "integer" },
active: { type: "boolean" },
scope: { type: "string", enum: ["user", "agent"] },
},
},
},
},
},
render: (_args, value) => [
{
type: "text",
text:
value.tabs.length === 0
? `[session ${value.session}] no matching tabs`
: value.tabs
.map(
(tab) =>
`${tab.tabId} [${tab.scope ?? "unknown"}]${tab.active === true ? " [active]" : ""} ` +
`${tab.title ?? "(untitled)"} — ${tab.url ?? "(unknown URL)"}`,
)
.join("\n"),
},
],
},
isConcurrencySafe: () => true,
async execute(args, exec) {
const sessionId = registry.resolve(args.session, "browser_tab_list");
const cmdArgs = ["tab", "list", "--session", sessionId];
if (args.scope !== undefined) cmdArgs.push("--scope", args.scope);
const reply = (await runtime.run(exec, cmdArgs, "tab list", sessionId)) as {
tabs: RawTabInfo[];
};
return {
session: sessionId,
tabs: reply.tabs.map((tab) => ({
tabId: tab.tab_id,
...(tab.title !== undefined ? { title: tab.title } : {}),
...(tab.url !== undefined ? { url: tab.url } : {}),
...(tab.window_id !== undefined ? { windowId: tab.window_id } : {}),
...(tab.active !== undefined ? { active: tab.active } : {}),
...(tab.scope !== undefined ? { scope: tab.scope } : {}),
})),
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"tab",
"list",
"--session",
args.session ?? "(current)",
...(args.scope !== undefined ? ["--scope", args.scope] : []),
]),
description: "List browser tabs",
}),
presentResult: runtime.presentTerminalResult,
}),
);
register(
defineTool({
name: "browser_tab_create",
description:
"Create a tab inside the owned session's Agent Window. The tab is focused by default; " +
"set active=false to open it in the background.",
parameters: {
session: SESSION_PARAM,
url: { type: "string", description: "Initial URL (default: chrome://newtab/)." },
active: { type: "boolean", description: "Whether to focus the new tab (default: true)." },
index: { type: "integer", description: "Insertion index in the Agent Window tab strip." },
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
windowId: { type: "integer", required: true },
url: { type: "string", required: true },
},
},
render: (_args, value) => [
{
type: "text",
text: `[session ${value.session}] created tab ${value.tabId} in window ${value.windowId} (${value.url || "pending URL"})`,
},
],
},
async execute(args, exec) {
if (args.url !== undefined) requireNonEmpty(args.url, "url");
const sessionId = registry.resolve(args.session, "browser_tab_create");
const cmdArgs = ["tab", "create", "--session", sessionId];
if (args.url !== undefined) cmdArgs.push("--url", args.url);
if (args.active === false) cmdArgs.push("--no-active");
if (args.index !== undefined) cmdArgs.push("--index", String(args.index));
const reply = (await runtime.run(exec, cmdArgs, "tab create", sessionId)) as {
tab_id: number;
window_id: number;
url: string;
};
if (args.active !== false && reply.url.length > 0) {
deps.observation.setUrl(sessionId, reply.url);
}
return {
session: sessionId,
tabId: reply.tab_id,
windowId: reply.window_id,
url: reply.url,
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"tab",
"create",
"--session",
args.session ?? "(current)",
...(args.url !== undefined ? ["--url", args.url] : []),
]),
description: "Create an Agent Window tab",
}),
presentResult: runtime.presentTerminalResult,
}),
);
register(
defineTool({
name: "browser_tab_close",
description: "Close a tab in the owned session's Agent Window.",
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
},
},
render: (_args, value) => [
{ type: "text", text: `[session ${value.session}] closed tab ${value.tabId}` },
],
},
async execute(args, exec) {
const sessionId = registry.resolve(args.session, "browser_tab_close");
const reply = (await runtime.run(
exec,
["tab", "close", String(args.tabId), "--session", sessionId],
"tab close",
sessionId,
)) as { tab_id: number };
return { session: sessionId, tabId: reply.tab_id };
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"tab",
"close",
String(args.tabId),
"--session",
args.session ?? "(current)",
]),
description: "Close an Agent Window tab",
}),
presentResult: runtime.presentTerminalResult,
}),
);
register(
defineTool({
name: "browser_tab_select",
description: "Focus a tab in the owned session's Agent Window.",
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
windowId: { type: "integer", required: true },
},
},
render: (_args, value) => [
{
type: "text",
text: `[session ${value.session}] selected tab ${value.tabId} in window ${value.windowId}`,
},
],
},
async execute(args, exec) {
const sessionId = registry.resolve(args.session, "browser_tab_select");
const reply = (await runtime.run(
exec,
["tab", "select", String(args.tabId), "--session", sessionId],
"tab select",
sessionId,
)) as { tab_id: number; window_id: number };
return { session: sessionId, tabId: reply.tab_id, windowId: reply.window_id };
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"tab",
"select",
String(args.tabId),
"--session",
args.session ?? "(current)",
]),
description: "Select an Agent Window tab",
}),
presentResult: runtime.presentTerminalResult,
}),
);
register(
defineTool({
name: "browser_tab_borrow",
description:
"Move a user-window tab into the owned session's Agent Window for controlled interaction. " +
"Return it with browser_tab_return when finished; stopping the session also auto-returns it.",
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
originalWindowId: { type: "integer", required: true },
originalIndex: { type: "integer", required: true },
agentWindowId: { type: "integer", required: true },
},
},
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] borrowed tab ${value.tabId} from window ` +
`${value.originalWindowId} into Agent Window ${value.agentWindowId}`,
},
],
},
async execute(args, exec) {
const sessionId = registry.resolve(args.session, "browser_tab_borrow");
const reply = (await runtime.run(
exec,
["tab", "borrow", String(args.tabId), "--session", sessionId],
"tab borrow",
sessionId,
)) as {
tab_id: number;
original_window_id: number;
original_index: number;
agent_window_id: number;
};
return {
session: sessionId,
tabId: reply.tab_id,
originalWindowId: reply.original_window_id,
originalIndex: reply.original_index,
agentWindowId: reply.agent_window_id,
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"tab",
"borrow",
String(args.tabId),
"--session",
args.session ?? "(current)",
]),
description: "Borrow a user tab",
}),
presentResult: runtime.presentTerminalResult,
}),
);
register(
defineTool({
name: "browser_tab_return",
description: "Return a borrowed tab to its original user window and tab-strip position.",
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
returnedToWindowId: { type: "integer", required: true },
returnedToIndex: { type: "integer", required: true },
fallback: { type: "boolean", required: true },
},
},
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] returned tab ${value.tabId} to window ` +
`${value.returnedToWindowId} at index ${value.returnedToIndex}` +
(value.fallback ? " (original window unavailable; used fallback)" : ""),
},
],
},
async execute(args, exec) {
const sessionId = registry.resolve(args.session, "browser_tab_return");
const reply = (await runtime.run(
exec,
["tab", "return", String(args.tabId), "--session", sessionId],
"tab return",
sessionId,
)) as {
tab_id: number;
returned_to_window_id: number;
returned_to_index: number;
fallback?: boolean;
};
return {
session: sessionId,
tabId: reply.tab_id,
returnedToWindowId: reply.returned_to_window_id,
returnedToIndex: reply.returned_to_index,
fallback: reply.fallback ?? false,
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"tab",
"return",
String(args.tabId),
"--session",
args.session ?? "(current)",
]),
description: "Return a borrowed tab",
}),
presentResult: runtime.presentTerminalResult,
}),
);
}
@@ -0,0 +1,18 @@
import type { PhaseOneRuntime, ToolRegistrar } from "./phase-one-runtime";
import { registerPhaseOneInteractionTools } from "./phase-one-tools-interaction";
import { registerPhaseOneNavigationTools } from "./phase-one-tools-navigation";
import { registerPhaseOneSupportTools } from "./phase-one-tools-support";
import { registerPhaseOneTabTools } from "./phase-one-tools-tabs";
import type { ToolDeps } from "./tools";
/** Register the first DSH capability-parity tranche on the existing runtime. */
export function registerPhaseOneTools(
deps: ToolDeps,
register: ToolRegistrar,
runtime: PhaseOneRuntime,
): void {
registerPhaseOneInteractionTools(deps, register, runtime);
registerPhaseOneTabTools(deps, register, runtime);
registerPhaseOneNavigationTools(deps, register, runtime);
registerPhaseOneSupportTools(deps, register, runtime);
}

Some files were not shown because too many files have changed in this diff Show More